Appearance
| 1 | namespace Syntax.Parsers is | |
| 2 | use IO.Std; | |
| 3 | ||
| 4 | use Logging; | |
| 5 | use Source; | |
| 6 | ||
| 7 | use Ghul.Pipes; | |
| 8 | ||
| 9 | // Reference type, not a struct: `let use` disposal and the explicit | |
| 10 | // commit()/backtrack()/cancel() calls must act on the same instance, or | |
| 11 | // a speculation can be committed twice (once explicitly, once by the | |
| 12 | // disposer working from a stale copy) — an unbalanced commit underflows | |
| 13 | // the token queue's mark stack and leaves speculation mode stuck on. | |
| 14 | class TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT: Disposable is | |
| 15 | _context: CONTEXT?; | |
| 16 | ||
| 17 | is_speculating: bool => _context?; | |
| 18 | ||
| 19 | init(context: CONTEXT) is | |
| 20 | _context = context; | |
| 21 | _context.tokenizer_speculate(); | |
| 22 | si | |
| 23 | ||
| 24 | // _context is cleared before the delegated call, not after: if | |
| 25 | // tokenizer_backtrack/commit throws (a detected speculation loop | |
| 26 | // does throw), the disposer must not run a second time and | |
| 27 | // unbalance the token queue's mark stack. | |
| 28 | backtrack() is | |
| 29 | let context = _context; | |
| 30 | _context = null; | |
| 31 | context!.tokenizer_backtrack(); | |
| 32 | si | |
| 33 | ||
| 34 | backtrack_if_speculating() is | |
| 35 | if _context? then | |
| 36 | backtrack(); | |
| 37 | fi | |
| 38 | si | |
| 39 | ||
| 40 | commit() is | |
| 41 | let context = _context; | |
| 42 | _context = null; | |
| 43 | context!.tokenizer_commit(); | |
| 44 | si | |
| 45 | ||
| 46 | commit_if_speculating() is | |
| 47 | if _context? then | |
| 48 | commit(); | |
| 49 | fi | |
| 50 | si | |
| 51 | ||
| 52 | cancel() is | |
| 53 | _context = null; | |
| 54 | si | |
| 55 | ||
| 56 | dispose() is | |
| 57 | commit_if_speculating(); | |
| 58 | si | |
| 59 | si | |
| 60 | ||
| 61 | // Reference type, not a struct — see TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT. | |
| 62 | class TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK: Disposable is | |
| 63 | _context: CONTEXT?; | |
| 64 | ||
| 65 | // False for a single, provably-bounded probe whose backtrack must not | |
| 66 | // feed the speculation-loop detector (see TOKEN_LOOKAHEAD.backtrack). | |
| 67 | _check_for_loop: bool; | |
| 68 | ||
| 69 | is_speculating: bool => _context?; | |
| 70 | ||
| 71 | init(context: CONTEXT) is | |
| 72 | init(context, true); | |
| 73 | si | |
| 74 | ||
| 75 | init(context: CONTEXT, check_for_loop: bool) is | |
| 76 | _context = context; | |
| 77 | _check_for_loop = check_for_loop; | |
| 78 | _context.tokenizer_speculate(); | |
| 79 | si | |
| 80 | ||
| 81 | backtrack() is | |
| 82 | let context = _context; | |
| 83 | _context = null; | |
| 84 | context!.tokenizer_backtrack(_check_for_loop); | |
| 85 | si | |
| 86 | ||
| 87 | backtrack_if_speculating() is | |
| 88 | if _context? then | |
| 89 | backtrack(); | |
| 90 | fi | |
| 91 | si | |
| 92 | ||
| 93 | commit() is | |
| 94 | let context = _context; | |
| 95 | _context = null; | |
| 96 | context!.tokenizer_commit(); | |
| 97 | si | |
| 98 | ||
| 99 | commit_if_speculating() is | |
| 100 | if _context? then | |
| 101 | commit(); | |
| 102 | fi | |
| 103 | si | |
| 104 | ||
| 105 | cancel() is | |
| 106 | _context = null; | |
| 107 | si | |
| 108 | ||
| 109 | dispose() is | |
| 110 | backtrack_if_speculating(); | |
| 111 | si | |
| 112 | si | |
| 113 | ||
| 114 | // Reference type, not a struct — see TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT. | |
| 115 | // | |
| 116 | // Speculate the diagnostics stream and the repeated-error recovery | |
| 117 | // cursor as one unit. The cursor — last_error_location / | |
| 118 | // last_error_message on CONTEXT — drives the recovery in | |
| 119 | // CONTEXT.error(): on an exact-repeat error it drops the duplicate and | |
| 120 | // consumes a token to force progress. A speculative parse that raises | |
| 121 | // an error and is then abandoned has to rewind that cursor along with | |
| 122 | // the diagnostics, or the real parse of the same tokens raises the same | |
| 123 | // error, error() reads it as a repeat, and recovery eats both the | |
| 124 | // diagnostic and a token. Capturing the cursor here makes the rewind | |
| 125 | // automatic: backtrack() (and dispose) restore it, commit() keeps it. | |
| 126 | class DIAGNOSTICS_SPECULATE_THEN_BACKTRACK: Disposable is | |
| 127 | _context: CONTEXT?; | |
| 128 | _last_error_location: LOCATION; | |
| 129 | _last_error_message: string; | |
| 130 | ||
| 131 | is_speculating: bool => _context?; | |
| 132 | ||
| 133 | init(context: CONTEXT) is | |
| 134 | _context = context; | |
| 135 | _last_error_location = context.last_error_location; | |
| 136 | _last_error_message = context.last_error_message; | |
| 137 | _context.logger_speculate(); | |
| 138 | si | |
| 139 | ||
| 140 | backtrack() is | |
| 141 | let context = _context; | |
| 142 | _context = null; | |
| 143 | context!.logger_backtrack(); | |
| 144 | context.restore_last_error(_last_error_location, _last_error_message); | |
| 145 | si | |
| 146 | ||
| 147 | backtrack_if_speculating() is | |
| 148 | if _context? then | |
| 149 | backtrack(); | |
| 150 | fi | |
| 151 | si | |
| 152 | ||
| 153 | commit() is | |
| 154 | let context = _context; | |
| 155 | _context = null; | |
| 156 | context!.logger_commit(); | |
| 157 | si | |
| 158 | ||
| 159 | commit_if_speculating() is | |
| 160 | if _context? then | |
| 161 | commit(); | |
| 162 | fi | |
| 163 | si | |
| 164 | ||
| 165 | cancel() is | |
| 166 | _context = null; | |
| 167 | si | |
| 168 | ||
| 169 | dispose() is | |
| 170 | backtrack_if_speculating(); | |
| 171 | si | |
| 172 | si | |
| 173 | ||
| 174 | class CONTEXT is | |
| 175 | _repeated_error_count_at_eof: int; | |
| 176 | ||
| 177 | last_error_location: LOCATION; | |
| 178 | last_error_message: string; | |
| 179 | ||
| 180 | allow_tuple_element: bool public; | |
| 181 | in_trait: bool public; | |
| 182 | ||
| 183 | in_global_function: bool public; | |
| 184 | in_classy: bool public; | |
| 185 | in_member: bool public; | |
| 186 | ||
| 187 | // Depth of enclosing `namespace ... is ... si` blocks. Zero only at | |
| 188 | // the file root, where a file with no namespace may carry bare | |
| 189 | // top-level statements collected into a synthesised entry point. | |
| 190 | // GLOBAL_LIST (reused as the namespace body parser) consults this to | |
| 191 | // confine that handling to depth 0; inside any namespace it behaves | |
| 192 | // exactly as before. | |
| 193 | namespace_depth: int public; | |
| 194 | ||
| 195 | // True only while parsing the `[ ... ]` type-parameter list of a | |
| 196 | // generic type declaration. Distinguishes a type-parameter | |
| 197 | // declaration from a named-tuple-element type (both produce | |
| 198 | // NAMED_TUPLE_ELEMENT via the same parse path) so the `out` / | |
| 199 | // `in` variance modifier is recognised only on a type parameter. | |
| 200 | in_type_parameters: bool public; | |
| 201 | ||
| 202 | // True only while parsing the formal-argument list of an | |
| 203 | // `init` method. The `..` splice marker is permitted there | |
| 204 | // (it's expanded by the primary-constructor rewrite); anywhere | |
| 205 | // else `..` in a variable-list position is a syntax error. | |
| 206 | in_init_arguments: bool public; | |
| 207 | ||
| 208 | // True only while parsing the primary-ctor parameter list of a | |
| 209 | // class or struct header. The variable parser consults this to | |
| 210 | // decide whether to read a trailing modifier list on each | |
| 211 | // parameter — in other variable-parsing contexts (`let`, formal | |
| 212 | // args, secondary inits) it leaves the modifier slot alone, so | |
| 213 | // a stray `public` / `init` there falls through to the caller's | |
| 214 | // normal "unexpected token" handling. The modifier parser itself | |
| 215 | // matches `init` unconditionally; gating happens only here at | |
| 216 | // the variable-parser layer. | |
| 217 | in_primary_ctor_params: bool public; | |
| 218 | ||
| 219 | // True only while parsing the formal-argument list of a | |
| 220 | // function or method definition. The variable parser consults | |
| 221 | // this to decide whether an `@` before a parameter starts an | |
| 222 | // attribute pragma — elsewhere (`let`, primary-ctor headers, | |
| 223 | // variant field lists) a stray `@` falls through to the | |
| 224 | // caller's normal "unexpected token" handling. | |
| 225 | in_formal_arguments: bool public; | |
| 226 | ||
| 227 | // intentation is not part of the syntax, but it is used for | |
| 228 | // for error recovery: we may want to skip over a block of code | |
| 229 | // out to a given indentation level | |
| 230 | global_indent: int public; | |
| 231 | member_indent: int public; | |
| 232 | ||
| 233 | tokenizer: Lexical.TOKEN_LOOKAHEAD; | |
| 234 | logger: Logger; | |
| 235 | ||
| 236 | location: LOCATION => current.location; | |
| 237 | ||
| 238 | is_end_of_file: bool; | |
| 239 | ||
| 240 | current: Lexical.TOKEN_PAIR; | |
| 241 | ||
| 242 | current_token: Lexical.TOKEN => current.token; | |
| 243 | ||
| 244 | current_string: string? => current.value_string; | |
| 245 | ||
| 246 | current_token_name: string is | |
| 247 | let result mut = Lexical.TOKEN_NAMES[current_token]; | |
| 248 | ||
| 249 | if result == null then | |
| 250 | result = "unknown"; | |
| 251 | fi | |
| 252 | ||
| 253 | if current_string? /\ current_string !~ result then | |
| 254 | result = "{result} {current_string}"; | |
| 255 | fi | |
| 256 | ||
| 257 | return result; | |
| 258 | si | |
| 259 | ||
| 260 | init(tokenizer: Lexical.TOKEN_LOOKAHEAD, logger: Logger) is | |
| 261 | self.tokenizer = tokenizer; | |
| 262 | self.logger = logger; | |
| 263 | ||
| 264 | current = tokenizer.read_token(); | |
| 265 | ||
| 266 | // next_token() sets this on every read after the first; the | |
| 267 | // first-token read here has to match. | |
| 268 | is_end_of_file = current.token == Lexical.TOKEN.END_OF_INPUT; | |
| 269 | ||
| 270 | last_error_location = self.location; | |
| 271 | last_error_message = ""; | |
| 272 | si | |
| 273 | ||
| 274 | tokenizer_speculate_then_commit() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT => | |
| 275 | TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT(self); | |
| 276 | ||
| 277 | tokenizer_speculate_then_backtrack() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK => | |
| 278 | TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK(self); | |
| 279 | ||
| 280 | // A single, provably-bounded probe: its backtrack does not feed the | |
| 281 | // speculation-loop detector. | |
| 282 | tokenizer_speculate_then_backtrack_bounded() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK => | |
| 283 | TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK(self, false); | |
| 284 | ||
| 285 | logger_speculate_then_commit() -> LOGGER_SPECULATE_THEN_COMMIT => logger.speculate_then_commit(); | |
| 286 | ||
| 287 | diagnostics_speculate_then_backtrack() -> DIAGNOSTICS_SPECULATE_THEN_BACKTRACK => | |
| 288 | DIAGNOSTICS_SPECULATE_THEN_BACKTRACK(self); | |
| 289 | ||
| 290 | tokenizer_speculate() is | |
| 291 | tokenizer.speculate(); | |
| 292 | si | |
| 293 | ||
| 294 | tokenizer_commit() is | |
| 295 | tokenizer.commit(); | |
| 296 | si | |
| 297 | ||
| 298 | tokenizer_backtrack() is | |
| 299 | current = tokenizer.backtrack(); | |
| 300 | si | |
| 301 | ||
| 302 | tokenizer_backtrack(check_for_loop: bool) is | |
| 303 | current = tokenizer.backtrack(check_for_loop); | |
| 304 | si | |
| 305 | ||
| 306 | logger_speculate() is | |
| 307 | logger.speculate(); | |
| 308 | si | |
| 309 | ||
| 310 | logger_commit() is | |
| 311 | logger.commit(); | |
| 312 | si | |
| 313 | ||
| 314 | logger_backtrack() -> DIAGNOSTICS_STATE => logger.roll_back(); | |
| 315 | ||
| 316 | // Rewind the repeated-error recovery cursor read by error() below. | |
| 317 | // Called by DIAGNOSTICS_SPECULATE_THEN_BACKTRACK when a speculative | |
| 318 | // parse is abandoned; see that class for why the cursor has to move | |
| 319 | // in step with the diagnostics. | |
| 320 | restore_last_error(location: LOCATION, message: string) is | |
| 321 | last_error_location = location; | |
| 322 | last_error_message = message; | |
| 323 | si | |
| 324 | expect_format_specifier() is | |
| 325 | tokenizer.expect_format_specifier(); | |
| 326 | si | |
| 327 | ||
| 328 | next_token() -> bool is | |
| 329 | current = tokenizer.read_token(); | |
| 330 | ||
| 331 | is_end_of_file = current.token == Lexical.TOKEN.END_OF_INPUT; | |
| 332 | ||
| 333 | return is_end_of_file; | |
| 334 | si | |
| 335 | ||
| 336 | next_token(token: Lexical.TOKEN, message: string) -> bool is | |
| 337 | if expect_token(token, message) then | |
| 338 | next_token(); | |
| 339 | return true; | |
| 340 | fi | |
| 341 | return false; | |
| 342 | si | |
| 343 | ||
| 344 | next_token(tokens: Collections.List[Lexical.TOKEN], message: string) -> bool is | |
| 345 | if expect_token(tokens, message) then | |
| 346 | next_token(); | |
| 347 | return true; | |
| 348 | fi | |
| 349 | return false; | |
| 350 | si | |
| 351 | ||
| 352 | next_token(token: Lexical.TOKEN) -> bool => next_token(token, "syntax error"); | |
| 353 | next_token(tokens: Collections.List[Lexical.TOKEN]) -> bool => next_token(tokens, "syntax error"); | |
| 354 | location_and_next() -> LOCATION is | |
| 355 | let result = location; | |
| 356 | next_token(); | |
| 357 | return result; | |
| 358 | si | |
| 359 | ||
| 360 | skip_token(tokens: Collections.List[Lexical.TOKEN], message: string) is | |
| 361 | let start = location; | |
| 362 | ||
| 363 | do | |
| 364 | if is_end_of_file then | |
| 365 | error(start::location, "{message}: expected {Lexical.TOKEN_NAMES[current_token]}"); | |
| 366 | return; | |
| 367 | elif tokens |> any(t => t == current_token) then | |
| 368 | error(start::location, "{message}: expected {Lexical.TOKEN_NAMES[current_token]}"); | |
| 369 | next_token(); | |
| 370 | return; | |
| 371 | else | |
| 372 | next_token(); | |
| 373 | fi | |
| 374 | od | |
| 375 | si | |
| 376 | ||
| 377 | skip_token(token: Lexical.TOKEN, message: string) is | |
| 378 | let t = Collections.LIST[Lexical.TOKEN](); | |
| 379 | t.add(token); | |
| 380 | skip_token(t, message); | |
| 381 | si | |
| 382 | ||
| 383 | expect_token(token: Lexical.TOKEN, message: string) -> bool is | |
| 384 | if current_token != token then | |
| 385 | if current_token != Lexical.TOKEN.CANCEL_STRING then | |
| 386 | error(location, "{message}: expected {Lexical.TOKEN_NAMES[token]} but found {current_token_name}"); | |
| 387 | fi | |
| 388 | ||
| 389 | return false; | |
| 390 | else | |
| 391 | return true; | |
| 392 | fi | |
| 393 | si | |
| 394 | ||
| 395 | expect_token(token: Lexical.TOKEN) -> bool => expect_token(token, "syntax error"); | |
| 396 | expect_token(tokens: Collections.List[Lexical.TOKEN], message: string) -> bool is | |
| 397 | if !tokens |> any(t => t == current_token) then | |
| 398 | if current_token != Lexical.TOKEN.CANCEL_STRING then | |
| 399 | error(location, "{message}: expected {Lexical.TOKEN_NAMES[tokens]} but found {current_token_name}"); | |
| 400 | fi | |
| 401 | ||
| 402 | return false; | |
| 403 | else | |
| 404 | return true; | |
| 405 | fi | |
| 406 | si | |
| 407 | ||
| 408 | expect_token(tokens: Collections.List[Lexical.TOKEN]) -> bool => | |
| 409 | expect_token(tokens, "syntax error"); | |
| 410 | ||
| 411 | warn(location: LOCATION, message: string) is | |
| 412 | logger.warn(location, message); | |
| 413 | si | |
| 414 | ||
| 415 | error(location: LOCATION, message: string) is | |
| 416 | if location !~ last_error_location \/ message !~ last_error_message then | |
| 417 | _repeated_error_count_at_eof = 0; | |
| 418 | ||
| 419 | last_error_location = location; | |
| 420 | last_error_message = message; | |
| 421 | ||
| 422 | logger.error(location, message); | |
| 423 | elif !is_end_of_file then | |
| 424 | next_token(); | |
| 425 | else | |
| 426 | _repeated_error_count_at_eof = _repeated_error_count_at_eof + 1; | |
| 427 | ||
| 428 | if (_repeated_error_count_at_eof > 10) then | |
| 429 | throw Compiler.PARSE_EXCEPTION("repeated errors at end of file"); | |
| 430 | fi | |
| 431 | fi | |
| 432 | si | |
| 433 | si | |
| 434 | si |