Appearance
| 1 | namespace Syntax.Parsers.Definitions is | |
| 2 | use System.Exception; | |
| 3 | use IO.Std; | |
| 4 | ||
| 5 | use Source; | |
| 6 | ||
| 7 | class GLOBAL_LIST( | |
| 8 | definition_parser: Parser[Trees.Definitions.Definition], | |
| 9 | statement_parser: Parser[Trees.Statements.Statement] | |
| 10 | ): Base[Trees.Definitions.LIST] is | |
| 11 | description: string => "definition list"; | |
| 12 | ||
| 13 | super(); | |
| 14 | ||
| 15 | parse(context: CONTEXT) -> Trees.Definitions.LIST is | |
| 16 | let start = context.location; | |
| 17 | let end mut = context.location; | |
| 18 | let definitions = Collections.LIST[Trees.Definitions.Definition](); | |
| 19 | let statements = Collections.LIST[Trees.Statements.Statement](); | |
| 20 | ||
| 21 | // `si` closes a namespace body (depth > 0); at the file root | |
| 22 | // (depth 0) it is stray input the definition parser reports, so it | |
| 23 | // must not end the loop there. | |
| 24 | while | |
| 25 | !context.is_end_of_file /\ | |
| 26 | (context.namespace_depth == 0 \/ context.current.token != Lexical.TOKEN.SI) | |
| 27 | do | |
| 28 | try | |
| 29 | // Script mode: at the file root (no enclosing namespace) a | |
| 30 | // leading token that isn't a definition keyword may begin a | |
| 31 | // top-level statement. Inside any namespace this is skipped | |
| 32 | // and parsing is unchanged. | |
| 33 | if context.namespace_depth == 0 /\ !_is_definition_head(context.current.token) then | |
| 34 | if _is_ambiguous_head(context.current.token) then | |
| 35 | // IDENTIFIER / OPERATOR / [ can each begin a global | |
| 36 | // member or a statement; a shallow peek routes the | |
| 37 | // clear definitions. The rest are parsed as a | |
| 38 | // statement, but speculatively: an incomplete | |
| 39 | // definition that the peek couldn't recognise (e.g. | |
| 40 | // `foo(` with no close) fails to parse as a statement | |
| 41 | // and falls back to the definition parser's recovery. | |
| 42 | if _starts_definition(context) \/ !_try_collect_statement(context, statements) then | |
| 43 | _parse_definition(context, definitions); | |
| 44 | end = _end_of(definitions, end); | |
| 45 | else | |
| 46 | end = _end_of_statement(statements, end); | |
| 47 | fi | |
| 48 | elif _is_statement_head(context.current.token) then | |
| 49 | _parse_statement(context, statements); | |
| 50 | end = _end_of_statement(statements, end); | |
| 51 | else | |
| 52 | // Not a definition keyword and not a statement head: | |
| 53 | // stray input, reported by the definition parser as | |
| 54 | // before. | |
| 55 | _parse_definition(context, definitions); | |
| 56 | end = _end_of(definitions, end); | |
| 57 | fi | |
| 58 | else | |
| 59 | _parse_definition(context, definitions); | |
| 60 | end = _end_of(definitions, end); | |
| 61 | fi | |
| 62 | catch ue: UNWIND_TO_GLOBAL_EXCEPTION | |
| 63 | // carry on from here | |
| 64 | catch e: Exception | |
| 65 | IoC.CONTAINER.instance.logger.exception(context.current.location, e, "parse exception: {e.message}"); | |
| 66 | ||
| 67 | while | |
| 68 | !context.is_end_of_file /\ | |
| 69 | context.current.token != Lexical.TOKEN.SI /\ | |
| 70 | context.current.token != Lexical.TOKEN.SEMICOLON | |
| 71 | do | |
| 72 | context.next_token(); | |
| 73 | od | |
| 74 | ||
| 75 | if context.is_end_of_file then | |
| 76 | break; | |
| 77 | fi | |
| 78 | yrt | |
| 79 | od | |
| 80 | ||
| 81 | let result = Trees.Definitions.LIST(start::end, definitions); | |
| 82 | ||
| 83 | // Hand any bare top-level statements to synthesise-top-level-entry, | |
| 84 | // which wraps them into a global entry point. Only reachable at the | |
| 85 | // file root (namespace depth 0). | |
| 86 | if statements.count > 0 then | |
| 87 | result.top_level_statements = Trees.Statements.LIST(start::end, statements); | |
| 88 | fi | |
| 89 | ||
| 90 | return result; | |
| 91 | si | |
| 92 | ||
| 93 | _parse_definition(context: CONTEXT, definitions: Collections.MutableList[Trees.Definitions.Definition]) is | |
| 94 | let definition = definition_parser.parse(context); | |
| 95 | ||
| 96 | if definition? then | |
| 97 | definitions.add(definition); | |
| 98 | fi | |
| 99 | si | |
| 100 | ||
| 101 | _parse_statement(context: CONTEXT, statements: Collections.MutableList[Trees.Statements.Statement]) is | |
| 102 | let statement = statement_parser.parse(context); | |
| 103 | ||
| 104 | if statement? then | |
| 105 | statements.add(statement); | |
| 106 | _skip_semicolon(context); | |
| 107 | elif !context.is_end_of_file then | |
| 108 | context.next_token(); | |
| 109 | fi | |
| 110 | si | |
| 111 | ||
| 112 | // Speculatively parse a statement at an ambiguous head, collecting it | |
| 113 | // and returning true only if it parses cleanly to a statement boundary. | |
| 114 | // Otherwise the tokens are rewound — tokenizer (the exempt bounded-probe | |
| 115 | // kind, so it does not feed the loop detector) and diagnostics — so the | |
| 116 | // caller can hand them to the definition parser instead. A well-formed | |
| 117 | // call commits with no rewind; only an incomplete definition misrouted | |
| 118 | // here takes the fallback. | |
| 119 | _try_collect_statement(context: CONTEXT, statements: Collections.MutableList[Trees.Statements.Statement]) -> bool is | |
| 120 | let use diagnostics_snapshot = context.diagnostics_speculate_then_backtrack(); | |
| 121 | let use snapshot = context.tokenizer_speculate_then_backtrack_bounded(); | |
| 122 | ||
| 123 | let errors_before = context.logger.error_count; | |
| 124 | ||
| 125 | let statement = statement_parser.parse(context); | |
| 126 | ||
| 127 | if | |
| 128 | statement? /\ | |
| 129 | !statement.is_poisoned /\ | |
| 130 | context.logger.error_count == errors_before /\ | |
| 131 | ( | |
| 132 | context.current.token == Lexical.TOKEN.SEMICOLON \/ | |
| 133 | context.current.token == Lexical.TOKEN.SI \/ | |
| 134 | context.is_end_of_file | |
| 135 | ) | |
| 136 | then | |
| 137 | snapshot.commit(); | |
| 138 | diagnostics_snapshot.commit(); | |
| 139 | ||
| 140 | statements.add(statement); | |
| 141 | _skip_semicolon(context); | |
| 142 | ||
| 143 | return true; | |
| 144 | fi | |
| 145 | ||
| 146 | return false; | |
| 147 | si | |
| 148 | ||
| 149 | _end_of(definitions: Collections.List[Trees.Definitions.Definition], previous: Source.LOCATION) -> Source.LOCATION => | |
| 150 | if definitions.count > 0 then definitions[definitions.count - 1].location else previous fi; | |
| 151 | ||
| 152 | _end_of_statement(statements: Collections.List[Trees.Statements.Statement], previous: Source.LOCATION) -> Source.LOCATION => | |
| 153 | if statements.count > 0 then statements[statements.count - 1].location else previous fi; | |
| 154 | ||
| 155 | // Definition-introducing keywords: everything the definition parser | |
| 156 | // dispatches on that a statement can never begin with. The three | |
| 157 | // tokens a statement *can* also begin with (IDENTIFIER, OPERATOR, | |
| 158 | // SQUARE_OPEN) are deliberately excluded and disambiguated instead. | |
| 159 | _is_definition_head(token: Lexical.TOKEN) -> bool => | |
| 160 | token == Lexical.TOKEN.NAMESPACE \/ | |
| 161 | token == Lexical.TOKEN.USE \/ | |
| 162 | token == Lexical.TOKEN.CLASS \/ | |
| 163 | token == Lexical.TOKEN.TRAIT \/ | |
| 164 | token == Lexical.TOKEN.STRUCT \/ | |
| 165 | token == Lexical.TOKEN.PARTIAL \/ | |
| 166 | token == Lexical.TOKEN.IMPL \/ | |
| 167 | token == Lexical.TOKEN.UNION \/ | |
| 168 | token == Lexical.TOKEN.ENUM \/ | |
| 169 | token == Lexical.TOKEN.AT; | |
| 170 | ||
| 171 | // The tokens shared between a global-member definition and a statement. | |
| 172 | _is_ambiguous_head(token: Lexical.TOKEN) -> bool => | |
| 173 | token == Lexical.TOKEN.IDENTIFIER \/ | |
| 174 | token == Lexical.TOKEN.OPERATOR \/ | |
| 175 | token == Lexical.TOKEN.SQUARE_OPEN; | |
| 176 | ||
| 177 | // Tokens that unambiguously begin a statement (the statement parser's | |
| 178 | // dispatch set, minus the ambiguous heads handled above). A token that | |
| 179 | // is neither a definition head, an ambiguous head, nor one of these is | |
| 180 | // stray input handled by the definition parser. | |
| 181 | _is_statement_head(token: Lexical.TOKEN) -> bool => | |
| 182 | token == Lexical.TOKEN.PAREN_OPEN \/ | |
| 183 | token == Lexical.TOKEN.NEW \/ | |
| 184 | token == Lexical.TOKEN.CAST \/ | |
| 185 | token == Lexical.TOKEN.ISA \/ | |
| 186 | token == Lexical.TOKEN.TYPEOF \/ | |
| 187 | token == Lexical.TOKEN.INT_LITERAL \/ | |
| 188 | token == Lexical.TOKEN.FLOAT_LITERAL \/ | |
| 189 | token == Lexical.TOKEN.STRING_LITERAL \/ | |
| 190 | token == Lexical.TOKEN.ENTER_STRING \/ | |
| 191 | token == Lexical.TOKEN.CHAR_LITERAL \/ | |
| 192 | token == Lexical.TOKEN.TRUE \/ | |
| 193 | token == Lexical.TOKEN.FALSE \/ | |
| 194 | token == Lexical.TOKEN.NULL \/ | |
| 195 | token == Lexical.TOKEN.SELF \/ | |
| 196 | token == Lexical.TOKEN.SUPER \/ | |
| 197 | token == Lexical.TOKEN.REC \/ | |
| 198 | token == Lexical.TOKEN.VAL \/ | |
| 199 | token == Lexical.TOKEN.LET \/ | |
| 200 | token == Lexical.TOKEN.IF \/ | |
| 201 | token == Lexical.TOKEN.CASE \/ | |
| 202 | token == Lexical.TOKEN.AWAIT \/ | |
| 203 | token == Lexical.TOKEN.ASSERT \/ | |
| 204 | token == Lexical.TOKEN.RETURN \/ | |
| 205 | token == Lexical.TOKEN.YIELD \/ | |
| 206 | token == Lexical.TOKEN.THROW \/ | |
| 207 | token == Lexical.TOKEN.WHILE \/ | |
| 208 | token == Lexical.TOKEN.FOR \/ | |
| 209 | token == Lexical.TOKEN.DO \/ | |
| 210 | token == Lexical.TOKEN.TRY \/ | |
| 211 | token == Lexical.TOKEN.BREAK \/ | |
| 212 | token == Lexical.TOKEN.CONTINUE; | |
| 213 | ||
| 214 | // Decide whether an ambiguous head begins a global definition, by a | |
| 215 | // shallow bounded token peek — never a full parse, so it does not feed | |
| 216 | // the speculation-loop detector (its rewind is the exempt bounded-probe | |
| 217 | // kind). Because parameters are always typed, a definition is | |
| 218 | // recognisable from the first couple of tokens: a `: type` annotation, | |
| 219 | // a `(name: ...` / `[name: ...` typed parameter list, or an empty `()` | |
| 220 | // followed by a signature tail. The peek stops at a call argument's | |
| 221 | // opening token, so a formatted string argument is never read into and | |
| 222 | // never mis-lexed. | |
| 223 | _starts_definition(context: CONTEXT) -> bool is | |
| 224 | let use snapshot = context.tokenizer_speculate_then_backtrack_bounded(); | |
| 225 | ||
| 226 | let head = context.current.token; | |
| 227 | ||
| 228 | context.next_token(); | |
| 229 | ||
| 230 | if head == Lexical.TOKEN.SQUARE_OPEN then | |
| 231 | // [i: int] -> T (indexer) vs [a, b]... (array literal). | |
| 232 | return _is_typed_parameter_head(context); | |
| 233 | fi | |
| 234 | ||
| 235 | // IDENTIFIER or OPERATOR. | |
| 236 | let after = context.current.token; | |
| 237 | ||
| 238 | // name: type — global variable or typed member. | |
| 239 | if after == Lexical.TOKEN.COLON then | |
| 240 | return true; | |
| 241 | fi | |
| 242 | ||
| 243 | // name[T](...) — generic function definition vs name[i]... (index). | |
| 244 | if after == Lexical.TOKEN.SQUARE_OPEN then | |
| 245 | _skip_balanced(context, Lexical.TOKEN.SQUARE_OPEN, Lexical.TOKEN.SQUARE_CLOSE); | |
| 246 | ||
| 247 | return _parameter_list_starts_definition(context); | |
| 248 | fi | |
| 249 | ||
| 250 | // name(...) — function definition vs call statement. | |
| 251 | if after == Lexical.TOKEN.PAREN_OPEN then | |
| 252 | return _parameter_list_starts_definition(context); | |
| 253 | fi | |
| 254 | ||
| 255 | return false; | |
| 256 | si | |
| 257 | ||
| 258 | // The current token is expected to be `(`. Distinguishes a parameter | |
| 259 | // list (definition) from a call argument list (statement). A parameter | |
| 260 | // is always `name: type`, so a definition is `()` followed by a | |
| 261 | // definition tail, or `(name: …`; every other shape after `(` — a | |
| 262 | // literal or compound-expression argument, or `(name` continued by | |
| 263 | // anything but `:` (`,`, `)`, `.`, `(`, `[`, an operator, …) — is a | |
| 264 | // call argument list, i.e. a statement. | |
| 265 | _parameter_list_starts_definition(context: CONTEXT) -> bool is | |
| 266 | if context.current.token != Lexical.TOKEN.PAREN_OPEN then | |
| 267 | return false; | |
| 268 | fi | |
| 269 | ||
| 270 | context.next_token(); | |
| 271 | ||
| 272 | if context.current.token == Lexical.TOKEN.PAREN_CLOSE then | |
| 273 | context.next_token(); | |
| 274 | ||
| 275 | return _begins_definition_tail(context.current.token); | |
| 276 | fi | |
| 277 | ||
| 278 | if context.current.token != Lexical.TOKEN.IDENTIFIER then | |
| 279 | return false; | |
| 280 | fi | |
| 281 | ||
| 282 | context.next_token(); | |
| 283 | ||
| 284 | return context.current.token == Lexical.TOKEN.COLON; | |
| 285 | si | |
| 286 | ||
| 287 | // The current token should be the first parameter name, followed by `:`. | |
| 288 | _is_typed_parameter_head(context: CONTEXT) -> bool is | |
| 289 | if context.current.token != Lexical.TOKEN.IDENTIFIER then | |
| 290 | return false; | |
| 291 | fi | |
| 292 | ||
| 293 | context.next_token(); | |
| 294 | ||
| 295 | return context.current.token == Lexical.TOKEN.COLON; | |
| 296 | si | |
| 297 | ||
| 298 | // What can follow the `()` of an empty parameter list in a function | |
| 299 | // definition: the body (`is` / `=>`), a return type (`->`), or a | |
| 300 | // trailing modifier (`static`, `public`, …). A call statement's `()` | |
| 301 | // is followed by a statement continuation (`.`, `;`, an operator, …) | |
| 302 | // instead. | |
| 303 | _begins_definition_tail(token: Lexical.TOKEN) -> bool => | |
| 304 | token == Lexical.TOKEN.IS \/ | |
| 305 | token == Lexical.TOKEN.ARROW_FAT \/ | |
| 306 | token == Lexical.TOKEN.ARROW_THIN \/ | |
| 307 | token == Lexical.TOKEN.STATIC \/ | |
| 308 | token == Lexical.TOKEN.PUBLIC \/ | |
| 309 | token == Lexical.TOKEN.PRIVATE \/ | |
| 310 | token == Lexical.TOKEN.PROTECTED \/ | |
| 311 | token == Lexical.TOKEN.ABSTRACT \/ | |
| 312 | token == Lexical.TOKEN.FIELD; | |
| 313 | ||
| 314 | // Consume a balanced bracket group. On entry the current token is the | |
| 315 | // opening bracket; on return it is the token following the matching | |
| 316 | // close (or end-of-input / SI if unterminated). | |
| 317 | _skip_balanced(context: CONTEXT, open: Lexical.TOKEN, close: Lexical.TOKEN) is | |
| 318 | let depth mut = 0; | |
| 319 | ||
| 320 | while !context.is_end_of_file /\ context.current.token != Lexical.TOKEN.SI do | |
| 321 | let token = context.current.token; | |
| 322 | ||
| 323 | context.next_token(); | |
| 324 | ||
| 325 | if token == open then | |
| 326 | depth = depth + 1; | |
| 327 | elif token == close then | |
| 328 | depth = depth - 1; | |
| 329 | ||
| 330 | if depth <= 0 then | |
| 331 | return; | |
| 332 | fi | |
| 333 | fi | |
| 334 | od | |
| 335 | si | |
| 336 | ||
| 337 | _skip_semicolon(context: CONTEXT) is | |
| 338 | if context.current.token == Lexical.TOKEN.SEMICOLON then | |
| 339 | context.next_token(); | |
| 340 | fi | |
| 341 | si | |
| 342 | si | |
| 343 | si |