src/lexical/tokenizer.ghul

1		namespace Lexical is
2		use System.Exception;
3
4		use Logging;
5		use Source;
6
7		use Ghul.Pipes;
8
9		get_stack_trace() -> string is
10		try
11		return System.Diagnostics.StackTrace()
12		.to_string()
13		.split(['\n']) \|>
14		skip(2) \|>
15		take(5) \|>
16		map(s => s.trim().replace("at ", "")) \|> join(" ");
17		catch e: System.Exception
18		return "unknown";
19		yrt
20		si
21
22		class TOKENIZER_EXCEPTION(s: string): Exception is
23		super(s);
24		si
25
26		class TOKEN_MAP is
27		_map: Collections.MAP[string,TOKEN];
28
29		init() is
30		super.init();
31		_map = Collections.MAP[string,TOKEN](223);
32		si
33
34		[s: string]: TOKEN public =>
35		if _map.contains_key(s) then
36		_map[s]
37		else
38		TOKEN.IDENTIFIER;
39		fi,
40		= t is
41		_map[s] = t;
42		si
43		si
44
45		class TOKEN_PAIR(token: TOKEN, location: LOCATION, value: string init) is
46		value_string: string;
47
48		name: string => TOKEN_NAMES[token];
49
50		to_string() -> string =>
51		"{location}: {TOKEN_NAMES[token]} {if value_string.length > 0 then "'{value_string}'" else "" fi}";
52
53		to_short_string() -> string =>
54		"{if value_string.length > 0 /\ value_string !~ TOKEN_NAMES[token] then "{TOKEN_NAMES[token]} \"{value_string}\"" else "'{TOKEN_NAMES[token]}'" fi}";
55
56		init(..) is
57		value_string = value;
58		si
59		si
60
61		class TOKENIZER: TokenSource is
62		_prev_count: int;
63
64		_logger: Logger;
65		_token_pair: TOKEN_PAIR;
66		_symbol_tokens: TOKEN_MAP static;
67		_operator_chars: Collections.SET[char] static;
68		_operator_tokens: Collections.MAP[string,TOKEN] static;
69		_input: IO.TextReader;
70		_end_of_file: bool;
71		_prev_char: char;
72		_cursor: LOCATION_CURSOR;
73		_token_name: string[];
74
75		_interpolation_depth: int;
76		_expect_format_string: bool;
77
78		_trivia: Collections.LIST[TRIVIA];
79
80		// Comments and blank-line markers, in source order. Discarded by the
81		// parser; consumed by the formatter to round-trip non-token content.
82		trivia: Collections.Iterable[TRIVIA] => _trivia;
83
84		static_init() static is
85		if _symbol_tokens == null then
86		_operator_chars = Collections.SET[char]([
87		'!', '$', '%', '^', '&', '*', '-', '+', '=', '\|', ':', '@', '~', '#', '\\', '<', '>', '.', '?', '/'
88		]: char );
89
90		_operator_tokens = Collections.MAP[string,TOKEN]();
91		_operator_tokens["="] = TOKEN.ASSIGN;
92		_operator_tokens[":"] = TOKEN.COLON;
93		_operator_tokens["."] = TOKEN.DOT;
94		_operator_tokens["->"] = TOKEN.ARROW_THIN;
95		_operator_tokens["=>"] = TOKEN.ARROW_FAT;
96		_operator_tokens["\|>"] = TOKEN.BAR_ARROW;
97		_operator_tokens["?"] = TOKEN.QUESTION;
98		_operator_tokens["@"] = TOKEN.AT;
99		_symbol_tokens = TOKEN_MAP();
100		_symbol_tokens["abstract"] = TOKEN.ABSTRACT;
101		_symbol_tokens["namespace"] = TOKEN.NAMESPACE;
102		_symbol_tokens["class"] = TOKEN.CLASS;
103		_symbol_tokens["struct"] = TOKEN.STRUCT;
104		_symbol_tokens["union"] = TOKEN.UNION;
105		_symbol_tokens["partial"] = TOKEN.PARTIAL;
106		_symbol_tokens["impl"] = TOKEN.IMPL;
107		_symbol_tokens["enum"] = TOKEN.ENUM;
108		_symbol_tokens["public"] = TOKEN.PUBLIC;
109		_symbol_tokens["protected"] = TOKEN.PROTECTED;
110		_symbol_tokens["private"] = TOKEN.PRIVATE;
111		_symbol_tokens["field"] = TOKEN.FIELD;
112		_symbol_tokens["static"] = TOKEN.STATIC;
113		_symbol_tokens["innate"] = TOKEN.INNATE;
114		_symbol_tokens["rec"] = TOKEN.REC;
115		_symbol_tokens["if"] = TOKEN.IF;
116		_symbol_tokens["else"] = TOKEN.ELSE;
117		_symbol_tokens["while"] = TOKEN.WHILE;
118		_symbol_tokens["do"] = TOKEN.DO;
119		_symbol_tokens["for"] = TOKEN.FOR;
120		_symbol_tokens["in"] = TOKEN.IN;
121		_symbol_tokens["case"] = TOKEN.CASE;
122		_symbol_tokens["when"] = TOKEN.WHEN;
123		_symbol_tokens["default"] = TOKEN.DEFAULT;
124		_symbol_tokens["break"] = TOKEN.BREAK;
125		_symbol_tokens["continue"] = TOKEN.CONTINUE;
126		_symbol_tokens["ref"] = TOKEN.REF;
127		_symbol_tokens["ptr"] = TOKEN.PTR;
128		_symbol_tokens["new"] = TOKEN.NEW;
129		_symbol_tokens["throw"] = TOKEN.THROW;
130		_symbol_tokens["return"] = TOKEN.RETURN;
131		_symbol_tokens["cast"] = TOKEN.CAST;
132		_symbol_tokens["try"] = TOKEN.TRY;
133		_symbol_tokens["let"] = TOKEN.LET;
134		_symbol_tokens["mut"] = TOKEN.MUT;
135		_symbol_tokens["await"] = TOKEN.AWAIT;
136		_symbol_tokens["catch"] = TOKEN.CATCH;
137		_symbol_tokens["finally"] = TOKEN.FINALLY;
138		_symbol_tokens["self"] = TOKEN.SELF;
139		_symbol_tokens["super"] = TOKEN.SUPER;
140		_symbol_tokens["null"] = TOKEN.NULL;
141		_symbol_tokens["use"] = TOKEN.USE;
142		_symbol_tokens["trait"] = TOKEN.TRAIT;
143		_symbol_tokens["isa"] = TOKEN.ISA;
144		_symbol_tokens["typeof"] = TOKEN.TYPEOF;
145		// _symbol_tokens["operator"] = TOKEN.OPERATOR;
146		_symbol_tokens["is"] = TOKEN.IS;
147		_symbol_tokens["si"] = TOKEN.SI;
148		_symbol_tokens["then"] = TOKEN.THEN;
149		_symbol_tokens["elif"] = TOKEN.ELIF;
150		_symbol_tokens["fi"] = TOKEN.FI;
151		_symbol_tokens["esac"] = TOKEN.ESAC;
152		_symbol_tokens["lav"] = TOKEN.LAV;
153		_symbol_tokens["val"] = TOKEN.VAL;
154		_symbol_tokens["od"] = TOKEN.OD;
155		_symbol_tokens["yield"] = TOKEN.YIELD;
156		_symbol_tokens["yrt"] = TOKEN.YRT;
157		_symbol_tokens["true"] = TOKEN.TRUE;
158		_symbol_tokens["false"] = TOKEN.FALSE;
159		_symbol_tokens["assert"] = TOKEN.ASSERT;
160		fi
161		si
162
163		// True when a bare occurrence of `name` would tokenise as a keyword
164		// rather than an identifier, so writing it back out unadorned would
165		// change its meaning. Such a name can only have reached the parser as
166		// an identifier via a backtick escape.
167		is_reserved_word(name: string?) -> bool static is
168		static_init();
169		return name? /\ _symbol_tokens[name] != TOKEN.IDENTIFIER;
170		si
171
172		// True when a bare occurrence of `name` would tokenise as a numeric
173		// literal rather than an identifier: it is composed only of identifier
174		// characters but begins with a digit. Such a name reached the parser
175		// as an identifier via a backtick escape.
176		is_numeric_identifier(name: string) -> bool static is
177		if name == null \/ name.length == 0 then
178		return false;
179		fi
180
181		let first = name.get_chars(0);
182
183		if !(first >= '0' /\ first <= '9') then
184		return false;
185		fi
186
187		for c in name do
188		if !((c >= 'a' /\ c <= 'z') \/ (c >= 'A' /\ c <= 'Z') \/ (c >= '0' /\ c <= '9') \/ c == '_') then
189		return false;
190		fi
191		od
192
193		return true;
194		si
195
196		// True when `name` is composed entirely of operator characters, so a
197		// bare occurrence would tokenise as an operator and a backtick escape
198		// is needed to refer to it as a plain identifier.
199		is_operator_name(name: string) -> bool static is
200		static_init();
201
202		if name == null \/ name.length == 0 then
203		return false;
204		fi
205
206		for c in name do
207		if !_operator_chars.contains(c) then
208		return false;
209		fi
210		od
211
212		return true;
213		si
214
215		init(logger: Logger, file_name: string, i: IO.TextReader, is_internal_file: bool) is
216		super.init();
217
218		_logger = logger;
219
220		_end_of_file = false;
221		_input = i;
222
223		_trivia = Collections.LIST[TRIVIA]();
224
225		static_init();
226
227		if is_internal_file then
228		_cursor = INTERNAL_LOCATION_CURSOR();
229		else
230		_cursor = LOCATION_CURSOR(file_name);
231		fi
232
233		_token_pair = TOKEN_PAIR(TOKEN.UNKNOWN, location, null);
234		si
235
236		is_end_of_file: bool => _end_of_file;
237
238		location: LOCATION => _cursor.location;
239
240		character_location: LOCATION => _cursor.character_location;
241
242		advance_cursor(c: char) is
243		if c == cast char(13) then
244		return;
245		fi
246
247		_cursor.next_column();
248
249		if c == '\n' then
250		_cursor.next_line();
251		fi
252		si
253
254		next_char() -> char is
255		let c: char mut;
256
257		if _prev_char != cast char(0) then
258		c = _prev_char;
259		advance_cursor(c);
260		_prev_char = cast char(0);
261		return c;
262		fi
263
264		if _end_of_file then
265		return ' ';
266		fi
267
268		let c0: int mut = _input.read();
269
270		if c0 == 13 then
271		c0 = 32;
272		fi
273
274		if c0 == -1 \/ c0 == 12 then
275		_end_of_file = true;
276		return ' ';
277		fi
278
279		c = cast char(c0);
280
281		_cursor.save();
282		advance_cursor(c);
283
284		return c;
285		si
286
287		prev_char(c: char) is
288		_cursor.restore();
289		_prev_char = c;
290		si
291
292		current_string: string =>
293		_token_pair.value_string;
294
295		read_escape() -> char is
296		let c: char mut = next_char();
297		let result: int mut = 0;
298		if c == 't' then
299		return cast char(9);
300		elif c == 'n' then
301		return '\n';
302		elif c == 'r' then
303		return cast char(13);
304		elif c == '\\' then
305		return '\\';
306		elif c>='0' /\ c<='7' then
307		while c >= '0' /\ c <= '7' do
308		result = 8 * result + cast int(c - '0');
309		c = next_char();
310		od
311
312		prev_char(c);
313		return cast char(result);
314		else
315		return c;
316		fi
317		si
318
319		skip_white_space() -> char is
320		let c: char mut = _;
321		let newlines mut = 0;
322		let blank_location: LOCATION? mut = null;
323		do
324		c = next_char();
325
326		if c == '\n' then
327		newlines = newlines + 1;
328		if newlines == 2 then
329		blank_location = character_location;
330		fi
331		fi
332
333		let is_white_space = (c==' ' \/ c==cast char(9) \/ c=='\n');
334
335		let should_break = is_end_of_file \/ !is_white_space;
336
337		if is_end_of_file \/ !is_white_space then
338		break;
339		fi
340		od
341
342		if newlines >= 2 /\ blank_location? then
343		_trivia.add(TRIVIA.BLANK_LINE(blank_location));
344		fi
345
346		return c;
347		si
348
349		expect_format_specifier() is
350		_expect_format_string = true;
351		si
352
353		read_token() -> TOKEN_PAIR is
354		let r: TOKEN mut;
355
356		let c mut = skip_white_space();
357
358		if _end_of_file then
359		return TOKEN_PAIR(TOKEN.END_OF_INPUT, location, null);
360		fi
361
362		_cursor.start();
363
364		let _buffer mut = System.Text.StringBuilder();
365
366		if _expect_format_string then
367		_expect_format_string = false;
368
369		_buffer = System.Text.StringBuilder();
370
371		while c != '\n' /\ c != '}' /\ c != '"' do
372		_buffer.append(c);
373
374		c = next_char();
375		od
376
377		if c == '}' then
378		prev_char(c);
379		elif c == '\n' then
380		prev_char(c);
381		_logger.lexer_error(character_location, "newline in interpolation format string");
382		return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string());
383		elif c == '"' then
384		_logger.lexer_error(character_location, "expected '}}' after format string");
385		return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string());
386		fi
387
388		return TOKEN_PAIR(TOKEN.FORMAT_STRING, location, _buffer.to_string());
389
390		elif c >= '0' /\ c <= '9' then
391		let is_float mut = false;
392
393		_buffer = System.Text.StringBuilder();
394		_buffer.append(c);
395		c = next_char();
396
397		if c=='x' \/ c=='X' then
398		_buffer.append(c);
399		c = next_char();
400		while
401		(c>='0'/\c<='9') \/ (c>='A'/\c<='F') \/ (c>='a'/\c<='f') \/ c == '_'
402		do
403		if c != '_' then
404		_buffer.append(c);
405		fi
406		c = next_char();
407		od
408		else
409		let seen_dot mut = false;
410		let pc: char mut = _;
411
412		while
413		(c >= '0' /\ c <= '9') \/ c == '.' \/ c == '_'
414		do
415		if c == '.' then
416		if pc == '.' then
417		is_float = false;
418
419		// bodge: can only push back one char, so use Unicode '‥' to signal we actually want to push back '..'
420		c = '‥';
421		break;
422		elif seen_dot then
423		break;
424		else
425		is_float = true;
426		seen_dot = true;
427		fi
428		else
429		if pc == '.' then
430		_buffer.append('.');
431		fi
432
433		if c != '_' then
434		_buffer.append(c);
435		fi
436		fi
437
438		pc = c;
439		c = next_char();
440		od
441		fi
442
443		if is_float then
444		if c == 'e' \/ c == 'E' then
445		_buffer.append(c);
446		c = next_char();
447
448		if c == '-' then
449		_buffer.append(c);
450		c = next_char();
451		fi
452
453		if c >= '0' \/ c <= '9' \/ c == '_' then
454		while (c >= '0' /\ c <= '9') \/ c == '_' do
455		if c != '_' then
456		_buffer.append(c);
457		fi
458
459		c = next_char();
460		od
461		else
462		_logger.lexer_error(location, "expected exponent in float literal");
463		fi
464		fi
465
466		if c == 's' \/ c == 'S' \/ c == 'd' \/ c == 'D' \/ c == 'm' \/ c == 'M' then
467		_buffer.append(c);
468		c = next_char();
469		else
470		_buffer.append('S');
471		fi
472		else
473		if c == 'm' \/ c == 'M' then
474		_buffer.append(c);
475		c = next_char();
476		is_float = true;
477		else
478		if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then
479		_buffer.append(c);
480		c = next_char();
481		fi
482
483		if "bBcCsSiIlLwW".contains(c) then
484		_buffer.append(c);
485		c = next_char();
486		fi
487		fi
488		fi
489
490		prev_char(c);
491
492		if is_float then
493		return TOKEN_PAIR(TOKEN.FLOAT_LITERAL, location, _buffer.to_string());
494		else
495		return TOKEN_PAIR(TOKEN.INT_LITERAL, location, _buffer.to_string());
496		fi
497
498		elif (c>='a'/\c<='z') \/ (c>='A'/\c<='Z') \/ c=='_' \/ c == '`' then
499		let was_escaped: bool mut = false;
500
501		if c == '`' then
502		c = next_char();
503
504		if c == '[' then
505		return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, null);
506		elif _operator_chars.contains(c) \/ (cast int(c) > 0x7E /\ char.is_symbol(c)) then
507		let o = read_operator(c);
508		return TOKEN_PAIR(TOKEN.IDENTIFIER, o.location, o.value_string);
509		else
510		was_escaped = true;
511		fi
512		fi
513
514		_buffer = System.Text.StringBuilder();
515
516		while (c>='a'/\c<='z') \/ (c>='A'/\c<='Z') \/ (c>='0'/\c<='9') \/ c=='_' do
517		_buffer.append(c);
518		c = next_char();
519		od
520
521		prev_char(c);
522		let s = _buffer.to_string();
523
524		if was_escaped then
525		if s.length == 0 then
526		_logger.lexer_error(location, "expected an identifier after backtick");
527		fi
528
529		r = TOKEN.IDENTIFIER;
530		else
531		r = _symbol_tokens[s];
532		fi
533
534		return TOKEN_PAIR(r, location, s);
535		elif c == '/' then
536		let comment_location = location;
537		c = next_char();
538		if c == '/' then
539		_buffer = System.Text.StringBuilder("//");
540		do
541		c = next_char();
542		if c != '\n' then
543		_buffer.append(c);
544		fi
545
546		if !(!_end_of_file/\c!='\n') then
547		break;
548		fi
549		od
550
551		_trivia.add(
552		TRIVIA.LINE_COMMENT(
553		_buffer.to_string().trim_end(),
554		comment_location
555		)
556		);
557
558		return read_token();
559		elif c == '*' then
560		_buffer = System.Text.StringBuilder("/*");
561		do
562		c = next_char();
563
564		if c == '*' then
565		c = next_char();
566		if c == '/' then
567		break;
568		fi
569
570		_buffer.append('*');
571		fi
572
573		_buffer.append(c);
574
575		if _end_of_file then
576		break;
577		fi
578		od
579
580		_buffer.append("*/");
581
582		_trivia.add(
583		TRIVIA.BLOCK_COMMENT(
584		_buffer.to_string(),
585		comment_location
586		)
587		);
588
589		return read_token();
590		else
591		prev_char(c);
592		return read_operator('/');
593		fi
594		elif _operator_chars.contains(c) \/ (cast int(c) > 0x7E /\ char.is_symbol(c)) then
595		return read_operator(c);
596		fi
597
598		case c
599
600		when '\'' then
601		_buffer = System.Text.StringBuilder();
602
603		c = next_char();
604		while c != cast char(39) do
605		if c == cast char(92) then
606		c = read_escape();
607		_buffer.append(c);
608		c = next_char();
609		else
610		_buffer.append(c);
611		c = next_char();
612		fi
613
614		if _end_of_file then
615		_logger.lexer_error(location, "end of file in character literal");
616		break;
617		fi
618		od
619
620		if _buffer.length < 1 then
621		_logger.lexer_error(location, "zero length character literal");
622		elif _buffer.length > 1 then
623		_logger.lexer_error(location, "character literal is too long");
624		fi
625		return TOKEN_PAIR(TOKEN.CHAR_LITERAL, location, _buffer.to_string());
626
627		when '(' then
628		return TOKEN_PAIR(TOKEN.PAREN_OPEN, location, null);
629
630		when ')' then
631		return TOKEN_PAIR(TOKEN.PAREN_CLOSE, location, null);
632
633		when '[' then
634		c = next_char();
635		if _end_of_file then
636		return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, null);
637		elif c == ']' then
638		return TOKEN_PAIR(TOKEN.ARRAY_DEF, location, null);
639		else
640		prev_char(c);
641		return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, null);
642		fi
643
644		when ']' then
645		return TOKEN_PAIR(TOKEN.SQUARE_CLOSE, location, null);
646
647		when ',' then
648		return TOKEN_PAIR(TOKEN.COMMA, location, null);
649
650		when ';' then
651		return TOKEN_PAIR(TOKEN.SEMICOLON, location, null);
652
653		when '‥' then
654		return TOKEN_PAIR(TOKEN.OPERATOR, location, "..");
655
656		when '⟦' then
657		return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, null);
658
659		when '"' then
660		return string_enter();
661
662		when '}' then
663		return interpolation_exit('}');
664
665		else
666		return TOKEN_PAIR(TOKEN.UNKNOWN, location, "{c}");
667		esac
668		si
669
670		read_operator(c: char mut) -> TOKEN_PAIR is
671		let _buffer = System.Text.StringBuilder();
672		let first = c;
673
674		// A `.` immediately after a bare `!` or `?` ends the operator: it
675		// begins a member access (`x!.foo`, `x?.foo`), not a longer infix
676		// operator. Longer operators starting with `!`/`?` (e.g. `!=`) and
677		// dot operators (`..`) are unaffected.
678		while
679		(_operator_chars.contains(c) \/ (cast int(c) > 0x7E /\ char.is_symbol(c)))
680		/\ !(_buffer.length == 1 /\ (first == '!' \/ first == '?') /\ c == '.')
681		do
682		_buffer.append(c);
683		c = next_char();
684		od
685
686		prev_char(c);
687
688		if _buffer.length < 1 then
689		return TOKEN_PAIR(TOKEN.FIRST, location, null);
690		fi
691
692		let s = _buffer.to_string();
693
694		let r mut = TOKEN.OPERATOR;
695
696		if _operator_tokens.contains_key(s) then
697		r = _operator_tokens[s];
698		fi
699
700		return TOKEN_PAIR(r, location, s);
701		si
702
703		read_string_fragment() -> (fragment: string, c: char) is
704		let fragment = System.Text.StringBuilder();
705
706		let c mut = next_char();
707
708		while c != '\n' /\ c != '"' do
709		if c == cast char(92) then
710		c = read_escape();
711		fragment.append(c);
712		c = next_char();
713		elif c == '{' then
714		c = next_char();
715
716		if c == '{' then
717		fragment.append(c);
718		c = next_char();
719		else
720		prev_char(c);
721		c = '{';
722
723		break;
724		fi
725		elif c == '}' then
726		c = next_char();
727
728		if c == '}' then
729		fragment.append(c);
730		c = next_char();
731		else
732		_logger.lexer_error(character_location, "unmatched '}}' in string interpolation");
733
734		fragment.append('}');
735		fi
736		else
737		fragment.append(c);
738		c = next_char();
739		fi
740		od
741
742		if c == '\n' then
743		_logger.lexer_error(character_location, "newline in string literal");
744		fi
745
746		return (fragment.to_string(), c);
747		si
748
749		// Adjacent string-literal concatenation: while a closed `"..."`
750		// fragment is followed (after whitespace) by another `"`, fold
751		// the next fragment into this token. Stops at the first fragment
752		// that doesn't close with `"` (i.e. starts an interpolation or
753		// hits a newline).
754		//
755		// We treat the whole `"..." "..." "..."` run as a single token.
756		// The token's end position is wherever the final fragment
757		// terminated — we snapshot the cursor's location after each
758		// successful fragment close. When the chain fails (we consume
759		// whitespace then see a non-`"` char), the cursor is past the
760		// whitespace, but we return the snapshot taken at the previous
761		// close so the token's location end isn't smeared across the
762		// following whitespace. The stream stays at the non-`"` char;
763		// the next read_token picks up from there normally.
764		chain_adjacent_string_fragments(initial_fragment: string, initial_terminator: char) -> (fragment: string, c: char, loc: Source.LOCATION) is
765		let buffer = System.Text.StringBuilder();
766		buffer.append(initial_fragment);
767		let c mut = initial_terminator;
768		let saved_loc mut = location;
769
770		let done mut = false;
771		while !done /\ c == '"' do
772		let peeked = skip_white_space();
773		if peeked == '"' then
774		let (next_fragment, next_terminator) = read_string_fragment();
775		buffer.append(next_fragment);
776		c = next_terminator;
777		saved_loc = location;
778		else
779		prev_char(peeked);
780		done = true;
781		fi
782		od
783
784		return (buffer.to_string(), c, saved_loc);
785		si
786
787		string_enter() -> TOKEN_PAIR is
788		let (fragment, c) mut = read_string_fragment();
789
790		let loc: Source.LOCATION mut;
791		(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c);
792
793		if c == '\n' then
794		_interpolation_depth = 0;
795
796		return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment);
797		elif c == '"' then
798		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment);
799		elif c == '{' then
800		_interpolation_depth = _interpolation_depth + 1;
801
802		return TOKEN_PAIR(TOKEN.ENTER_STRING, loc, fragment);
803		else
804		_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation");
805
806		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment);
807		fi
808		si
809
810		interpolation_clear() is
811		_interpolation_depth = 0;
812		si
813
814		interpolation_exit(c: char mut) -> TOKEN_PAIR is
815		if _interpolation_depth == 0 then
816		_logger.lexer_error(character_location, "unmatched '\"'");
817		next_char();
818		return read_token();
819		fi
820
821		let fragment mut = "";
822
823		if c == '}' then
824		(fragment, c) = read_string_fragment();
825		fi
826
827		let loc: Source.LOCATION mut;
828		(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c);
829
830		if c == '\n' then
831		_interpolation_depth = 0;
832		return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment);
833		elif c == '"' then
834		_interpolation_depth = _interpolation_depth - 1;
835
836		return TOKEN_PAIR(TOKEN.EXIT_STRING, loc, fragment);
837		elif c == '{' then
838		return TOKEN_PAIR(TOKEN.CONTINUE_STRING, loc, fragment);
839		else
840		_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation");
841
842		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment);
843		fi
844		si
845		si
846		si