Theory Lexer_Table_Tests
theory Lexer_Table_Tests
imports Lexer_Command_Tests TIGER_Lexer.Lexer_Check
begin
section ‹Tests of the generated lexers›
text ‹The lexers that @{command lexer} builds for the specs of ‹Lexer_Command_Tests›,
checked with @{command lexer_check} where no theorem is needed.›
subsection ‹Identifiers and operators›
text ‹‹αβ →⇒›, a newline and ‹A›. The final stack is the start stack:›
lexer_check ‹cp_lex [0x3B1, 0x3B2, 0x20, 0x2192, 0x21D2, 0x0A, 0x41] =
(Lexed []
[⦇tok = LIDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = [0x3B1, 0x3B2]⦈,
trivia = [0x20]⦈,
⦇tok = OP, span = ⦇s_line = 0, s_col = 3, s_off = 3, s_len = 2, s_txt = [0x2192, 0x21D2]⦈,
trivia = [0x0A]⦈,
⦇tok = UIDENT, span = ⦇s_line = 1, s_col = 0, s_off = 6, s_len = 1, s_txt = [0x41]⦈,
trivia = []⦈],
[], [()])›
subsection ‹Nested comments›
text ‹‹a(*(**)*)1›: the nested comment is the trailing trivia of ‹a›:›
lexer_check ‹cmt_lex [0x61, 0x28, 0x2A, 0x28, 0x2A, 0x2A, 0x29, 0x2A, 0x29, 0x31] =
(Lexed []
[⦇tok = Word, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 1, s_txt = [0x61]⦈,
trivia = [0x28, 0x2A, 0x28, 0x2A, 0x2A, 0x29, 0x2A, 0x29]⦈,
⦇tok = Num, span = ⦇s_line = 0, s_col = 9, s_off = 9, s_len = 1, s_txt = [0x31]⦈,
trivia = []⦈],
[], [Code])›
text ‹An unterminated comment consumes the input but ends in ‹Comment›, which is not
final, so the scan fails:›
lexer_check ‹cmt_lex [0x61, 0x28, 0x2A, 0x62] =
(Lexed []
[⦇tok = Word, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 1, s_txt = [0x61]⦈,
trivia = [0x28, 0x2A, 0x62]⦈],
[], [Comment, Code])›
lexer_check ‹cmt_lex.ok [0x61, 0x62]› ‹¬ cmt_lex.ok [0x61, 0x28, 0x2A, 0x62]›
‹¬ cmt_lex.ok [0x61, 0x2B]›
text ‹The correctness theorem on a concrete input: the result of the lexer is a
derivation of \<^const>‹symbolic.lexes› from the start mode:›
lemma cmt_lex_ab:
‹cmt_lex [0x61, 0x62] =
(Lexed [] [⦇tok = Word, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2,
s_txt = [0x61, 0x62]⦈, trivia = []⦈], [], [Code])›
by eval
lemma
‹symbolic.lexes in_cls cmt.spec cmt.is_newline [cmt.start] ⦇line = 0, col = 0, off = 0⦈
[0x61, 0x62]
(Lexed [] [⦇tok = Word, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2,
s_txt = [0x61, 0x62]⦈, trivia = []⦈]) [] [Code]›
by (rule cmt_lex.correct[THEN iffD1]) (simp_all add: cmt_lex_ab)
subsection ‹Bytes›
text ‹The same spec on bytes. Its classes stay below ‹0x100›, apart from the
complement in the comment rule, which reaches the end of the codepoints:›