Theory Driver_Tests
theory Driver_Tests
imports TIGER_Lexer.TIGER_Tables
begin
section ‹Tests of the driver›
text ‹A two-mode spec on bytes with nested comments. Mode ‹0› lexes identifiers and
skips whitespace, and ‹(*› pushes mode ‹1›. In mode ‹1›, ‹(*› pushes mode ‹1› again,
‹*)› pops it, and every other character is trivia (a single ‹(› or ‹*› loses to
‹(*› and ‹*)› by length). The theorem about the lexer is an instance of
@{thm [source] char_lexer_list_iff_lexes}.›
datatype tok = IDENT
definition ‹chr c = Pred [(c, c)]›
definition ‹letter = [(CHR ''a'', CHR ''z'')]›
definition ‹space = [(CHR '' '', CHR '' ''), (CHR 0x0A, CHR 0x0A)]›
text ‹Every byte but ‹(› (‹0x28›) and ‹*› (‹0x2A›):›
definition ‹other = [(CHR 0x00, CHR 0x27), (CHR 0x29, CHR 0x29), (CHR 0x2B, CHR 0xFF)]›
‹ = Times (chr CHR ''('') (chr CHR ''*'')›
‹ = Times (chr CHR ''*'') (chr CHR '')'')›
:: ‹(tok, char cls, char, nat) rule list list› where
‹comment_rules = [
[⦇match = open_comment, act = λ_ st. (None, 1 # st)⦈,
⦇match = Times (Pred letter) (Star (Pred letter)), act = λ_ st. (Some IDENT, st)⦈,
⦇match = Times (Pred space) (Star (Pred space)), act = λ_ st. (None, st)⦈],
[⦇match = open_comment, act = λ_ st. (None, 1 # st)⦈,
⦇match = close_comment, act = λ_ st. (None, tl st)⦈,
⦇match = Pred other, act = λ_ st. (None, st)⦈,
⦇match = chr CHR ''('', act = λ_ st. (None, st)⦈,
⦇match = chr CHR ''*'', act = λ_ st. (None, st)⦈]]›
‹ = char_lexer_tables comment_rules›
local_setup ‹eval_table (\<^binding>‹›, \<^term>‹comment_tables›)›
declare comment_tables_code[code]
definition ‹is_lf c ⟷ c = CHR 0x0A›
definition ‹start = ⦇line = 0, col = 0, off = 0⦈›
‹ st = char_lexer comment_tables id (list_spec comment_rules) is_lf st start›
text ‹The lexer on the tables is the driver of the spec:›
theorem :
‹comment_lex st s = (l, r, st⇩f) ⟷
char_cls.lexes (list_spec comment_rules) is_lf st start s l r st⇩f›
by (simp add: comment_lex_def comment_tables_def char_lexer_list_iff_lexes)
text ‹A nested comment across a newline is the trailing trivia of ‹ab›. The span of
‹xy› is on line ‹1›, column ‹4›, offset ‹22›:›
definition ‹nested = ''ab (* c (* d *) e'' @ [CHR 0x0A] @ '' *) xy''›
value ‹comment_lex [0] nested›
lemma ‹comment_lex [0] nested =
(Lexed []
[⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
trivia = '' (* c (* d *) e'' @ [CHR 0x0A] @ '' *) ''⦈,
⦇tok = IDENT, span = ⦇s_line = 1, s_col = 4, s_off = 22, s_len = 2, s_txt = ''xy''⦈,
trivia = []⦈],
[], [0])›
by eval
text ‹Leading trivia go into the header:›
lemma ‹comment_lex [0] '' (*c*) ab'' =
(Lexed '' (*c*) ''
[⦇tok = IDENT, span = ⦇s_line = 0, s_col = 7, s_off = 7, s_len = 2, s_txt = ''ab''⦈,
trivia = []⦈],
[], [0])›
by eval
text ‹An unmatched character stops the lexer with the residue:›
lemma ‹comment_lex [0] ''ab $ cd'' =
(Lexed []
[⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
trivia = '' ''⦈],
''$ cd'', [0])›
by eval
text ‹Popping the last mode is no step either: from the stack ‹[1]›, ‹*)› stays
unread:›
lemma ‹comment_lex [1] ''c *) ab'' = (Lexed ''c '' [], ''*) ab'', [1])›
by eval
text ‹An unterminated comment consumes the input. The residue is empty, and the final
stack, one entry per open comment, says that the input ended in mode ‹1›:›
lemma ‹comment_lex [0] ''ab (* c'' =
(Lexed []
[⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
trivia = '' (* c''⦈],
[], [1, 0])›
by eval
lemma ‹comment_lex [0] ''(* a (* b *)'' = (Lexed ''(* a (* b *)'' [], [], [1, 0])›
by eval
end