Theory Driver_Tests

(* Author: Basil Fürer *)

theory Driver_Tests
  imports TIGER_Lexer.TIGER_Tables
begin

section ‹Tests of the driver›

text ‹A two-mode spec on bytes with nested comments. Mode ‹0› lexes identifiers and
  skips whitespace, and ‹(*› pushes mode ‹1›. In mode ‹1›, ‹(*› pushes mode ‹1› again,
  ‹*)› pops it, and every other character is trivia (a single ‹(› or ‹*› loses to
  ‹(*› and ‹*)› by length). The theorem about the lexer is an instance of
  @{thm [source] char_lexer_list_iff_lexes}.›

datatype tok = IDENT

definition ‹chr c = Pred [(c, c)]›
definition ‹letter = [(CHR ''a'', CHR ''z'')]›
definition ‹space = [(CHR '' '', CHR '' ''), (CHR 0x0A, CHR 0x0A)]›

text ‹Every byte but ‹(› (‹0x28›) and ‹*› (‹0x2A›):›

definition ‹other = [(CHR 0x00, CHR 0x27), (CHR 0x29, CHR 0x29), (CHR 0x2B, CHR 0xFF)]›

definition ‹open_comment = Times (chr CHR ''('') (chr CHR ''*'')›
definition ‹close_comment = Times (chr CHR ''*'') (chr CHR '')'')›

definition comment_rules :: ‹(tok, char cls, char, nat) rule list list› where
  ‹comment_rules = [
    [⦇match = open_comment, act = λ_ st. (None, 1 # st)⦈,
     ⦇match = Times (Pred letter) (Star (Pred letter)), act = λ_ st. (Some IDENT, st)⦈,
     ⦇match = Times (Pred space) (Star (Pred space)), act = λ_ st. (None, st)⦈],
    [⦇match = open_comment, act = λ_ st. (None, 1 # st)⦈,
     ⦇match = close_comment, act = λ_ st. (None, tl st)⦈,
     ⦇match = Pred other, act = λ_ st. (None, st)⦈,
     ⦇match = chr CHR ''('', act = λ_ st. (None, st)⦈,
     ⦇match = chr CHR ''*'', act = λ_ st. (None, st)⦈]]›

definition ‹comment_tables = char_lexer_tables comment_rules›

(* ~5s *)
local_setup ‹eval_table (binding‹comment_tables_code›, term‹comment_tables›)›

declare comment_tables_code[code]

definition ‹is_lf c ⟷ c = CHR 0x0A›

definition ‹start = ⦇line = 0, col = 0, off = 0⦈›

definition ‹comment_lex st = char_lexer comment_tables id (list_spec comment_rules) is_lf st start›

text ‹The lexer on the tables is the driver of the spec:›

theorem comment_lex_iff_lexes:
  ‹comment_lex st s = (l, r, stf) ⟷
    char_cls.lexes (list_spec comment_rules) is_lf st start s l r stf›
  by (simp add: comment_lex_def comment_tables_def char_lexer_list_iff_lexes)

text ‹A nested comment across a newline is the trailing trivia of ‹ab›. The span of
  ‹xy› is on line ‹1›, column ‹4›, offset ‹22›:›

definition ‹nested = ''ab (* c (* d *) e'' @ [CHR 0x0A] @ '' *) xy''›

value ‹comment_lex [0] nested›

lemma ‹comment_lex [0] nested =
  (Lexed []
    [⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
      trivia = '' (* c (* d *) e'' @ [CHR 0x0A] @ '' *) ''⦈,
     ⦇tok = IDENT, span = ⦇s_line = 1, s_col = 4, s_off = 22, s_len = 2, s_txt = ''xy''⦈,
      trivia = []⦈],
   [], [0])›
  by eval

text ‹Leading trivia go into the header:›

lemma ‹comment_lex [0] '' (*c*) ab'' =
  (Lexed '' (*c*) ''
    [⦇tok = IDENT, span = ⦇s_line = 0, s_col = 7, s_off = 7, s_len = 2, s_txt = ''ab''⦈,
      trivia = []⦈],
   [], [0])›
  by eval

text ‹An unmatched character stops the lexer with the residue:›

lemma ‹comment_lex [0] ''ab $ cd'' =
  (Lexed []
    [⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
      trivia = '' ''⦈],
   ''$ cd'', [0])›
  by eval

text ‹Popping the last mode is no step either: from the stack ‹[1]›, ‹*)› stays
  unread:›

lemma ‹comment_lex [1] ''c *) ab'' = (Lexed ''c '' [], ''*) ab'', [1])›
  by eval

text ‹An unterminated comment consumes the input. The residue is empty, and the final
  stack, one entry per open comment, says that the input ended in mode ‹1›:›

lemma ‹comment_lex [0] ''ab (* c'' =
  (Lexed []
    [⦇tok = IDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = ''ab''⦈,
      trivia = '' (* c''⦈],
   [], [1, 0])›
  by eval

lemma ‹comment_lex [0] ''(* a (* b *)'' = (Lexed ''(* a (* b *)'' [], [], [1, 0])›
  by eval

end