Theory Codepoint_Tests

(* Author: Basil Fürer *)

theory Codepoint_Tests
  imports TIGER_Lexer.TIGER_Codepoints
begin

section ‹Tests of the codepoint instance›

text ‹Identifiers over ASCII and Greek letters: a lower
  case letter starts a value identifier, an upper case letter a constructor, and both
  continue with letters, digits, ‹_› and ‹'›. ‹λ› (‹U+03BB›) is no letter, and
  ‹U+03A2› is unassigned. Operators are runs of ‹U+2190 .. U+22FF› (arrows and
  mathematical operators).›

datatype tok = LIDENT | UIDENT | OP

definition lower :: ‹nat cls› where ‹lower = [(0x61, 0x7A), (0x3B1, 0x3BA), (0x3BC, 0x3C9)]›
definition upper :: ‹nat cls› where ‹upper = [(0x41, 0x5A), (0x391, 0x3A1), (0x3A3, 0x3A9)]›
definition ‹idchar = lower @ upper @ [(0x30, 0x39), (0x5F, 0x5F), (0x27, 0x27)]›
definition opchar :: ‹nat cls› where ‹opchar = [(0x2190, 0x22FF)]›
definition space :: ‹nat cls› where ‹space = [(0x20, 0x20), (0x0A, 0x0A)]›

definition ‹many p = Times (Pred p) (Star (Pred p))›

definition cp_rules :: ‹(tok, nat cls, nat, nat) rule list list› where
  ‹cp_rules = [[
    ⦇match = Times (Pred lower) (Star (Pred idchar)), act = λ_ st. (Some LIDENT, st)⦈,
    ⦇match = Times (Pred upper) (Star (Pred idchar)), act = λ_ st. (Some UIDENT, st)⦈,
    ⦇match = many opchar, act = λ_ st. (Some OP, st)⦈,
    ⦇match = many space, act = λ_ st. (None, st)⦈]]›

definition ‹cp_tables_test = cp_lexer_tables cp_rules›

local_setup ‹eval_table (binding‹cp_tables_test_code›, term‹cp_tables_test›)›

declare cp_tables_test_code[code]

definition ‹is_lf c ⟷ c = (0x0A :: nat)›

definition ‹start = ⦇line = 0, col = 0, off = 0⦈›

definition ‹cp_lex s = cp_lexer cp_tables_test id (list_spec cp_rules) is_lf [0] start s›

text ‹On an input in range the lexer is the driver of the spec with the plain reading of
  classes:›

theorem cp_lex_iff_lexes:
  ‹∀c ∈ set s. c ≤ 0x10FFFF ⟹
    cp_lex s = (l, r, stf) ⟷
      symbolic.lexes in_cls (list_spec cp_rules) is_lf [0] start s l r stf›
  by (simp add: cp_lex_def cp_tables_test_def cp_lexer_list_iff_lexes_in_range)

text ‹‹αβ →⇒ Λ'›, then a newline and ‹x1›. Columns count codepoints:›

lemma ‹cp_lex [0x3B1, 0x3B2, 0x20, 0x2192, 0x21D2, 0x20, 0x39B, 0x27, 0x0A, 0x78, 0x31] =
  (Lexed []
    [⦇tok = LIDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = [0x3B1, 0x3B2]⦈,
      trivia = [0x20]⦈,
     ⦇tok = OP, span = ⦇s_line = 0, s_col = 3, s_off = 3, s_len = 2, s_txt = [0x2192, 0x21D2]⦈,
      trivia = [0x20]⦈,
     ⦇tok = UIDENT, span = ⦇s_line = 0, s_col = 6, s_off = 6, s_len = 2, s_txt = [0x39B, 0x27]⦈,
      trivia = [0x0A]⦈,
     ⦇tok = LIDENT, span = ⦇s_line = 1, s_col = 0, s_off = 9, s_len = 2, s_txt = [0x78, 0x31]⦈,
      trivia = []⦈],
   [], [0])›
  by eval

text ‹‹λx› stops at ‹λ›, and a surrogate stops too:›

lemma ‹cp_lex [0x3BB, 0x78] = (Lexed [] [], [0x3BB, 0x78], [0])›
  by eval

lemma ‹cp_lex [0x61, 0x20, 0xD800] =
  (Lexed []
    [⦇tok = LIDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 1, s_txt = [0x61]⦈,
      trivia = [0x20]⦈],
   [0xD800], [0])›
  by eval

text ‹A number above the range lexes as ‹U+10FFFF›. No class of ‹cp_rules› holds
  of it, and a class reaching ‹U+10FFFF› does:›

lemma ‹cp_lex [0x61, 0x110000] =
  (Lexed []
    [⦇tok = LIDENT, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 1, s_txt = [0x61]⦈,
      trivia = []⦈],
   [0x110000], [0])›
  by eval

definition wide_rules :: ‹(tok, nat cls, nat, nat) rule list list› where
  ‹wide_rules = [[⦇match = many [(0x80, 0x10FFFF)], act = λ_ st. (Some OP, st)⦈]]›

definition ‹wide_tables = cp_lexer_tables wide_rules›

local_setup ‹eval_table (binding‹wide_tables_code›, term‹wide_tables›)›

declare wide_tables_code[code]

lemma ‹cp_lexer wide_tables id (list_spec wide_rules) is_lf [0] start [0x10FFFF, 0x110000, 0x61] =
  (Lexed []
    [⦇tok = OP, span = ⦇s_line = 0, s_col = 0, s_off = 0, s_len = 2, s_txt = [0x10FFFF, 0x110000]⦈,
      trivia = []⦈],
   [0x61], [0])›
  by eval

end