Theory Lexer_Command_Tests

(* Author: Basil Fürer *)

theory Lexer_Command_Tests
  imports Codepoint_Tests TIGER_Lexer.Lexer_Command
begin

section ‹Tests of the lexer command›

subsection ‹Identifiers and operators›

text ‹The rules of ‹Codepoint_Tests› as a @{command lexer}. Without ‹modes› the mode
  type is typ‹unit›:›

lexer cp :: ‹tok›
  where
    lower = ‹[a-zα-κμ-ω]› | upper = ‹[A-ZΑ-ΡΣ-Ω]›
  | idchar = ‹lower | upper | [0-9_']›
  | many(p) = ‹p p*›
  rules
    ‹lower idchar*› ⇒ LIDENT
  | ‹upper idchar*› ⇒ UIDENT
  | ‹many([\u{2190}-\u{22FF}])› ⇒ OP
  | ‹many([ \n])› ⇒ skip

lemma ‹cp.spec () =
  [⦇match = Times (Pred cp.lower) (Star (Pred cp.idchar)),
     act = mk_act (λk m. m) (λu. Some LIDENT) (λu. True) Keep⦈,
   ⦇match = Times (Pred cp.upper) (Star (Pred cp.idchar)),
     act = mk_act (λk m. m) (λu. Some UIDENT) (λu. True) Keep⦈,
   ⦇match = Times (Pred [(0x2190, 0x22FF)]) (Star (Pred [(0x2190, 0x22FF)])),
     act = mk_act (λk m. m) (λu. Some OP) (λu. True) Keep⦈,
   ⦇match = Times (Pred [(0x0A, 0x0A), (0x20, 0x20)]) (Star (Pred [(0x0A, 0x0A), (0x20, 0x20)])),
     act = mk_act (λk m. m) (λu. None) (λu. True) Keep⦈]›
  by simp

text ‹The regexes are those of const‹cp_rules› up to the normal form of classes, and
  the actions agree:›

lemma ‹map (map_rexp cp_norm ∘ match) (cp.spec ()) = map (map_rexp cp_norm ∘ match) (hd cp_rules)›
  by eval

lemma ‹map (λr. fst (act r u [()])) (cp.spec ()) = [Some LIDENT, Some UIDENT, Some OP, None]›
  by (simp add: mk_act_def)

lemma ‹cp.start = ()› ‹cp.final ()› ‹cp.is_newline 0x0A› ‹¬ cp.is_newline 0x20›
  by (simp_all add: cp.start_def cp.final_def cp.is_newline_def)

subsection ‹Nested comments›

text ‹A mode for comments, entered by ‹push› and left by ‹pop›. ‹Comment› is exclusive,
  so the untagged rules are not active there:›

datatype ctok = Word | Num

lexer cmt :: ‹ctok›
  modes Code Comment
  exclusive Comment
  where letter = ‹[a-z]›
  rules
    ‹letter+› ⇒ Word
  | ‹[0-9]+› ⇒ Num
  | ‹[ \n]+› ⇒ skip
  | "(*" ⇒ skip, push Comment
  | <Comment> (
      "(*" ⇒ skip, push Comment
    | "*)" ⇒ skip, pop
    | ‹[^*(]+ | [*(]› ⇒ skip
  )

lemma ‹cmt.spec Comment =
  [⦇match = Times (Pred [(0x28, 0x28)]) (Pred [(0x2A, 0x2A)]),
     act = mk_act (λk m. m) (λu. None) (λu. True) (Push Comment)⦈,
   ⦇match = Times (Pred [(0x2A, 0x2A)]) (Pred [(0x29, 0x29)]),
     act = mk_act (λk m. m) (λu. None) (λu. True) Pop⦈,
   ⦇match =
      Plus (Times (Pred (cp_neg [(0x28, 0x28), (0x2A, 0x2A)]))
          (Star (Pred (cp_neg [(0x28, 0x28), (0x2A, 0x2A)]))))
        (Pred [(0x28, 0x28), (0x2A, 0x2A)]),
     act = mk_act (λk m. m) (λu. None) (λu. True) Keep⦈]›
  by simp

lemma ‹length (cmt.spec Code) = 4› ‹cmt.start = Code› ‹cmt.final Code› ‹¬ cmt.final Comment›
  by (simp_all add: cmt.start_def cmt.final_def)

text ‹A comment opens and closes, and a pop of the last mode is the empty stack:›

lemma
  ‹act (cmt.spec Code ! 3) u [Code] = (None, [Comment, Code])›
  ‹act (cmt.spec Comment ! 0) u [Comment, Code] = (None, [Comment, Comment, Code])›
  ‹act (cmt.spec Comment ! 1) u [Comment, Code] = (None, [Code])›
  ‹act (cmt.spec Comment ! 1) u [Comment] = (None, [])›
  by (simp_all add: mk_act_def)

subsection ‹The hook›

text ‹A minus is negation at the start of an operand and subtraction after one. The hook
  computes the mode from the token, trivia included, and a tag selects the rule:›

datatype otok = Id | Minus | Neg | LParen | RParen

datatype omode = Start | AfterOperand

fun after_tok :: ‹otok option ⇒ omode ⇒ omode› where
  ‹after_tok (Some Id) _ = AfterOperand›
| ‹after_tok (Some RParen) _ = AfterOperand›
| ‹after_tok _ _ = Start›

lexer ops :: ‹otok›
  modes ‹omode›
  after after_tok
  rules
    ‹[a-z]+› ⇒ Id | "(" ⇒ LParen | ")" ⇒ RParen | ‹[ ]+› ⇒ skip
  | <Start> "-" ⇒ Neg
  | <AfterOperand> "-" ⇒ Minus

lemma ‹map (λr. act r u [Start]) (ops.spec Start) =
  [(Some Id, [AfterOperand]), (Some LParen, [Start]), (Some RParen, [AfterOperand]),
   (None, [Start]), (Some Neg, [Start])]›
  by (simp add: mk_act_def)

lemma ‹map (λr. act r u [AfterOperand]) (ops.spec AfterOperand) =
  [(Some Id, [AfterOperand]), (Some LParen, [Start]), (Some RParen, [AfterOperand]),
   (None, [Start]), (Some Minus, [Start])]›
  by (simp add: mk_act_def)

subsection ‹Errors›

text ‹Each spec below fails with the given message. A spec is parsed from a string and
  elaborated in the theory:›

definition ‹dflt = Start›

ML ‹
fun lexer_fails ctxt text msg =
  let
    val toks =
      Token.explode (Thy_Header.get_keywords' ctxt) Position.none text
      |> filter Token.is_proper;
    val (opts, spec) =
      #1 (Scan.finite Token.stopper (Lexer_Syntax.options -- Lexer_Syntax.spec) toks);
    val thy = Proof_Context.theory_of ctxt;
  in
    (case Exn.capture (fn () => Named_Target.theory_map (Lexer_Syntax.lexer opts spec) thy) () of
      Exn.Res _ => error ("No error, expected " ^ quote msg)
    | Exn.Exn exn =>
        let val msg' = Runtime.exn_message exn in
          if String.isSubstring msg msg' then ()
          else error ("Expected " ^ quote msg ^ ", got " ^ quote msg')
        end)
  end;
›

ML_command ‹
  lexer_fails context "e :: ‹otok› rules \"x\" ⇒ Foo"
    "unknown token constructor \"Foo\"";
  lexer_fails context "e :: ‹otok› rules \"x\" ⇒ ‹1 :: nat›"
    "the action must be";
  lexer_fails context "e :: ‹otok› modes A B exclusive A B rules \"x\" ⇒ Id"
    "active in no mode";
  lexer_fails context "e :: ‹otok› modes ‹omode› rules <Foo> \"x\" ⇒ Id"
    "unknown mode constructor \"Foo\"";
  lexer_fails context "e :: ‹otok› modes ‹omode› rules <dflt> \"x\" ⇒ Id"
    "matches no mode";
  lexer_fails context
    "e :: ‹otok› where a = ‹b› | b = ‹[x]› rules ‹a› ⇒ Id"
    "\"b\" is used before its definition";
  lexer_fails context "e :: ‹otok› modes ‹omode› rules \"x\" ⇒ Id, push dflt"
    "is no mode";
  lexer_fails context "e :: ‹otok› modes ‹nat› rules \"x\" ⇒ Id"
    "is recursive";
  lexer_fails context "e :: ‹otok› rules \"x\" ⇒ error, pop"
    "error takes no guard and no mode operation"
›

end