From a9b3d295fd60f258a00cb197524361dac9b8db9a Mon Sep 17 00:00:00 2001 From: Christian Rinderknecht Date: Fri, 31 Jan 2020 12:31:25 +0100 Subject: [PATCH] Fixes to the lexer and the parser API. * The parameter for logging the lexer is now mandatory. * The ParserAPI now thread the logging of the lexer. * LexerMain.ml now call the logging of the lexers (CameLIGO, ReasonLIGO). * Fixed bug in lexer when a line comment ends with EOF. --- src/passes/1-parser/cameligo/LexerMain.ml | 5 ++++ src/passes/1-parser/pascaligo/LexerMain.ml | 4 +-- src/passes/1-parser/reasonligo/LexerMain.ml | 5 ++++ src/passes/1-parser/shared/Lexer.mli | 2 +- src/passes/1-parser/shared/Lexer.mll | 15 +++++----- src/passes/1-parser/shared/ParserAPI.ml | 32 ++++++++++++++------- src/passes/1-parser/shared/ParserAPI.mli | 9 +++++- src/passes/1-parser/shared/ParserUnit.ml | 2 +- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/passes/1-parser/cameligo/LexerMain.ml b/src/passes/1-parser/cameligo/LexerMain.ml index e9775b803..6c0729851 100644 --- a/src/passes/1-parser/cameligo/LexerMain.ml +++ b/src/passes/1-parser/cameligo/LexerMain.ml @@ -7,3 +7,8 @@ module IO = end module M = LexerUnit.Make (IO) (Lexer.Make (LexToken)) + +let () = + match M.trace () with + Stdlib.Ok () -> () + | Error msg -> Utils.highlight msg diff --git a/src/passes/1-parser/pascaligo/LexerMain.ml b/src/passes/1-parser/pascaligo/LexerMain.ml index 042b0930a..ba2925172 100644 --- a/src/passes/1-parser/pascaligo/LexerMain.ml +++ b/src/passes/1-parser/pascaligo/LexerMain.ml @@ -10,5 +10,5 @@ module M = LexerUnit.Make (IO) (Lexer.Make (LexToken)) let () = match M.trace () with - Stdlib.Ok _ -> () - | Error msg -> Printf.eprintf "\027[31m%s\027[0m%!" msg + Stdlib.Ok () -> () + | Error msg -> Utils.highlight msg diff --git a/src/passes/1-parser/reasonligo/LexerMain.ml b/src/passes/1-parser/reasonligo/LexerMain.ml index 756a2f103..8681f49e8 100644 --- a/src/passes/1-parser/reasonligo/LexerMain.ml +++ b/src/passes/1-parser/reasonligo/LexerMain.ml @@ -7,3 +7,8 @@ module IO = end module M = LexerUnit.Make (IO) (Lexer.Make (LexToken)) + +let () = + match M.trace () with + Stdlib.Ok () -> () + | Error msg -> Utils.highlight msg diff --git a/src/passes/1-parser/shared/Lexer.mli b/src/passes/1-parser/shared/Lexer.mli index 1d338d719..034f8b252 100644 --- a/src/passes/1-parser/shared/Lexer.mli +++ b/src/passes/1-parser/shared/Lexer.mli @@ -136,7 +136,7 @@ module type S = val slide : token -> window -> window type instance = { - read : ?log:logger -> Lexing.lexbuf -> token; + read : log:logger -> Lexing.lexbuf -> token; buffer : Lexing.lexbuf; get_win : unit -> window; get_pos : unit -> Pos.t; diff --git a/src/passes/1-parser/shared/Lexer.mll b/src/passes/1-parser/shared/Lexer.mll index 73b33b804..97d864d52 100644 --- a/src/passes/1-parser/shared/Lexer.mll +++ b/src/passes/1-parser/shared/Lexer.mll @@ -158,7 +158,7 @@ module type S = val slide : token -> window -> window type instance = { - read : ?log:logger -> Lexing.lexbuf -> token; + read : log:logger -> Lexing.lexbuf -> token; buffer : Lexing.lexbuf; get_win : unit -> window; get_pos : unit -> Pos.t; @@ -424,13 +424,14 @@ module Make (Token: TOKEN) : (S with module Token = Token) = Hint: Remove the leading minus sign.\n" | Broken_string -> "The string starting here is interrupted by a line break.\n\ - Hint: Remove the break, close the string before or insert a backslash.\n" + Hint: Remove the break, close the string before or insert a \ + backslash.\n" | Invalid_character_in_string -> "Invalid character in string.\n\ Hint: Remove or replace the character.\n" | Reserved_name s -> - "Reserved name: " ^ s ^ ".\n\ - Hint: Change the name.\n" + sprintf "Reserved name: \"%s\".\n\ + Hint: Change the name.\n" s | Invalid_symbol -> "Invalid symbol.\n\ Hint: Check the LIGO syntax you use.\n" @@ -779,7 +780,7 @@ and scan_line thread state = parse and thread = push_string nl thread and state = {state with pos = state.pos#new_line nl} in thread, state } -| eof { fail thread.opening Unterminated_comment } +| eof { thread, state } | _ { let () = rollback lexbuf in let len = thread.len in let thread, @@ -855,7 +856,7 @@ and scan_utf8 thread state = parse type logger = Markup.t list -> token -> unit type instance = { - read : ?log:logger -> Lexing.lexbuf -> token; + read : log:logger -> Lexing.lexbuf -> token; buffer : Lexing.lexbuf; get_win : unit -> window; get_pos : unit -> Pos.t; @@ -934,7 +935,7 @@ let open_token_stream file_path_opt = in fail region Missing_break | _ -> () in - let rec read_token ?(log=fun _ _ -> ()) buffer = + let rec read_token ~log buffer = match FQueue.deq !state.units with None -> scan buffer; diff --git a/src/passes/1-parser/shared/ParserAPI.ml b/src/passes/1-parser/shared/ParserAPI.ml index aabb1efef..2f0ed7598 100644 --- a/src/passes/1-parser/shared/ParserAPI.ml +++ b/src/passes/1-parser/shared/ParserAPI.ml @@ -2,6 +2,12 @@ module Region = Simple_utils.Region +module type IO = + sig + val ext : string (* LIGO file extension *) + val options : EvalOpt.options (* CLI options *) + end + module type PARSER = sig (* The type of tokens, abstract syntax trees and expressions *) @@ -42,7 +48,8 @@ module type PARSER = (* Main functor *) -module Make (Lexer: Lexer.S) +module Make (IO : IO) + (Lexer: Lexer.S) (Parser: PARSER with type token = Lexer.Token.token) (ParErr: sig val message : int -> string end) = struct @@ -116,18 +123,21 @@ module Make (Lexer: Lexer.S) module Incr = Parser.Incremental + module Log = LexerLog.Make (Lexer) + let log = Log.output_token ~offsets:IO.options#offsets + IO.options#mode IO.options#cmd stdout + let incr_contract Lexer.{read; buffer; get_win; close; _} = - let supplier = I.lexer_lexbuf_to_supplier read buffer - and failure = failure get_win in - let parser = Incr.contract buffer.Lexing.lex_curr_p in - let ast = I.loop_handle success failure supplier parser + let supplier = I.lexer_lexbuf_to_supplier (read ~log) buffer + and failure = failure get_win in + let parser = Incr.contract buffer.Lexing.lex_curr_p in + let ast = I.loop_handle success failure supplier parser in close (); ast - let incr_expr Lexer.{read; buffer; get_win; close; _} = - let supplier = I.lexer_lexbuf_to_supplier read buffer - and failure = failure get_win in - let parser = Incr.interactive_expr buffer.Lexing.lex_curr_p in - let expr = I.loop_handle success failure supplier parser + let incr_expr Lexer.{read; buffer; get_win; close; _} = + let supplier = I.lexer_lexbuf_to_supplier (read ~log) buffer + and failure = failure get_win in + let parser = Incr.interactive_expr buffer.Lexing.lex_curr_p in + let expr = I.loop_handle success failure supplier parser in close (); expr - end diff --git a/src/passes/1-parser/shared/ParserAPI.mli b/src/passes/1-parser/shared/ParserAPI.mli index 396a8698c..6fce46381 100644 --- a/src/passes/1-parser/shared/ParserAPI.mli +++ b/src/passes/1-parser/shared/ParserAPI.mli @@ -2,6 +2,12 @@ module Region = Simple_utils.Region +module type IO = + sig + val ext : string (* LIGO file extension *) + val options : EvalOpt.options (* CLI options *) + end + (* The signature generated by Menhir with additional type definitions for [ast] and [expr]. *) @@ -43,7 +49,8 @@ module type PARSER = end end -module Make (Lexer: Lexer.S) +module Make (IO: IO) + (Lexer: Lexer.S) (Parser: PARSER with type token = Lexer.Token.token) (ParErr: sig val message : int -> string end) : sig diff --git a/src/passes/1-parser/shared/ParserUnit.ml b/src/passes/1-parser/shared/ParserUnit.ml index dff827a56..eb4eb61d1 100644 --- a/src/passes/1-parser/shared/ParserUnit.ml +++ b/src/passes/1-parser/shared/ParserUnit.ml @@ -85,7 +85,7 @@ module Make (Lexer: Lexer.S) (* Instantiating the parser *) - module Front = ParserAPI.Make (Lexer)(Parser)(ParErr) + module Front = ParserAPI.Make (IO)(Lexer)(Parser)(ParErr) let format_error = Front.format_error