Fixes to the lexer and the parser API.

* The parameter for logging the lexer is now mandatory.
  * The ParserAPI now thread the logging of the lexer.
  * LexerMain.ml now call the logging of the lexers (CameLIGO, ReasonLIGO).
  * Fixed bug in lexer when a line comment ends with EOF.

src/passes/1-parser/cameligo/LexerMain.ml

@@ -7,3 +7,8 @@ module IO =
   end
 
 module M = LexerUnit.Make (IO) (Lexer.Make (LexToken))
+
+let () =
+  match M.trace () with
+    Stdlib.Ok () -> ()
+  | Error msg -> Utils.highlight msg

src/passes/1-parser/pascaligo/LexerMain.ml

@@ -10,5 +10,5 @@ module M = LexerUnit.Make (IO) (Lexer.Make (LexToken))
 
 let () =
   match M.trace () with
-    Stdlib.Ok _ -> ()
+    Stdlib.Ok () -> ()
-  | Error msg -> Printf.eprintf "\027[31m%s\027[0m%!" msg
+  | Error msg -> Utils.highlight msg

src/passes/1-parser/reasonligo/LexerMain.ml

@@ -7,3 +7,8 @@ module IO =
   end
 
 module M = LexerUnit.Make (IO) (Lexer.Make (LexToken))
+
+let () =
+  match M.trace () with
+    Stdlib.Ok () -> ()
+  | Error msg -> Utils.highlight msg

src/passes/1-parser/shared/Lexer.mli

@@ -136,7 +136,7 @@ module type S =
     val slide : token -> window -> window
 
     type instance = {
-      read     : ?log:logger -> Lexing.lexbuf -> token;
+      read     : log:logger -> Lexing.lexbuf -> token;
       buffer   : Lexing.lexbuf;
       get_win  : unit -> window;
       get_pos  : unit -> Pos.t;

src/passes/1-parser/shared/Lexer.mll

@@ -158,7 +158,7 @@ module type S =
     val slide : token -> window -> window
 
     type instance = {
-      read     : ?log:logger -> Lexing.lexbuf -> token;
+      read     : log:logger -> Lexing.lexbuf -> token;
       buffer   : Lexing.lexbuf;
       get_win  : unit -> window;
       get_pos  : unit -> Pos.t;
@@ -424,13 +424,14 @@ module Make (Token: TOKEN) : (S with module Token = Token) =
          Hint: Remove the leading minus sign.\n"
     | Broken_string ->
         "The string starting here is interrupted by a line break.\n\
-         Hint: Remove the break, close the string before or insert a backslash.\n"
+         Hint: Remove the break, close the string before or insert a \
+         backslash.\n"
     | Invalid_character_in_string ->
         "Invalid character in string.\n\
          Hint: Remove or replace the character.\n"
     | Reserved_name s ->
-        "Reserved name: " ^ s ^ ".\n\
+        sprintf "Reserved name: \"%s\".\n\
-         Hint: Change the name.\n"
+         Hint: Change the name.\n" s
     | Invalid_symbol ->
         "Invalid symbol.\n\
          Hint: Check the LIGO syntax you use.\n"
@@ -779,7 +780,7 @@ and scan_line thread state = parse
              and thread = push_string nl thread
              and state  = {state with pos = state.pos#new_line nl}
              in thread, state }
-| eof      { fail thread.opening Unterminated_comment }
+| eof      { thread, state }
 | _        { let     () = rollback lexbuf in
              let len    = thread.len in
              let thread,
@@ -855,7 +856,7 @@ and scan_utf8 thread state = parse
 type logger = Markup.t list -> token -> unit
 
 type instance = {
-  read     : ?log:logger -> Lexing.lexbuf -> token;
+  read     : log:logger -> Lexing.lexbuf -> token;
   buffer   : Lexing.lexbuf;
   get_win  : unit -> window;
   get_pos  : unit -> Pos.t;
@@ -934,7 +935,7 @@ let open_token_stream file_path_opt =
             in fail region Missing_break
       | _ -> () in
 
-  let rec read_token ?(log=fun _ _ -> ()) buffer =
+  let rec read_token ~log buffer =
     match FQueue.deq !state.units with
       None ->
         scan buffer;

src/passes/1-parser/shared/ParserAPI.ml

@@ -2,6 +2,12 @@
 
 module Region = Simple_utils.Region
 
+module type IO =
+  sig
+    val ext : string              (* LIGO file extension *)
+    val options : EvalOpt.options (* CLI options *)
+  end
+
 module type PARSER =
   sig
     (* The type of tokens, abstract syntax trees and expressions *)
@@ -42,7 +48,8 @@ module type PARSER =
 
 (* Main functor *)
 
-module Make (Lexer: Lexer.S)
+module Make (IO : IO)
+            (Lexer: Lexer.S)
             (Parser: PARSER with type token = Lexer.Token.token)
             (ParErr: sig val message : int -> string end) =
   struct
@@ -116,18 +123,21 @@ module Make (Lexer: Lexer.S)
 
     module Incr = Parser.Incremental
 
+    module Log = LexerLog.Make (Lexer)
+    let log    = Log.output_token ~offsets:IO.options#offsets
+                                  IO.options#mode IO.options#cmd stdout
+
     let incr_contract Lexer.{read; buffer; get_win; close; _} =
-      let supplier = I.lexer_lexbuf_to_supplier read buffer
+      let supplier  = I.lexer_lexbuf_to_supplier (read ~log) buffer
-      and failure  = failure get_win in
+      and failure   = failure get_win in
-      let parser   = Incr.contract buffer.Lexing.lex_curr_p in
+      let parser    = Incr.contract buffer.Lexing.lex_curr_p in
-      let ast      = I.loop_handle success failure supplier parser
+      let ast       = I.loop_handle success failure supplier parser
       in close (); ast
 
-    let incr_expr  Lexer.{read; buffer; get_win; close; _} =
+    let incr_expr Lexer.{read; buffer; get_win; close; _} =
-      let supplier = I.lexer_lexbuf_to_supplier read buffer
+      let supplier   = I.lexer_lexbuf_to_supplier (read ~log) buffer
-      and failure  = failure get_win in
+      and failure    = failure get_win in
-      let parser   = Incr.interactive_expr buffer.Lexing.lex_curr_p in
+      let parser     = Incr.interactive_expr buffer.Lexing.lex_curr_p in
-      let expr     = I.loop_handle success failure supplier parser
+      let expr       = I.loop_handle success failure supplier parser
       in close (); expr
-
   end

src/passes/1-parser/shared/ParserAPI.mli

@@ -2,6 +2,12 @@
 
 module Region = Simple_utils.Region
 
+module type IO =
+  sig
+    val ext : string              (* LIGO file extension *)
+    val options : EvalOpt.options (* CLI options *)
+  end
+
 (* The signature generated by Menhir with additional type definitions
    for [ast] and [expr]. *)
 
@@ -43,7 +49,8 @@ module type PARSER =
       end
   end
 
-module Make (Lexer: Lexer.S)
+module Make (IO: IO)
+            (Lexer: Lexer.S)
             (Parser: PARSER with type token = Lexer.Token.token)
             (ParErr: sig val message : int -> string end) :
   sig

src/passes/1-parser/shared/ParserUnit.ml

@@ -85,7 +85,7 @@ module Make (Lexer: Lexer.S)
 
     (* Instantiating the parser *)
 
-    module Front = ParserAPI.Make (Lexer)(Parser)(ParErr)
+    module Front = ParserAPI.Make (IO)(Lexer)(Parser)(ParErr)
 
     let format_error = Front.format_error