Added basic support for Menhir's incremental API.

I added the token Bytes to ReasonLIGO's [LexToken.mll] for the build.

src/passes/1-parser/cameligo/.Parser.mly.tag

@@ -1 +1 @@
---explain --external-tokens LexToken --base Parser ParToken.mly
+--table --strict --explain --external-tokens LexToken --base Parser ParToken.mly

src/passes/1-parser/cameligo/ParserAPI.ml

@@ -0,0 +1,57 @@
+(** Generic parser for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) =
+  struct
+
+    module I = Parser.MenhirInterpreter
+
+    (* The parser has successfully produced a semantic value. *)
+
+    let success v = v
+
+    (* The parser has suspended itself because of a syntax error. Stop. *)
+
+    let fail _checkpoint = raise Parser.Error
+
+    (* The generic parsing function *)
+
+    let incr_contract Lexer.{read; buffer; close; _} : AST.t =
+      let supplier = I.lexer_lexbuf_to_supplier read buffer in
+      let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in
+      let ast = I.loop_handle success fail supplier parser
+      in close (); ast
+
+    let mono_contract = Parser.contract
+
+  end

src/passes/1-parser/cameligo/ParserAPI.mli

@@ -0,0 +1,39 @@
+(** Generic parser API for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) :
+  sig
+    val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t
+    val incr_contract : Lexer.instance -> AST.t
+  end

src/passes/1-parser/cameligo/ParserMain.ml

@@ -76,11 +76,11 @@ let () =
 (** {1 Instanciating the lexer} *)
 
 module Lexer = Lexer.Make (LexToken)
-
 module Log = LexerLog.Make (Lexer)
+module ParserFront = ParserAPI.Make (Lexer) (Parser)
 
-let Lexer.{read; buffer; get_pos; get_last; close} =
-  Lexer.open_token_stream (Some pp_input)
+let lexer_inst = Lexer.open_token_stream (Some pp_input)
+let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst
 
 and cout = stdout
 
@@ -97,7 +97,10 @@ let tokeniser = read ~log
 
 let () =
   try
-    let ast = Parser.contract tokeniser buffer in
+    (* The incremental API *)
+    let ast = ParserFront.incr_contract lexer_inst in
+    (* The monolithic API *)
+    (* let ast = ParserFront.mono_contract tokeniser buffer in *)
     if Utils.String.Set.mem "ast" options#verbose
     then let buffer = Buffer.create 131 in
          let state = ParserLog.mk_state

src/passes/1-parser/cameligo/dune

@@ -3,38 +3,33 @@
 (menhir
  (merge_into Parser)
  (modules ParToken Parser)
- (flags -la 1 --explain --external-tokens LexToken))
+ (flags -la 1 --table --strict --explain --external-tokens LexToken))
 
 (library
  (name parser_cameligo)
  (public_name ligo.parser.cameligo)
  (modules AST cameligo Parser ParserLog LexToken)
  (libraries
+   menhirLib
    parser_shared
    str
    simple-utils
    tezos-utils
-   getopt   
- )
- (flags (:standard -open Simple_utils -open Parser_shared ))
-)
+   getopt)
+ (flags (:standard -open Simple_utils -open Parser_shared )))
 
 (executable
   (name LexerMain)
-  (libraries 
+  (libraries
    parser_cameligo)
-  (modules 
-    LexerMain
-  )
-  (flags (:standard -open Parser_shared -open Parser_cameligo))
-)
+  (modules
+    LexerMain)
+  (flags (:standard -open Parser_shared -open Parser_cameligo)))
 
 (executable
   (name ParserMain)
-  (libraries 
+  (libraries
    parser_cameligo)
-  (modules    
-    ParserMain
-  )
-  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo))
-)
+  (modules
+    ParserMain)
+  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo)))

src/passes/1-parser/pascaligo/.Parser.mly.tag

@@ -1 +1 @@
---table --explain --external-tokens LexToken --base Parser ParToken.mly
+--table --strict --explain --external-tokens LexToken --base Parser ParToken.mly

src/passes/1-parser/pascaligo/ParserAPI.ml

@@ -0,0 +1,57 @@
+(** Generic parser for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) =
+  struct
+
+    module I = Parser.MenhirInterpreter
+
+    (* The parser has successfully produced a semantic value. *)
+
+    let success v = v
+
+    (* The parser has suspended itself because of a syntax error. Stop. *)
+
+    let fail _checkpoint = raise Parser.Error
+
+    (* The generic parsing function *)
+
+    let incr_contract Lexer.{read; buffer; close; _} : AST.t =
+      let supplier = I.lexer_lexbuf_to_supplier read buffer in
+      let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in
+      let ast = I.loop_handle success fail supplier parser
+      in close (); ast
+
+    let mono_contract = Parser.contract
+
+  end

src/passes/1-parser/pascaligo/ParserAPI.mli

@@ -0,0 +1,39 @@
+(** Generic parser API for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) :
+  sig
+    val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t
+    val incr_contract : Lexer.instance -> AST.t
+  end

src/passes/1-parser/pascaligo/ParserMain.ml

@@ -76,11 +76,11 @@ let () =
 (** {1 Instanciating the lexer} *)
 
 module Lexer = Lexer.Make (LexToken)
-
 module Log = LexerLog.Make (Lexer)
+module ParserFront = ParserAPI.Make (Lexer) (Parser)
 
-let Lexer.{read; buffer; get_pos; get_last; close} =
-  Lexer.open_token_stream (Some pp_input)
+let lexer_inst = Lexer.open_token_stream (Some pp_input)
+let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst
 
 and cout = stdout
 
@@ -97,7 +97,10 @@ let tokeniser = read ~log
 
 let () =
   try
-    let ast = Parser.contract tokeniser buffer in
+    (* The incremental API *)
+    let ast = ParserFront.incr_contract lexer_inst in
+    (* The monolithic API *)
+    (* let ast = ParserFront.mono_contract tokeniser buffer in *)
     if Utils.String.Set.mem "ast" options#verbose
     then let buffer = Buffer.create 131 in
          let state = ParserLog.mk_state

src/passes/1-parser/pascaligo/dune

@@ -3,18 +3,18 @@
 (menhir
  (merge_into Parser)
  (modules ParToken Parser)
- (flags -la 1 --explain --external-tokens LexToken))
+ (flags -la 1 --table --strict --explain --external-tokens LexToken))
 
 (library
  (name parser_pascaligo)
  (public_name ligo.parser.pascaligo)
  (modules AST pascaligo Parser ParserLog LexToken)
  (libraries
+   menhirLib
    parser_shared
    hex
    simple-utils
-   tezos-utils
- )
+   tezos-utils)
  (flags (:standard  -open Parser_shared -open Simple_utils))
 )
 
@@ -26,20 +26,16 @@
    tezos-utils
    parser_pascaligo)
   (modules
-    LexerMain
-  )
-  (flags (:standard -open Parser_shared -open Parser_pascaligo))
-)
+    LexerMain)
+  (flags (:standard -open Parser_shared -open Parser_pascaligo)))
 
 (executable
   (name ParserMain)
   (libraries
    parser_pascaligo)
   (modules
-    ParserMain
-  )
-  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_pascaligo))
-)
+    ParserMain)
+  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_pascaligo)))
 
 ;; Les deux directives (rule) qui suivent sont pour le dev local.
 ;; Il suffit de faire "dune build Parser.exe" pour avoir un Parser.exe dans le dossier.

src/passes/1-parser/reasonligo/.Parser.mly.tag

@@ -1 +1 @@
---explain --external-tokens LexToken --base Parser ParToken.mly
+--table --explain --external-tokens LexToken --base Parser ParToken.mly

src/passes/1-parser/reasonligo/ParToken.mly

@@ -5,12 +5,13 @@
 
   (* Literals *)
 
-%token         <string Region.reg> Ident  "<ident>"
-%token         <string Region.reg> Constr "<constr>"
-%token         <string Region.reg> String "<string>"
-%token <(string * Z.t) Region.reg> Int    "<int>"
-%token <(string * Z.t) Region.reg> Nat    "<nat>"
-%token <(string * Z.t) Region.reg> Mutez  "<mutez>"
+%token                    <string Region.reg> String "<string>"
+%token <(LexToken.lexeme * Hex.t) Region.reg> Bytes  "<bytes>"
+%token            <(string * Z.t) Region.reg> Int    "<int>"
+%token            <(string * Z.t) Region.reg> Nat    "<nat>"
+%token            <(string * Z.t) Region.reg> Mutez  "<mutez>"
+%token                    <string Region.reg> Ident  "<ident>"
+%token                    <string Region.reg> Constr "<constr>"
 
   (* Symbols *)
 

src/passes/1-parser/reasonligo/ParserAPI.ml

@@ -0,0 +1,57 @@
+(** Generic parser for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) =
+  struct
+
+    module I = Parser.MenhirInterpreter
+
+    (* The parser has successfully produced a semantic value. *)
+
+    let success v = v
+
+    (* The parser has suspended itself because of a syntax error. Stop. *)
+
+    let fail _checkpoint = raise Parser.Error
+
+    (* The generic parsing function *)
+
+    let incr_contract Lexer.{read; buffer; close; _} : AST.t =
+      let supplier = I.lexer_lexbuf_to_supplier read buffer in
+      let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in
+      let ast = I.loop_handle success fail supplier parser
+      in close (); ast
+
+    let mono_contract = Parser.contract
+
+  end

src/passes/1-parser/reasonligo/ParserAPI.mli

@@ -0,0 +1,39 @@
+(** Generic parser API for LIGO *)
+
+module type PARSER =
+  sig
+    (* The type of tokens *)
+
+    type token
+
+    (* This exception is raised by the monolithic API functions *)
+
+    exception Error
+
+    (* The monolithic API *)
+
+    val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t
+
+    (* The incremental API *)
+
+    module MenhirInterpreter :
+      sig
+        include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE
+                with type token = token
+      end
+
+    module Incremental :
+      sig
+        val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint
+      end
+
+  end
+
+(* Main functor *)
+
+module Make (Lexer: Lexer.S)
+            (Parser: PARSER with type token = Lexer.Token.token) :
+  sig
+    val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t
+    val incr_contract : Lexer.instance -> AST.t
+  end

src/passes/1-parser/reasonligo/ParserMain.ml

@@ -76,11 +76,11 @@ let () =
 (** {1 Instanciating the lexer} *)
 
 module Lexer = Lexer.Make (LexToken)
-
 module Log = LexerLog.Make (Lexer)
+module ParserFront = ParserAPI.Make (Lexer) (Parser)
 
-let Lexer.{read; buffer; get_pos; get_last; close} =
-  Lexer.open_token_stream (Some pp_input)
+let lexer_inst = Lexer.open_token_stream (Some pp_input)
+let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst
 
 and cout = stdout
 
@@ -97,7 +97,10 @@ let tokeniser = read ~log
 
 let () =
   try
-    let ast = Parser.contract tokeniser buffer in
+    (* The incremental API *)
+    let ast = ParserFront.incr_contract lexer_inst in
+    (* The monolithic API *)
+    (* let ast = ParserFront.mono_contract tokeniser buffer in *)
     if Utils.String.Set.mem "ast" options#verbose
     then let buffer = Buffer.create 131 in
          let state = ParserLog.mk_state

src/passes/1-parser/reasonligo/dune

@@ -3,39 +3,34 @@
 (menhir
  (merge_into Parser)
  (modules ParToken Parser)
- (flags -la 1 --explain --dump --strict --external-tokens LexToken))
+ (flags -la 1 --table --explain --strict --external-tokens LexToken))
 
 (library
  (name parser_reasonligo)
  (public_name ligo.parser.reasonligo)
  (modules reasonligo LexToken Parser)
  (libraries
+   menhirLib
    parser_shared
    parser_cameligo
    str
    simple-utils
    tezos-utils
-   getopt   
- )
- (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo ))
-)
+   getopt)
+ (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo)))
 
 (executable
   (name LexerMain)
-  (libraries 
+  (libraries
    parser_reasonligo)
-  (modules 
-    LexerMain
-  )
-  (flags (:standard -open Parser_shared -open Parser_reasonligo))
-)
+  (modules
+    LexerMain)
+  (flags (:standard -open Parser_shared -open Parser_reasonligo)))
 
 (executable
   (name ParserMain)
-  (libraries 
+  (libraries
    parser_reasonligo)
-  (modules    
-    ParserMain
-  )
-  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_reasonligo))
-)
+  (modules
+    ParserMain)
+  (flags (:standard -open Simple_utils -open Parser_shared -open Parser_reasonligo)))