From 8210a4e1863d9a1e61683e2e6d96331054a03200 Mon Sep 17 00:00:00 2001 From: Christian Rinderknecht Date: Tue, 17 Dec 2019 17:03:43 +0100 Subject: [PATCH] Added basic support for Menhir's incremental API. I added the token Bytes to ReasonLIGO's [LexToken.mll] for the build. --- src/passes/1-parser/cameligo/.Parser.mly.tag | 2 +- src/passes/1-parser/cameligo/ParserAPI.ml | 57 +++++++++++++++++++ src/passes/1-parser/cameligo/ParserAPI.mli | 39 +++++++++++++ src/passes/1-parser/cameligo/ParserMain.ml | 11 ++-- src/passes/1-parser/cameligo/dune | 29 ++++------ src/passes/1-parser/pascaligo/.Parser.mly.tag | 2 +- src/passes/1-parser/pascaligo/ParserAPI.ml | 57 +++++++++++++++++++ src/passes/1-parser/pascaligo/ParserAPI.mli | 39 +++++++++++++ src/passes/1-parser/pascaligo/ParserMain.ml | 11 ++-- src/passes/1-parser/pascaligo/dune | 18 +++--- .../1-parser/reasonligo/.Parser.mly.tag | 2 +- src/passes/1-parser/reasonligo/ParToken.mly | 13 +++-- src/passes/1-parser/reasonligo/ParserAPI.ml | 57 +++++++++++++++++++ src/passes/1-parser/reasonligo/ParserAPI.mli | 39 +++++++++++++ src/passes/1-parser/reasonligo/ParserMain.ml | 11 ++-- src/passes/1-parser/reasonligo/dune | 29 ++++------ 16 files changed, 350 insertions(+), 66 deletions(-) create mode 100644 src/passes/1-parser/cameligo/ParserAPI.ml create mode 100644 src/passes/1-parser/cameligo/ParserAPI.mli create mode 100644 src/passes/1-parser/pascaligo/ParserAPI.ml create mode 100644 src/passes/1-parser/pascaligo/ParserAPI.mli create mode 100644 src/passes/1-parser/reasonligo/ParserAPI.ml create mode 100644 src/passes/1-parser/reasonligo/ParserAPI.mli diff --git a/src/passes/1-parser/cameligo/.Parser.mly.tag b/src/passes/1-parser/cameligo/.Parser.mly.tag index 100f7bb69..37b0cae8c 100644 --- a/src/passes/1-parser/cameligo/.Parser.mly.tag +++ b/src/passes/1-parser/cameligo/.Parser.mly.tag @@ -1 +1 @@ ---explain --external-tokens LexToken --base Parser ParToken.mly +--table --strict --explain --external-tokens LexToken --base Parser ParToken.mly diff --git a/src/passes/1-parser/cameligo/ParserAPI.ml b/src/passes/1-parser/cameligo/ParserAPI.ml new file mode 100644 index 000000000..7ae5c5ad4 --- /dev/null +++ b/src/passes/1-parser/cameligo/ParserAPI.ml @@ -0,0 +1,57 @@ +(** Generic parser for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) = + struct + + module I = Parser.MenhirInterpreter + + (* The parser has successfully produced a semantic value. *) + + let success v = v + + (* The parser has suspended itself because of a syntax error. Stop. *) + + let fail _checkpoint = raise Parser.Error + + (* The generic parsing function *) + + let incr_contract Lexer.{read; buffer; close; _} : AST.t = + let supplier = I.lexer_lexbuf_to_supplier read buffer in + let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in + let ast = I.loop_handle success fail supplier parser + in close (); ast + + let mono_contract = Parser.contract + + end diff --git a/src/passes/1-parser/cameligo/ParserAPI.mli b/src/passes/1-parser/cameligo/ParserAPI.mli new file mode 100644 index 000000000..ff3fe4854 --- /dev/null +++ b/src/passes/1-parser/cameligo/ParserAPI.mli @@ -0,0 +1,39 @@ +(** Generic parser API for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) : + sig + val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t + val incr_contract : Lexer.instance -> AST.t + end diff --git a/src/passes/1-parser/cameligo/ParserMain.ml b/src/passes/1-parser/cameligo/ParserMain.ml index e1e35850b..faa7ce70a 100644 --- a/src/passes/1-parser/cameligo/ParserMain.ml +++ b/src/passes/1-parser/cameligo/ParserMain.ml @@ -76,11 +76,11 @@ let () = (** {1 Instanciating the lexer} *) module Lexer = Lexer.Make (LexToken) - module Log = LexerLog.Make (Lexer) +module ParserFront = ParserAPI.Make (Lexer) (Parser) -let Lexer.{read; buffer; get_pos; get_last; close} = - Lexer.open_token_stream (Some pp_input) +let lexer_inst = Lexer.open_token_stream (Some pp_input) +let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst and cout = stdout @@ -97,7 +97,10 @@ let tokeniser = read ~log let () = try - let ast = Parser.contract tokeniser buffer in + (* The incremental API *) + let ast = ParserFront.incr_contract lexer_inst in + (* The monolithic API *) + (* let ast = ParserFront.mono_contract tokeniser buffer in *) if Utils.String.Set.mem "ast" options#verbose then let buffer = Buffer.create 131 in let state = ParserLog.mk_state diff --git a/src/passes/1-parser/cameligo/dune b/src/passes/1-parser/cameligo/dune index 31e31a857..ed667617a 100644 --- a/src/passes/1-parser/cameligo/dune +++ b/src/passes/1-parser/cameligo/dune @@ -3,38 +3,33 @@ (menhir (merge_into Parser) (modules ParToken Parser) - (flags -la 1 --explain --external-tokens LexToken)) + (flags -la 1 --table --strict --explain --external-tokens LexToken)) (library (name parser_cameligo) (public_name ligo.parser.cameligo) (modules AST cameligo Parser ParserLog LexToken) (libraries + menhirLib parser_shared str simple-utils tezos-utils - getopt - ) - (flags (:standard -open Simple_utils -open Parser_shared )) -) + getopt) + (flags (:standard -open Simple_utils -open Parser_shared ))) (executable (name LexerMain) - (libraries + (libraries parser_cameligo) - (modules - LexerMain - ) - (flags (:standard -open Parser_shared -open Parser_cameligo)) -) + (modules + LexerMain) + (flags (:standard -open Parser_shared -open Parser_cameligo))) (executable (name ParserMain) - (libraries + (libraries parser_cameligo) - (modules - ParserMain - ) - (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo)) -) + (modules + ParserMain) + (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo))) diff --git a/src/passes/1-parser/pascaligo/.Parser.mly.tag b/src/passes/1-parser/pascaligo/.Parser.mly.tag index ab6790b0f..37b0cae8c 100644 --- a/src/passes/1-parser/pascaligo/.Parser.mly.tag +++ b/src/passes/1-parser/pascaligo/.Parser.mly.tag @@ -1 +1 @@ ---table --explain --external-tokens LexToken --base Parser ParToken.mly +--table --strict --explain --external-tokens LexToken --base Parser ParToken.mly diff --git a/src/passes/1-parser/pascaligo/ParserAPI.ml b/src/passes/1-parser/pascaligo/ParserAPI.ml new file mode 100644 index 000000000..7ae5c5ad4 --- /dev/null +++ b/src/passes/1-parser/pascaligo/ParserAPI.ml @@ -0,0 +1,57 @@ +(** Generic parser for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) = + struct + + module I = Parser.MenhirInterpreter + + (* The parser has successfully produced a semantic value. *) + + let success v = v + + (* The parser has suspended itself because of a syntax error. Stop. *) + + let fail _checkpoint = raise Parser.Error + + (* The generic parsing function *) + + let incr_contract Lexer.{read; buffer; close; _} : AST.t = + let supplier = I.lexer_lexbuf_to_supplier read buffer in + let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in + let ast = I.loop_handle success fail supplier parser + in close (); ast + + let mono_contract = Parser.contract + + end diff --git a/src/passes/1-parser/pascaligo/ParserAPI.mli b/src/passes/1-parser/pascaligo/ParserAPI.mli new file mode 100644 index 000000000..ff3fe4854 --- /dev/null +++ b/src/passes/1-parser/pascaligo/ParserAPI.mli @@ -0,0 +1,39 @@ +(** Generic parser API for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) : + sig + val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t + val incr_contract : Lexer.instance -> AST.t + end diff --git a/src/passes/1-parser/pascaligo/ParserMain.ml b/src/passes/1-parser/pascaligo/ParserMain.ml index 295d460d8..8e64c56eb 100644 --- a/src/passes/1-parser/pascaligo/ParserMain.ml +++ b/src/passes/1-parser/pascaligo/ParserMain.ml @@ -76,11 +76,11 @@ let () = (** {1 Instanciating the lexer} *) module Lexer = Lexer.Make (LexToken) - module Log = LexerLog.Make (Lexer) +module ParserFront = ParserAPI.Make (Lexer) (Parser) -let Lexer.{read; buffer; get_pos; get_last; close} = - Lexer.open_token_stream (Some pp_input) +let lexer_inst = Lexer.open_token_stream (Some pp_input) +let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst and cout = stdout @@ -97,7 +97,10 @@ let tokeniser = read ~log let () = try - let ast = Parser.contract tokeniser buffer in + (* The incremental API *) + let ast = ParserFront.incr_contract lexer_inst in + (* The monolithic API *) + (* let ast = ParserFront.mono_contract tokeniser buffer in *) if Utils.String.Set.mem "ast" options#verbose then let buffer = Buffer.create 131 in let state = ParserLog.mk_state diff --git a/src/passes/1-parser/pascaligo/dune b/src/passes/1-parser/pascaligo/dune index 03d27a37c..ab405f17b 100644 --- a/src/passes/1-parser/pascaligo/dune +++ b/src/passes/1-parser/pascaligo/dune @@ -3,18 +3,18 @@ (menhir (merge_into Parser) (modules ParToken Parser) - (flags -la 1 --explain --external-tokens LexToken)) + (flags -la 1 --table --strict --explain --external-tokens LexToken)) (library (name parser_pascaligo) (public_name ligo.parser.pascaligo) (modules AST pascaligo Parser ParserLog LexToken) (libraries + menhirLib parser_shared hex simple-utils - tezos-utils - ) + tezos-utils) (flags (:standard -open Parser_shared -open Simple_utils)) ) @@ -26,20 +26,16 @@ tezos-utils parser_pascaligo) (modules - LexerMain - ) - (flags (:standard -open Parser_shared -open Parser_pascaligo)) -) + LexerMain) + (flags (:standard -open Parser_shared -open Parser_pascaligo))) (executable (name ParserMain) (libraries parser_pascaligo) (modules - ParserMain - ) - (flags (:standard -open Simple_utils -open Parser_shared -open Parser_pascaligo)) -) + ParserMain) + (flags (:standard -open Simple_utils -open Parser_shared -open Parser_pascaligo))) ;; Les deux directives (rule) qui suivent sont pour le dev local. ;; Il suffit de faire "dune build Parser.exe" pour avoir un Parser.exe dans le dossier. diff --git a/src/passes/1-parser/reasonligo/.Parser.mly.tag b/src/passes/1-parser/reasonligo/.Parser.mly.tag index 100f7bb69..ab6790b0f 100644 --- a/src/passes/1-parser/reasonligo/.Parser.mly.tag +++ b/src/passes/1-parser/reasonligo/.Parser.mly.tag @@ -1 +1 @@ ---explain --external-tokens LexToken --base Parser ParToken.mly +--table --explain --external-tokens LexToken --base Parser ParToken.mly diff --git a/src/passes/1-parser/reasonligo/ParToken.mly b/src/passes/1-parser/reasonligo/ParToken.mly index 561f95265..4a94ddb6b 100644 --- a/src/passes/1-parser/reasonligo/ParToken.mly +++ b/src/passes/1-parser/reasonligo/ParToken.mly @@ -5,12 +5,13 @@ (* Literals *) -%token Ident "" -%token Constr "" -%token String "" -%token <(string * Z.t) Region.reg> Int "" -%token <(string * Z.t) Region.reg> Nat "" -%token <(string * Z.t) Region.reg> Mutez "" +%token String "" +%token <(LexToken.lexeme * Hex.t) Region.reg> Bytes "" +%token <(string * Z.t) Region.reg> Int "" +%token <(string * Z.t) Region.reg> Nat "" +%token <(string * Z.t) Region.reg> Mutez "" +%token Ident "" +%token Constr "" (* Symbols *) diff --git a/src/passes/1-parser/reasonligo/ParserAPI.ml b/src/passes/1-parser/reasonligo/ParserAPI.ml new file mode 100644 index 000000000..7ae5c5ad4 --- /dev/null +++ b/src/passes/1-parser/reasonligo/ParserAPI.ml @@ -0,0 +1,57 @@ +(** Generic parser for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) = + struct + + module I = Parser.MenhirInterpreter + + (* The parser has successfully produced a semantic value. *) + + let success v = v + + (* The parser has suspended itself because of a syntax error. Stop. *) + + let fail _checkpoint = raise Parser.Error + + (* The generic parsing function *) + + let incr_contract Lexer.{read; buffer; close; _} : AST.t = + let supplier = I.lexer_lexbuf_to_supplier read buffer in + let parser = Parser.Incremental.contract buffer.Lexing.lex_curr_p in + let ast = I.loop_handle success fail supplier parser + in close (); ast + + let mono_contract = Parser.contract + + end diff --git a/src/passes/1-parser/reasonligo/ParserAPI.mli b/src/passes/1-parser/reasonligo/ParserAPI.mli new file mode 100644 index 000000000..ff3fe4854 --- /dev/null +++ b/src/passes/1-parser/reasonligo/ParserAPI.mli @@ -0,0 +1,39 @@ +(** Generic parser API for LIGO *) + +module type PARSER = + sig + (* The type of tokens *) + + type token + + (* This exception is raised by the monolithic API functions *) + + exception Error + + (* The monolithic API *) + + val contract : (Lexing.lexbuf -> token) -> Lexing.lexbuf -> AST.t + + (* The incremental API *) + + module MenhirInterpreter : + sig + include MenhirLib.IncrementalEngine.INCREMENTAL_ENGINE + with type token = token + end + + module Incremental : + sig + val contract : Lexing.position -> AST.t MenhirInterpreter.checkpoint + end + + end + +(* Main functor *) + +module Make (Lexer: Lexer.S) + (Parser: PARSER with type token = Lexer.Token.token) : + sig + val mono_contract : (Lexing.lexbuf -> Lexer.token) -> Lexing.lexbuf -> AST.t + val incr_contract : Lexer.instance -> AST.t + end diff --git a/src/passes/1-parser/reasonligo/ParserMain.ml b/src/passes/1-parser/reasonligo/ParserMain.ml index f4e8058cd..f855beb52 100644 --- a/src/passes/1-parser/reasonligo/ParserMain.ml +++ b/src/passes/1-parser/reasonligo/ParserMain.ml @@ -76,11 +76,11 @@ let () = (** {1 Instanciating the lexer} *) module Lexer = Lexer.Make (LexToken) - module Log = LexerLog.Make (Lexer) +module ParserFront = ParserAPI.Make (Lexer) (Parser) -let Lexer.{read; buffer; get_pos; get_last; close} = - Lexer.open_token_stream (Some pp_input) +let lexer_inst = Lexer.open_token_stream (Some pp_input) +let Lexer.{read; buffer; get_pos; get_last; close} = lexer_inst and cout = stdout @@ -97,7 +97,10 @@ let tokeniser = read ~log let () = try - let ast = Parser.contract tokeniser buffer in + (* The incremental API *) + let ast = ParserFront.incr_contract lexer_inst in + (* The monolithic API *) + (* let ast = ParserFront.mono_contract tokeniser buffer in *) if Utils.String.Set.mem "ast" options#verbose then let buffer = Buffer.create 131 in let state = ParserLog.mk_state diff --git a/src/passes/1-parser/reasonligo/dune b/src/passes/1-parser/reasonligo/dune index 6d9da9551..fefe8c10e 100644 --- a/src/passes/1-parser/reasonligo/dune +++ b/src/passes/1-parser/reasonligo/dune @@ -3,39 +3,34 @@ (menhir (merge_into Parser) (modules ParToken Parser) - (flags -la 1 --explain --dump --strict --external-tokens LexToken)) + (flags -la 1 --table --explain --strict --external-tokens LexToken)) (library (name parser_reasonligo) (public_name ligo.parser.reasonligo) (modules reasonligo LexToken Parser) (libraries + menhirLib parser_shared parser_cameligo str simple-utils tezos-utils - getopt - ) - (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo )) -) + getopt) + (flags (:standard -open Simple_utils -open Parser_shared -open Parser_cameligo))) (executable (name LexerMain) - (libraries + (libraries parser_reasonligo) - (modules - LexerMain - ) - (flags (:standard -open Parser_shared -open Parser_reasonligo)) -) + (modules + LexerMain) + (flags (:standard -open Parser_shared -open Parser_reasonligo))) (executable (name ParserMain) - (libraries + (libraries parser_reasonligo) - (modules - ParserMain - ) - (flags (:standard -open Simple_utils -open Parser_shared -open Parser_reasonligo)) -) + (modules + ParserMain) + (flags (:standard -open Simple_utils -open Parser_shared -open Parser_reasonligo)))