Merge branch 'shared_parser' into 'dev'

Separate parts of Pascaligo into a shared parser library

See merge request ligolang/ligo!62

packages/ligo/ligo.dev/opam

@@ -18,6 +18,7 @@ depends: [
   "proto-alpha-utils"
   "yojson"
   "alcotest" { with-test }
+  "getopt"
 ]
 build: [
   [ "dune" "build" "-p" name "-j" jobs ]

repo

@@ -1,3 +1,3 @@
 opam-version: "2.0"
 archive-mirrors: "cache"
-stamp: "a989886f"
+stamp: "b4649b8f"

src/ligo.opam

@@ -17,6 +17,7 @@ depends: [
   "proto-alpha-utils"
   "yojson"
   "alcotest" { with-test }
+  "getopt"
 ]
 build: [
   [ "dune" "build" "-p" name "-j" jobs ]

src/parser/camligo/dune

@@ -21,7 +21,7 @@
 (rule
   (targets parser.ml parser.mli)
   (deps parser_generated.mly ast.ml)
-  (action (system "menhir --explain --external-tokens Lex.Token lex/token.mly parser_generated.mly --base parser"))
+  (action (system "menhir --explain --unused-tokens --external-tokens Lex.Token lex/token.mly parser_generated.mly --base parser"))
 )
 
 (rule

src/parser/dune

@@ -4,6 +4,7 @@
   (libraries
     simple-utils
     tezos-utils
+    parser_shared
     parser_pascaligo
     parser_camligo
     parser_ligodity
@@ -11,5 +12,5 @@
   (preprocess
     (pps simple-utils.ppx_let_generalized)
   )
-  (flags (:standard -w +1..62-4-9-44-40-42-48-30@39@33 -open Simple_utils ))
+  (flags (:standard -w +1..62-4-9-44-40-42-48-30@39@33 -open Simple_utils -open Parser_shared ))
 )

src/parser/ligodity/dune

@@ -15,6 +15,7 @@
    zarith
    simple-utils
    tezos-utils
+   getopt
  )
  (flags (:standard -open Simple_utils ))
 )

src/parser/pascaligo/.links

@@ -4,4 +4,18 @@ $HOME/git/ligo/vendors/ligo-utils/simple-utils/pos.mli
 $HOME/git/ligo/vendors/ligo-utils/simple-utils/pos.ml
 $HOME/git/ligo/vendors/ligo-utils/simple-utils/region.mli
 $HOME/git/ligo/vendors/ligo-utils/simple-utils/region.ml
+$HOME/git/ligo/src/parser/shared/Lexer.mli
+$HOME/git/ligo/src/parser/shared/Lexer.mll
+$HOME/git/ligo/src/parser/shared/Error.mli
+$HOME/git/ligo/src/parser/shared/EvalOpt.ml
+$HOME/git/ligo/src/parser/shared/EvalOpt.mli
+$HOME/git/ligo/src/parser/shared/FQueue.ml
+$HOME/git/ligo/src/parser/shared/FQueue.mli
+$HOME/git/ligo/src/parser/shared/LexerLog.mli
+$HOME/git/ligo/src/parser/shared/LexerLog.ml
+$HOME/git/ligo/src/parser/shared/Markup.ml
+$HOME/git/ligo/src/parser/shared/Markup.mli
+$HOME/git/ligo/src/parser/shared/Utils.mli
+$HOME/git/ligo/src/parser/shared/Utils.ml
+$HOME/git/ligo/src/parser/shared/Version.ml
 Stubs/Simple_utils.ml

src/parser/pascaligo/Doc/pascaligo.txt

@@ -11,10 +11,6 @@ The directory contains the following:
   Tests
     The directory containing tests.
 
-  Version.ml
-    A source containing a commit hash. It should be deleted, as Dune
-    knows how to generate and updated version.
-
   dune
     The Dune file for building the Pascaligo parser.
 
@@ -46,11 +42,6 @@ The directory contains the following:
     build only a standalone lexer or a standalone parser. Do not
     change, unless you change EvalOpt and use Christian's build system.
 
-  LexerLog.ml
-  LexerLog.mli
-    Source for instantiating a standalone lexer for LexerMain.ml and
-    ParserMain.ml. Ignore them.
-
   ParserLog.mli
   ParserLog.ml
     Source for printing the AST. Used by ParserMain.ml, pascaligo.ml
@@ -65,51 +56,6 @@ The directory contains the following:
   AST.ml
     The abstract syntax tree of Pascaligo.
 
-  EvalOpt.mli
-  EvalOpt.ml
-    The module EvalOpt parses the command-line for options to the
-    parser. That action is performed as a side-effect when the module
-    is initialised at run-time: this is ugly and easy to fix. See
-    ligo/src/parser/ligodity/EvalOpt.ml{i} for the right way to do
-    it. Ignore them: the file actually calling directly the parser is
-    ligo/src/parser/parser.ml. Note that, as a consequence, no option
-    is currently passed to the parser when building Pascaligo with
-    Dune. This should be made available.
-
-  Markup.mli
-  Markup.ml
-    The definition of markup in Pascaligo source files, and some some
-    functions to print or convert it to strings. You are unlikely
-    going to modify those files, as markup is pretty much the same for
-    all LIGO flavours.
-
-  FQueue.mli
-  FQueue.ml
-    A naive implementation of purely functional queues. Replace by an
-    imperative implementation if worst-case performance of single
-    operations (queue/enqueue) is an issue.
-
-  Error.mli
-    The definition of the open type for errors: the lexer will add its
-    own errors, the downside being that matching on errors requires a
-    catch-all clause "| _ -> assert false" at the end. Note: the rest
-    of the compiler uses an error monad.
-
-  Lexer.mli
-  Lexer.mll
-    The Pascaligo lexer is generated from two ocamllex
-    specifications. Lexer.mll is the first-level lexer. It exports a
-    functor [Make] parameterised over a module [Token] defining the
-    tokens, and returning a module whose signature is [Lexer.S]. (See
-    Lexer.mli for a rationale.) If you write a new flavour of LIGO,
-    this lexer is likely to be reused as is. Note that a great deal of
-    the complexity of this lexer stems from its purpose to report
-    stylistic errors (hence keeping temporarily scanned markup) and
-    handling UTF-8 encoded comments. The first goal implies sometimes
-    reading more than one token, and an extra-buffer has to be managed
-    above the ocamllex one, so the parser is not confused about the
-    location (region) of the token it has just read.
-
   LexToken.mli
   LexToken.mll
     The second-level lexer of Pascaligo, scanning the (lexical)

src/parser/pascaligo/dune

@@ -1,5 +1,4 @@
 (ocamllex LexToken)
-(ocamllex Lexer)
 
 (menhir
  (merge_into Parser)
@@ -9,17 +8,27 @@
 (library
  (name parser_pascaligo)
  (public_name ligo.parser.pascaligo)
- (modules AST FQueue Markup pascaligo Utils Version Lexer Error Parser ParserLog LexToken)
- (modules_without_implementation Error)
+ (modules AST pascaligo Parser ParserLog LexToken)
  (libraries
-   hex
-   str
-   uutf
-   zarith
+   parser_shared
+   hex   
    simple-utils
    tezos-utils
  )
- (flags (:standard -open Simple_utils ))
+ (flags (:standard  -open Parser_shared -open Simple_utils))
+)
+
+(executable
+  (name LexerMain)
+  (libraries 
+   hex
+   simple-utils
+   tezos-utils 
+   parser_pascaligo)
+  (modules 
+    LexerMain
+  )
+  (flags (:standard -open Parser_shared -open Parser_pascaligo))
 )
 
 ;; Les deux directives (rule) qui suivent sont pour le dev local.
@@ -36,9 +45,3 @@
 ;  (deps LexerMain.exe)
 ;  (action (copy LexerMain.exe Lexer.exe))
 ;  (mode promote-until-clean))
-
-(rule
-  (targets Version.ml)
-  (action
-    (progn (run "sh" "-c" "printf 'let version = \"%s\"'\\\\n \"$(echo UNKNOWN)\" > Version.ml")))
-  (mode promote-until-clean))

src/parser/shared/.links

@@ -0,0 +1,7 @@
+$HOME/git/OCaml-build/Makefile
+$HOME/git/OCaml-build/Makefile.cfg
+$HOME/git/ligo/vendors/ligo-utils/simple-utils/pos.mli
+$HOME/git/ligo/vendors/ligo-utils/simple-utils/pos.ml
+$HOME/git/ligo/vendors/ligo-utils/simple-utils/region.mli
+$HOME/git/ligo/vendors/ligo-utils/simple-utils/region.ml
+$HOME/git/ligo/vendors/ligo-utils/simple-utils/region.ml

src/parser/shared/Doc/shared.txt

@@ -0,0 +1,55 @@
+INTERNAL DOCUMENTATION OF THE SHARED PARSER FUNCTIONALITY
+
+  Version.ml
+    A source containing a commit hash. It should be deleted, as Dune
+    knows how to generate and updated version.
+  
+  EvalOpt.mli
+  EvalOpt.ml
+    The module EvalOpt parses the command-line for options to the
+    parser. That action is performed as a side-effect when the module
+    is initialised at run-time: this is ugly and easy to fix. See
+    ligo/src/parser/ligodity/EvalOpt.ml{i} for the right way to do
+    it. Ignore them: the file actually calling directly the parser is
+    ligo/src/parser/parser.ml. Note that, as a consequence, no option
+    is currently passed to the parser when building Pascaligo with
+    Dune. This should be made available.
+
+  Markup.mli
+  Markup.ml
+    The definition of markup in source files, and some functions to 
+    print or convert it to strings. You are unlikely going to modify 
+    those files, as markup is pretty much the same for all LIGO 
+    flavours.
+
+  FQueue.mli
+  FQueue.ml
+    A naive implementation of purely functional queues. Replace by an
+    imperative implementation if worst-case performance of single
+    operations (queue/enqueue) is an issue.
+
+  Error.mli
+    The definition of the open type for errors: the lexer will add its
+    own errors, the downside being that matching on errors requires a
+    catch-all clause "| _ -> assert false" at the end. Note: the rest
+    of the compiler uses an error monad.
+
+  Lexer.mli
+  Lexer.mll
+    The Pascaligo lexer is generated from two ocamllex
+    specifications. Lexer.mll is the first-level lexer. It exports a
+    functor [Make] parameterised over a module [Token] defining the
+    tokens, and returning a module whose signature is [Lexer.S]. (See
+    Lexer.mli for a rationale.) If you write a new flavour of LIGO,
+    this lexer is likely to be reused as is. Note that a great deal of
+    the complexity of this lexer stems from its purpose to report
+    stylistic errors (hence keeping temporarily scanned markup) and
+    handling UTF-8 encoded comments. The first goal implies sometimes
+    reading more than one token, and an extra-buffer has to be managed
+    above the ocamllex one, so the parser is not confused about the
+    location (region) of the token it has just read.
+
+  LexerLog.ml
+  LexerLog.mli
+    Source for instantiating a standalone lexer for LexerMain.ml and
+    ParserMain.ml. Ignore them.

src/parser/shared/dune

@@ -0,0 +1,28 @@
+(ocamllex Lexer)
+
+(library
+  (name parser_shared)
+  (public_name ligo.parser.shared)
+  (libraries
+    simple-utils
+    uutf
+    getopt
+  )
+  (modules
+    Error
+    Lexer
+    LexerLog
+    Utils
+    Markup
+    FQueue
+    EvalOpt
+    Version
+  )
+  (modules_without_implementation Error)
+)
+
+(rule
+  (targets Version.ml)
+  (action
+    (progn (run "sh" "-c" "printf 'let version = \"%s\"'\\\\n \"$(echo UNKNOWN)\" > Version.ml")))
+  (mode promote-until-clean))

urls.txt

@@ -1 +1 @@
-repo	604d7d8c5eb209596f929225538c2c3c	420
+repo	f9ec38c6d4dfb4ef9f64edb361326b32	420