Merge branch 'rinderknecht@fix_endif' into 'dev'

[Preprocessor] Missing scanning rules for eof and newline and changing the semantics of conditionals

See merge request ligolang/ligo!621

vendors/Preprocessor/Preproc.mli

@@ -33,6 +33,7 @@ type error =
 | File_not_found of string
 | Invalid_character of char
 | Unterminated_comment of string
+| Unterminated_inclusion
 
 val format :
   ?offsets:bool -> error Region.reg -> file:bool -> string Region.reg

vendors/Preprocessor/Preproc.mll

@@ -119,6 +119,7 @@ type error =
 | File_not_found of string
 | Invalid_character of char
 | Unterminated_comment of string
+| Unterminated_inclusion
 
 let error_to_string = function
   Directive_inside_line ->
@@ -180,6 +181,9 @@ let error_to_string = function
 | Unterminated_comment ending ->
     sprintf "Unterminated comment.\n\
              Hint: Close with \"%s\"." ending
+| Unterminated_inclusion ->
+    sprintf "Unterminated #include directive.\n\
+             Hint: Add as a string the name of the file to be included."
 
 let format ?(offsets=true) Region.{region; value} ~file =
   let msg   = error_to_string value
@@ -541,16 +545,22 @@ rule scan state = parse
         scan (reduce_cond state region) lexbuf
     | "define" ->
         let id, region = variable state lexbuf in
+        if state.mode = Copy then
           if id="true" || id="false"
-        then stop (Reserved_symbol id) state region;
+          then stop (Reserved_symbol id) state region
+          else
             if Env.mem id state.env
-        then stop (Multiply_defined_symbol id) state region;
+            then stop (Multiply_defined_symbol id) state region
+            else
               let state = {state with env = Env.add id state.env}
               in scan state lexbuf
+        else scan state lexbuf
     | "undef" ->
         let id, _ = variable state lexbuf in
+        if state.mode = Copy then
           let state = {state with env = Env.remove id state.env}
           in scan state lexbuf
+        else scan state lexbuf
     | "error" ->
         stop (Error_directive (message [] lexbuf)) state region
     | "region" ->
@@ -635,20 +645,22 @@ and symbol state = parse
 
 and skip_line state = parse
   nl     { proc_nl state lexbuf   }
-| blank+ { skip_line state lexbuf }
+| eof    { rollback lexbuf        }
-| _      { ()                     }
+| blank+
+| _      { skip_line state lexbuf }
 
 and message acc = parse
   nl     { Lexing.new_line lexbuf;
            mk_str (List.length acc) acc }
-| eof    { mk_str (List.length acc) acc }
+| eof    { rollback lexbuf;
+           mk_str (List.length acc) acc }
 | _ as c { message (c::acc) lexbuf      }
 
 (* Comments *)
 
 and in_line_com state = parse
   nl  { proc_nl state lexbuf; state                  }
-| eof { state                                        }
+| eof { rollback lexbuf; state                       }
 | _   { if state.mode = Copy then copy state lexbuf;
         in_line_com state lexbuf                     }
 
@@ -688,11 +700,11 @@ and in_block block opening state = parse
 and scan_inclusion state = parse
   blank+   { scan_inclusion state lexbuf                    }
 | '"'      { in_inclusion (mk_reg lexbuf) [] 0 state lexbuf }
+| nl | eof { fail Unterminated_inclusion state lexbuf       }
 
 and in_inclusion opening acc len state = parse
   '"'    { let closing = mk_reg lexbuf
-           in Region.cover opening closing,
+           in Region.cover opening closing, mk_str len acc    }
-              mk_str len acc                                  }
 | nl     { fail Newline_in_string state lexbuf                }
 | eof    { stop Unterminated_string state opening             }
 | _ as c { in_inclusion opening (c::acc) (len+1) state lexbuf }
@@ -702,7 +714,7 @@ and in_inclusion opening acc len state = parse
 and in_string opening state = parse
   "\\\"" { copy state lexbuf; in_string opening state lexbuf }
 | '"'    { copy state lexbuf; state                          }
-| eof    { state                                             }
+| eof    { rollback lexbuf; state                            }
 | _      { copy state lexbuf; in_string opening state lexbuf }
 
 and preproc state = parse

vendors/Preprocessor/PreprocMain.ml

@@ -4,7 +4,7 @@ module Region  = Simple_utils.Region
 module Preproc = Preprocessor.Preproc
 module EvalOpt = Preprocessor.EvalOpt
 
-let highlight msg = Printf.eprintf "\027[31m%s\027[0m%!" msg
+let highlight msg = Printf.eprintf "\027[31m%s\027[0m\n%!" msg
 
 let options =
   let open EvalOpt in