Switch to Jbuilder

This is a rewrite of the build system with `jbuilder`, with just a
minimal toplevel Makefile for backward compatibility.

This first patch preserves the project architecture, we only gain
proper dependencies handling and always up-to-date `.merlin` files.
A latter patch may split the project in smaller "sub-package",
i.e. multiple `.opam` files.

The embedded versions of the economic protocol are now compiled with
`jbuilder` instead of `tezos-protocol-compiler`, potentially allowing
proper inlining at the cost of slightly-less-stricter
sandboxing. Nevertheless, dynamically loaded protocol are still
compiled with the `tezos-protocol-compiler` and thus strictly
sandboxed ; and a CI rule also checks the proper sandboxing of
embedded protocols.

This patch is coauthored with @hnrgrgr

.dockerignore

@@ -1,51 +1,22 @@
-.git
-.gitignore
-.gitlab-ci.yml
 
-**/.depend
+_build
+tezos.install
 
 tezos-node
 tezos-protocol-compiler
 tezos-client
-tezos-attacker
 
-src/Makefile.local
+**/.merlin
-
-src/node/updater/environment_gen
-src/node/updater/proto_environment.mli
-src/compiler/embedded_cmis.ml
-
-src/proto/**/_tzbuild
-src/proto/register_client_*.ml
-src/client/embedded/**/_tzbuild
-
-src/client/embedded/alpha/concrete_lexer.ml
-src/client/embedded/alpha/concrete_parser.ml
-src/client/embedded/alpha/concrete_parser.mli
-
-test/reports
-test/**/test-*
-test/LOG.*
 
 **/*~
 **/\#*\#
 
-**/*.[oa]
-**/*.so
-**/*~
-**/*.cm[iaoxt]
-**/*.cmti
-**/*.cmxa
-**/*.cmxs
-**/*.cmp
-**/*.mli.deps
-**/*.ml.deps
-**/*.mli.deps.byte
-**/*.ml.deps.byte
-
-**/bisect*.out
-
 **/*.rej
 **/*.orig
 
-_opam
+_opam
+
+.git
+.gitignore
+.gitlab-ci.yml
+

.gitignore

@@ -1,51 +1,18 @@
 
+/_build
+/tezos.install
+
 /tezos-node
-/tezos-protocol-packer
 /tezos-protocol-compiler
 /tezos-client
-/tezos-attacker
 
-/src/.ocamlinit.utils
+.merlin
-/src/.ocamlinit.node
-/src/Makefile.local
-
-/src/environment/tezos_protocol_environment_sigs_v1.ml
-
-/src/compiler/tezos_compiler_embedded_cmis.ml
-
-/src/proto/**/_tzbuild
-/src/proto/register_client_*.ml
-/src/client/embedded/**/_tzbuild
-
-/src/client/embedded/alpha/concrete_lexer.ml
-/src/client/embedded/alpha/concrete_parser.ml
-/src/client/embedded/alpha/concrete_parser.mli
-
-/test/reports
-/test/*/test-*
-/test/LOG.*
-
-.depend
 
 *~
 \#*\#
 
-*.[oa]
-*.so
-*~
-*.cm[iaoxt]
-*.cmti
-*.cmxa
-*.cmxs
-*.cmp
-*.mli.deps
-*.ml.deps
-*.mli.deps.byte
-*.ml.deps.byte
-
-bisect*.out
-
 *.rej
 *.orig
 
-/_opam
+/_opam
+/Dockerfile

.gitlab-ci.yml

@@ -72,88 +72,78 @@ build:
   dependencies:
     - build
 
-test:utils:data-encoding:
+test:utils:
   <<: *test_definition
   script:
-    - make -C test/utils run-test-data-encoding
+    - jbuilder build @test/utils/runtest
 
-test:utils:merkle:
+test:shell:
   <<: *test_definition
   script:
-    - make -C test/utils run-test-merkle
+    - jbuilder build @test/shell/runtest
-
-test:shell:store:
-  <<: *test_definition
-  script:
-    - make -C test/shell run-test-store
-
-test:shell:context:
-  <<: *test_definition
-  script:
-    - make -C test/shell run-test-context
-
-test:shell:state:
-  <<: *test_definition
-  script:
-    - make -C test/shell run-test-state
 
 test:p2p:io-scheduler:
   <<: *test_definition
   script:
-    - make -C test/p2p run-test-p2p-io-scheduler
+    - jbuilder build @test/p2p/runtest_p2p_io_scheduler
 
 test:p2p:connection:
   <<: *test_definition
   script:
-    - make -C test/p2p run-test-p2p-connection
+    - jbuilder build @test/p2p/runtest_p2p_connection
 
 test:p2p:connection-pool:
   <<: *test_definition
   script:
-    - make -C test/p2p run-test-p2p-connection-pool
+    - jbuilder build @test/p2p/runtest_p2p_connection_pool
 
-test:proto_alpha:transactions:
+test:proto_alpha:transaction:
   <<: *test_definition
   script:
-    - make -C test/proto_alpha run-test-transaction
+    - jbuilder build @test/proto_alpha/runtest_transaction
   retry: 1
 
 test:proto_alpha:origination:
   <<: *test_definition
   script:
-    - make -C test/proto_alpha run-test-origination
+    - jbuilder build @test/proto_alpha/runtest_origination
   retry: 1
 
 test:proto_alpha:endorsement:
   <<: *test_definition
   script:
-    - make -C test/proto_alpha run-test-endorsement
+    - jbuilder build @test/proto_alpha/runtest_endorsement
   retry: 1
 
 test:proto_alpha:vote:
   <<: *test_definition
   script:
-    - make -C test/proto_alpha run-test-vote
+    - jbuilder build @test/proto_alpha/runtest_vote
   retry: 1
 
 test:basic.sh:
   <<: *test_definition
   script:
-    - make -C test run-basic.sh
+    - jbuilder build @test/runtest_basic.sh
   retry: 1
 
 test:contracts.sh:
   <<: *test_definition
   script:
-    - make -C test run-contracts.sh
+    - jbuilder build @test/runtest_contracts.sh
   retry: 1
 
 test:multinode.sh:
   <<: *test_definition
   script:
-    - make -C test run-multinode.sh
+    - jbuilder build @test/runtest_multinode.sh
   retry: 1
 
+test:proto:sandboxing:
+  <<: *test_definition
+  script:
+    - jbuilder build @src/proto/runtest_sandboxing
+
 ## Publishing (small) docker images with tezos binaries
 
 publish:docker:minimal:

Makefile

@@ -0,0 +1,24 @@
+
+all:
+	@jbuilder build tezos.install
+	@cp _build/default/src/node_main.exe tezos-node
+	@cp _build/default/src/client_main.exe tezos-client
+	@cp _build/default/src/compiler_main.exe tezos-protocol-compiler
+
+build-test:
+	@jbuilder build @buildtest
+
+test:
+	@jbuilder runtest
+
+build-deps:
+	@./scripts/install_build_deps.sh
+
+docker-image:
+	@./scripts/create_docker_image.sh
+
+clean:
+	@-jbuilder clean
+	@-rm -f tezos-node tezos-client tezos-protocol-compiler
+
+.PHONY: all test build-deps docker-image clean

jbuild

@@ -0,0 +1 @@
+(jbuild_version 1)

scripts/Dockerfile.build.in

@@ -1,10 +1,11 @@
 FROM $base_image
 
-COPY src /home/opam/tezos/src
+COPY src tezos/src
-COPY test /home/opam/tezos/test
+COPY test tezos/test
-COPY scripts /home/opam/tezos/scripts
+COPY scripts tezos/scripts
+COPY Makefile jbuild tezos.opam tezos/
 
 RUN sudo chown -R opam /home/opam/tezos && \
-    echo "PRODUCTION=yes" > /home/opam/tezos/src/Makefile.local && \
+    cd tezos && \
-    opam config exec -- make -C tezos/src -j4 && \
+    opam config exec -- make all && \
-    opam config exec -- make -C tezos/test -j4 build
+    opam config exec -- make build-test

scripts/Dockerfile.build_deps.in

@@ -1,9 +1,9 @@
 FROM $base_image
 
-COPY scripts/install_build_deps.sh scripts/version.sh scripts/
+COPY scripts/install_build_deps.sh scripts/version.sh tezos/scripts/
-COPY src/tezos-deps.opam src/
+COPY tezos.opam tezos/
 RUN sudo apk update && \
-    opam config exec -- ./scripts/install_build_deps.sh all && \
+    opam config exec -- ./tezos/scripts/install_build_deps.sh && \
     sudo rm -f /var/cache/apk/* && \
     rm -fr ~/.opam/log/ && \
     rm -fr "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi

scripts/client_lib.inc.sh

@@ -13,7 +13,8 @@ init_sandboxed_client() {
     rpc=$((18730 + id))
     client_dir="$(mktemp -d -t tezos-client.XXXXXXXX)"
     client_dirs+=("$client_dir")
-    client="$src_dir/tezos-client -base-dir $client_dir -addr 127.0.0.1 -port $rpc"
+    local_client=${local_client:-$src_dir/_build/default/src/client_main.exe}
+    client="$local_client -base-dir $client_dir -addr 127.0.0.1 -port $rpc"
 
 }
 

scripts/create_docker_image.build_deps.sh

@@ -28,7 +28,7 @@ sed -e 's|$base_image|'"$base_image"'|g' \
     scripts/Dockerfile.build_deps.in > Dockerfile
 
 ## Lookup for for prebuilt dependencies...
-dependencies="scripts/install_build_deps.sh src/tezos-deps.opam Dockerfile"
+dependencies="scripts/install_build_deps.sh tezos.opam Dockerfile"
 dependencies_sha1=$(docker inspect --format="{{ .RootFS.Layers }}" --type=image $base_image | sha1sum - $dependencies | sha1sum | tr -d ' -')
 if [ ! -z "$cached_image" ]; then
     echo

scripts/install_build_deps.sh

@@ -13,52 +13,19 @@ if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then
   exit 1;
 fi
 
-cmd="$1"
-if [ -z "$cmd" ]; then cmd=all; fi
-
-pin=false
-depext=false
-install=false
-
-case $cmd in
-    pin)
-	pin=true
-	;;
-    depext)
-	depext=true
-	;;
-    install)
-	install=true
-	;;
-    all)
-	pin=true
-	depext=true
-	install=true
-	;;
-    *)
-        echo "Unknown command '$cmd'."
-        echo "Usage: $0 [pin|depext|install|all|]"
-        exit 1
-esac
-
 set -e
 set -x
 
-if "$pin"; then
+opam pin --yes add --no-action --dev-repo sodium
-    opam pin --yes add --no-action --dev-repo sodium
+opam pin --yes add --no-action --dev-repo ocp-ocamlres
-    opam pin --yes add --no-action --dev-repo ocp-ocamlres
+opam pin --yes add --no-action --dev-repo ocplib-json-typed
-    opam pin --yes add --no-action --dev-repo ocplib-json-typed
+opam pin --yes add --no-action --dev-repo ocplib-resto
-    opam pin --yes add --no-action --dev-repo ocplib-resto
+opam pin --yes add --no-action --dev-repo jbuilder
-    ## Force opam to take account of the new `tezos-deps.opam`
+## Force opam to take account of the new `tezos-deps.opam`
-    opam pin --yes remove tezos-deps
+opam pin --yes remove tezos
-    opam pin --yes add --no-action tezos-deps $src_dir/src
+opam pin --yes add --no-action tezos $src_dir
-fi
 
-if "$depext"; then
+opam list --installed depext || opam install depext
-    opam list --installed depext || opam install depext
+opam depext tezos
-    opam depext $DEPEXTOPT tezos-deps
-fi
 
-if "$install"; then
+opam install tezos --deps-only
-    opam install tezos-deps
-fi

scripts/node_lib.inc.sh

@@ -20,7 +20,8 @@ start_sandboxed_node() {
         peers+=("127.0.0.1:$peer_port")
     done
     peers+=("--closed")
-    node="$src_dir/tezos-node"
+    local_node="${local_node:-$src_dir/_build/default/src/node_main.exe}"
+    node="$local_node"
     sandbox_file="${sandbox_file:-$script_dir/sandbox.json}"
     sandbox_param="--sandbox=$sandbox_file"
 

src/client/embedded/alpha/client_proto_alpha.ml

@@ -1,16 +0,0 @@
-(**************************************************************************)
-(*                                                                        *)
-(*    Copyright (c) 2014 - 2016.                                          *)
-(*    Dynamic Ledger Solutions, Inc. <contact@tezos.com>                  *)
-(*                                                                        *)
-(*    All rights reserved. No warranty, explicit or implicit, provided.   *)
-(*                                                                        *)
-(**************************************************************************)
-
-module Raw = Tezos_embedded_protocol_alpha
-
-module Environment = Tezos_protocol_environment.Make(Raw.Register.Name)()
-module P = Raw.Functor.Make(Environment)
-
-include P
-include Updater.LiftProtocol(Raw.Register.Name)(Environment)(P)

src/client/embedded/alpha/jbuild

@@ -0,0 +1,16 @@
+(jbuild_version 1)
+
+(library
+ ((name client_embedded_alpha)
+  (libraries (tezos_embedded_protocol_alpha
+              tezos_embedded_raw_protocol_alpha
+              client_lib))
+  (library_flags (:standard -linkall))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data
+                    -open Tezos_protocol_environment_alpha
+                    -open Tezos_embedded_raw_protocol_alpha
+                    -open Tezos_context))))

src/client/embedded/demo/client_proto_demo.ml

@@ -1,16 +0,0 @@
-(**************************************************************************)
-(*                                                                        *)
-(*    Copyright (c) 2014 - 2016.                                          *)
-(*    Dynamic Ledger Solutions, Inc. <contact@tezos.com>                  *)
-(*                                                                        *)
-(*    All rights reserved. No warranty, explicit or implicit, provided.   *)
-(*                                                                        *)
-(**************************************************************************)
-
-module Raw = Tezos_embedded_protocol_demo
-
-module Environment = Tezos_protocol_environment.Make(Raw.Register.Name)()
-module P = Raw.Functor.Make(Environment)
-
-include P
-include Updater.LiftProtocol(Raw.Register.Name)(Environment)(P)

src/client/embedded/genesis/client_proto_genesis.ml

@@ -1,16 +0,0 @@
-(**************************************************************************)
-(*                                                                        *)
-(*    Copyright (c) 2014 - 2016.                                          *)
-(*    Dynamic Ledger Solutions, Inc. <contact@tezos.com>                  *)
-(*                                                                        *)
-(*    All rights reserved. No warranty, explicit or implicit, provided.   *)
-(*                                                                        *)
-(**************************************************************************)
-
-module Raw = Tezos_embedded_protocol_genesis
-
-module Environment = Tezos_protocol_environment.Make(Raw.Register.Name)()
-module P = Raw.Functor.Make(Environment)
-
-include P
-include Updater.LiftProtocol(Raw.Register.Name)(Environment)(P)

src/client/embedded/genesis/client_proto_main.ml

@@ -8,6 +8,7 @@
 (**************************************************************************)
 
 open Client_commands
+open Tezos_embedded_raw_protocol_genesis
 
 let protocol =
   Protocol_hash.of_b58check_exn
@@ -73,7 +74,7 @@ let commands () =
        @@ stop)
       begin fun timestamp hash fitness seckey cctxt ->
         let fitness =
-          Client_embedded_alpha.Client_proto_alpha.Fitness_repr.from_int64 fitness in
+          Tezos_embedded_raw_protocol_alpha.Fitness_repr.from_int64 fitness in
         mine cctxt.rpc_config ?timestamp cctxt.config.block
           (Activate hash) fitness seckey >>=? fun hash ->
         cctxt.answer "Injected %a" Block_hash.pp_short hash >>= fun () ->
@@ -96,7 +97,7 @@ let commands () =
        @@ stop)
       begin fun timestamp hash fitness seckey cctxt ->
         let fitness =
-          Client_embedded_alpha.Client_proto_alpha.Fitness_repr.from_int64 fitness in
+          Tezos_embedded_raw_protocol_alpha.Fitness_repr.from_int64 fitness in
         mine cctxt.rpc_config ?timestamp cctxt.config.block
           (Activate_testnet (hash, Int64.mul 24L 3600L))
           fitness seckey >>=? fun hash ->

src/client/embedded/genesis/client_proto_main.mli

@@ -7,7 +7,7 @@
 (*                                                                        *)
 (**************************************************************************)
 
-open Client_proto_genesis
+open Tezos_embedded_raw_protocol_genesis
 
 val mine:
   Client_rpcs.config ->

src/client/embedded/genesis/jbuild

@@ -0,0 +1,16 @@
+(jbuild_version 1)
+
+(library
+ ((name client_embedded_genesis)
+  (libraries (tezos_embedded_raw_protocol_genesis
+              tezos_embedded_protocol_genesis
+              tezos_protocol_environment_alpha
+              client_lib))
+  (library_flags (:standard -linkall))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data
+                    -open Tezos_protocol_environment_genesis
+                    -open Tezos_embedded_raw_protocol_genesis))))

src/client/embedded/jbuild

@@ -0,0 +1 @@
+(jbuild_version 1)

src/client/jbuild

@@ -0,0 +1,17 @@
+(jbuild_version 1)
+
+(library
+ ((name client_lib)
+  (libraries (minutils
+              utils
+              cohttp
+              node_shell
+              node_db
+              node_updater
+              tezos_protocol_compiler))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/compiler/jbuild

@@ -0,0 +1,28 @@
+(jbuild_version 1)
+
+(rule
+ ((targets (embedded_cmis.ml))
+  (action
+    (run ${bin:ocp-ocamlres} -format ocaml -o ${@}
+           ${lib:stdlib:camlinternalFormatBasics.cmi}
+           ${path:registerer.cmi}
+           ${path:../environment/tezos_protocol_environment_sigs_v1.cmi}))))
+
+(library
+ ((name tezos_protocol_compiler)
+  (libraries (utils
+              minutils
+              tezos_protocol_environment_sigs
+              compiler-libs
+              compiler-libs.optcomp
+              lwt.unix
+              ocplib-endian
+              ocplib-ocamlres
+              unix))
+  (wrapped false)
+  (flags (:standard -w +27-30-40@8
+                    -opaque
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))))

src/environment/jbuild

@@ -0,0 +1,55 @@
+(jbuild_version 1)
+
+(rule
+ ((targets (tezos_protocol_environment_sigs_v1.ml))
+  (deps
+   (
+    ;; Part of OCaml stdlib
+    v1/pervasives.mli
+    v1/array.mli
+    v1/list.mli
+    v1/bytes.mli
+    v1/string.mli
+    v1/set.mli
+    v1/map.mli
+    v1/int32.mli
+    v1/int64.mli
+    v1/buffer.mli
+    v1/format.mli
+
+    ;; Part of external libraries
+    v1/z.mli
+    v1/lwt_sequence.mli
+    v1/lwt.mli
+    v1/lwt_list.mli
+    v1/uri.mli
+
+    ;; Tezos extended stdlib
+    v1/mBytes.mli
+    v1/hex_encode.mli
+    v1/compare.mli
+    v1/data_encoding.mli
+    v1/error_monad.mli
+    v1/logging.mli
+    v1/time.mli
+    v1/base58.mli
+    v1/hash.mli
+    v1/ed25519.mli
+
+    ;; Tezos specifics
+    v1/tezos_data.mli
+    v1/persist.mli
+    v1/context.mli
+    v1/RPC.mli
+    v1/updater.mli
+
+  ))
+  (action
+    (with-stdout-to ${@}
+      (run ${exe:sigs_packer/sigs_packer.exe} ${!^})))))
+
+(library
+ ((name tezos_protocol_environment_sigs)
+  (flags (:standard -nopervasives))
+  (wrapped false)
+  (modules ("Tezos_protocol_environment_sigs_v1"))))

src/environment/sigs_packer/jbuild

@@ -0,0 +1,6 @@
+(jbuild_version 1)
+
+(executable
+  ((name sigs_packer)
+  (public_name tezos-protocol-environment-sigs-packer)))
+

src/jbuild

@@ -0,0 +1,28 @@
+(jbuild_version 1)
+
+(executable
+ ((name compiler_main)
+  (public_name tezos-protocol-compiler)
+  (libraries (tezos_protocol_compiler))
+  (flags (:standard -w +27-30-40@8
+                    -linkall))
+  (modules (Compiler_main))))
+
+(executable
+ ((name node_main)
+  (public_name tezos-node)
+  (libraries (node_db node_main_lib node_net cmdliner
+              tezos_embedded_protocol_genesis
+              tezos_embedded_protocol_demo
+              tezos_embedded_protocol_alpha))
+  (flags (:standard -w +27-30-40@8
+                    -linkall))
+  (modules (Node_main))))
+
+(executable
+ ((name client_main)
+  (public_name tezos-client)
+  (libraries (lwt utils client_lib client_embedded_genesis client_embedded_alpha))
+  (flags (:standard -w +27-30-40@8
+                    -linkall))
+  (modules (Client_main))))

src/minutils/jbuild

@@ -0,0 +1,10 @@
+(jbuild_version 1)
+
+(library
+    ((name minutils)
+     (libraries (cstruct
+                 lwt
+                 ocplib-json-typed.bson
+                 ocplib-resto.directory))
+     (flags (:standard -w +27-30-40@8))
+     (wrapped false)))

src/node/db/jbuild

@@ -0,0 +1,11 @@
+(jbuild_version 1)
+
+(library
+ ((name node_db)
+  (libraries (utils minutils leveldb irmin irmin-unix))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/node/jbuild

@@ -0,0 +1 @@
+(jbuild_version 1)

src/node/main/jbuild

@@ -0,0 +1,11 @@
+(jbuild_version 1)
+
+(library
+ ((name node_main_lib)
+  (libraries (utils minutils cmdliner node_net node_shell))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/node/net/jbuild

@@ -0,0 +1,11 @@
+(jbuild_version 1)
+
+(library
+ ((name node_net)
+  (libraries (utils minutils conduit-lwt-unix cohttp cohttp-lwt-unix))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/node/shell/jbuild

@@ -0,0 +1,11 @@
+(jbuild_version 1)
+
+(library
+ ((name node_shell)
+  (libraries (utils minutils node_net node_db node_updater ezjsonm ocplib-json-typed.bson))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/node/updater/jbuild

@@ -0,0 +1,11 @@
+(jbuild_version 1)
+
+(library
+ ((name node_updater)
+  (libraries (utils minutils tezos_protocol_compiler node_db dynlink))
+  (flags (:standard -w +27-30-40@8
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))
+  (wrapped false)))

src/proto/alpha/jbuild

@@ -0,0 +1,57 @@
+(jbuild_version 1)
+
+(rule
+ ((targets (environment.ml))
+  (action
+    (write-file ${@}
+      "include Tezos_protocol_environment.Make(struct let name = \"alpha\" end)()"))))
+
+(rule
+ ((targets (registerer.ml))
+  (action
+    (write-file ${@}
+       "
+let () =
+  let module Ignored = State.Register_embedded_protocol
+    (Tezos_protocol_environment_alpha.Environment)
+    (Tezos_embedded_raw_protocol_alpha.Main)
+    (struct
+       let hash =
+         Some (Hash.Protocol_hash.of_b58check_exn
+                 \"ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK\")
+       let sources = Tezos_data.Protocol.{
+         expected_env = V1 ;
+         components = [(* FIXME ?? *)] ;
+    } end) in ()"))))
+
+(library
+  ((name tezos_protocol_environment_alpha)
+   (library_flags (:standard -linkall))
+   (libraries (node_updater))
+   (modules (Environment))))
+
+(library
+  ((name tezos_embedded_raw_protocol_alpha)
+   (libraries (tezos_protocol_environment_alpha))
+   (library_flags (:standard -linkall))
+   (flags (:standard -nopervasives -nostdlib
+                     -w +a-4-6-7-9-29-40..42-44-45-48
+                     -warn-error -a+8
+                     -open Tezos_protocol_environment_alpha__Environment
+                     -open Error_monad
+                     -open Hash
+                     -open Tezos_data))
+   (modules (:standard \ Environment Registerer))))
+
+(library
+  ((name tezos_embedded_protocol_alpha)
+   (library_flags (:standard -linkall))
+   (libraries (tezos_embedded_raw_protocol_alpha node_shell))
+   (modules (Registerer))))
+
+(alias
+ ((name runtest_sandboxing)
+  (deps ((glob_files *.ml)
+         (glob_files *.mli)
+         TEZOS_PROTOCOL))
+  (action (run ${exe:../../compiler_main.exe} dummy_alpha ${path-no-dep:.}))))

src/proto/demo/jbuild

@@ -0,0 +1,55 @@
+(jbuild_version 1)
+
+(rule
+ ((targets (environment.ml))
+  (action
+    (write-file ${@}
+      "include Tezos_protocol_environment.Make(struct let name = \"demo\" end)()"))))
+
+(rule
+ ((targets (registerer.ml))
+  (action
+    (write-file ${@}
+       "module Ignored =
+  State.Register_embedded_protocol
+    (Tezos_protocol_environment_demo.Environment)
+    (Tezos_embedded_raw_protocol_demo.Main)
+    (struct
+       let hash =
+         Some (Hash.Protocol_hash.of_b58check_exn
+                 \"ProtoDemoDemoDemoDemoDemoDemoDemoDemoDemoDemoD3c8k9\")
+       let sources = Tezos_data.Protocol.{
+         expected_env = V1 ;
+         components = [(* FIXME ?? *)] ;
+    } end)"))))
+
+(library
+  ((name tezos_protocol_environment_demo)
+  (libraries (node_updater))
+  (modules (Environment))))
+
+(library
+  ((name tezos_embedded_raw_protocol_demo)
+   (libraries (tezos_protocol_environment_demo))
+   (library_flags (:standard -linkall))
+   (flags (:standard -nopervasives -nostdlib
+                     -w +a-4-6-7-9-29-40..42-44-45-48
+                     -warn-error -a+8
+                     -open Tezos_protocol_environment_demo__Environment
+                     -open Error_monad
+                     -open Hash
+                     -open Tezos_data))
+   (modules (:standard \ Environment Registerer))))
+
+(library
+  ((name tezos_embedded_protocol_demo)
+   (library_flags (:standard -linkall))
+   (libraries (tezos_embedded_raw_protocol_demo node_shell))
+   (modules (Registerer))))
+
+(alias
+ ((name runtest_sandboxing)
+  (deps ((glob_files *.ml)
+         (glob_files *.mli)
+         TEZOS_PROTOCOL))
+  (action (run ${exe:../../compiler_main.exe} dummy_genesis ${path-no-dep:.}))))

src/proto/genesis/jbuild

@@ -0,0 +1,55 @@
+(jbuild_version 1)
+
+(rule
+ ((targets (environment.ml))
+  (action
+    (write-file ${@}
+      "include Tezos_protocol_environment.Make(struct let name = \"genesis\" end)()"))))
+
+(rule
+ ((targets (registerer.ml))
+  (action
+    (write-file ${@}
+       "module Ignored =
+  State.Register_embedded_protocol
+    (Tezos_protocol_environment_genesis.Environment)
+    (Tezos_embedded_raw_protocol_genesis.Main)
+    (struct
+       let hash =
+         Some (Hash.Protocol_hash.of_b58check_exn
+                 \"ProtoGenesisGenesisGenesisGenesisGenesisGenesk612im\")
+       let sources = Tezos_data.Protocol.{
+         expected_env = V1 ;
+         components = [(* FIXME ?? *)] ;
+    } end)"))))
+
+(library
+  ((name tezos_protocol_environment_genesis)
+  (libraries (node_updater))
+  (modules (Environment))))
+
+(library
+  ((name tezos_embedded_raw_protocol_genesis)
+   (libraries (tezos_protocol_environment_genesis))
+   (library_flags (:standard -linkall))
+   (flags (:standard -nopervasives -nostdlib
+                     -w +a-4-6-7-9-29-40..42-44-45-48
+                     -warn-error -a+8
+                     -open Tezos_protocol_environment_genesis__Environment
+                     -open Error_monad
+                     -open Hash
+                     -open Tezos_data))
+   (modules (:standard \ Environment Registerer))))
+
+(library
+  ((name tezos_embedded_protocol_genesis)
+   (library_flags (:standard -linkall))
+   (libraries (tezos_embedded_raw_protocol_genesis node_shell))
+   (modules (Registerer))))
+
+(alias
+ ((name runtest_sandboxing)
+  (deps ((glob_files *.ml)
+         (glob_files *.mli)
+         TEZOS_PROTOCOL))
+  (action (run ${exe:../../compiler_main.exe} dummy_genesis ${path-no-dep:.}))))

src/proto/jbuild

@@ -0,0 +1,5 @@
+(jbuild_version 1)
+
+(alias
+ ((name runtest)
+  (deps ((alias_rec runtest_sandboxing)))))

src/tezos-deps.opam

@@ -1,41 +0,0 @@
-opam-version: "1.2"
-name: "tezos-deps"
-version: "dev"
-maintainer: "Grégoire Henry <gregoire.henry@ocamlpro.com>"
-authors: [
-  "Arthur Breitman <arthur.breitman@gmail.com>"
-  "Benjamin Canou <benjamin@ocamlpro.com>"
-  "Pierre Chambart <pierre@ocamlpro.com>"
-  "Grégoire Henry <gregoire@ocamlpro.com>"
-]
-dev-repo: "https://github.com/tezos/tezos.git"
-homepage: "https://gihub.com/tezos/tezos"
-bug-reports: "https://github.com/tezos/tezos/issues"
-depends: [
-  "ocamlfind" {build}
-  "base-bigarray"
-  "base-threads"
-  "conf-libev"
-  "calendar"
-  "cohttp" {>= "0.21" }
-  "conduit"
-  "ezjsonm" {>= "0.5.0" }
-  "git"
-  "git-unix"
-  "irmin" {>= "1.3" }
-  "irmin-unix" {>= "1.3" }
-  "lwt" {>= "3.0.0" }
-  "lwt_ssl"
-  "menhir"
-  "mtime" { >= "1.0.0" }
-  "ocp-ocamlres" {>= "dev" }
-  "ocplib-endian"
-  "ocplib-json-typed"
-  "ocplib-resto" {>= "dev" }
-  "reactiveData"
-  "sodium" {>= "0.3.0" }
-  "magic-mime"
-  "leveldb" {>= "1.1.2" }
-  "kaputt" # { test }
-  "bisect_ppx" # { test }
-]

src/utils/jbuild

@@ -0,0 +1,23 @@
+(jbuild_version 1)
+
+(library
+ ((name utils)
+  (libraries
+   (
+    ;; External
+    base64
+    calendar
+    ezjsonm
+    ipaddr.unix
+    lwt.unix
+    mtime.clock.os
+    nocrypto
+    sodium
+    zarith
+    ;; Internal
+    minutils
+    ))
+  (flags (:standard -w +27-30-40@8))
+  (wrapped false)))
+
+

test/jbuild

@@ -0,0 +1,48 @@
+(jbuild_version 1)
+
+(alias
+ ((name runtest_basic.sh)
+  (deps (../src/node_main.exe
+         ../src/client_main.exe
+         sandbox.json
+         test_basic.sh
+         lib/test_lib.inc.sh
+         ../scripts/node_lib.inc.sh
+         ../scripts/client_lib.inc.sh
+         (glob_files contracts/*)
+         ))
+  (locks  (/tcp-port/18731
+           /tcp-port/19731))
+  (action (run bash ${path:test_basic.sh}))))
+
+(alias
+ ((name runtest_contracts.sh)
+  (deps (../src/node_main.exe
+         ../src/client_main.exe
+         sandbox.json
+         test_contracts.sh
+         lib/test_lib.inc.sh
+         ../scripts/node_lib.inc.sh
+         ../scripts/client_lib.inc.sh
+         (glob_files contracts/*)
+         ))
+  (locks  (/tcp-port/18732
+           /tcp-port/19732))
+  (action (run bash ${path:test_contracts.sh}))))
+
+(alias
+ ((name runtest_multinode.sh)
+  (deps (../src/node_main.exe
+         ../src/client_main.exe
+         sandbox.json
+         test_multinode.sh
+         lib/test_lib.inc.sh
+         ../scripts/node_lib.inc.sh
+         ../scripts/client_lib.inc.sh
+         (glob_files contracts/*)
+         ))
+  (locks  (/tcp-port/18731 /tcp-port/18732 /tcp-port/18733 /tcp-port/18734
+           /tcp-port/18735 /tcp-port/18736 /tcp-port/18737 /tcp-port/18738
+           /tcp-port/19731 /tcp-port/19732 /tcp-port/19733 /tcp-port/19734
+           /tcp-port/19735 /tcp-port/19736 /tcp-port/19737 /tcp-port/19738))
+  (action (run bash ${path:test_multinode.sh}))))

test/lib/jbuild

@@ -0,0 +1,6 @@
+(jbuild_version 1)
+
+(library
+ ((name test_lib)
+  (libraries (kaputt utils minutils))
+  (wrapped false)))

test/lib/node_helpers.ml

@@ -26,7 +26,6 @@ let handle_error res log_file_name =
       ignore (Sys.command (Printf.sprintf "cat %s" log_file_name) : int) ;
       raise Node_exited_prematurely
 
-
 let fork_node ?(timeout = 4) ?(port = 18732) ?sandbox () =
   let data_dir =
     Printf.sprintf
@@ -37,9 +36,11 @@ let fork_node ?(timeout = 4) ?(port = 18732) ?sandbox () =
     Filename.open_temp_file "tezos_node_" ".log" in
   let log_fd = Unix.descr_of_out_channel log_file in
   let null_fd = Unix.(openfile "/dev/null" [O_RDONLY] 0o644) in
+  let exe =
+    let (//) = Filename.concat in
+    Filename.(Sys.getcwd () // ".." // "src" // "node_main.exe") in
   let pid =
-    Unix.create_process
+    Unix.create_process exe
-      Filename.(concat (dirname (Sys.getcwd ())) "tezos-node")
       [| "tezos-node" ;
          "run" ;
          "--data-dir"; data_dir ;

test/lib/test_lib.inc.sh

@@ -6,6 +6,8 @@ src_dir="$(dirname "$test_dir")"
 cd "$test_dir"
 
 sandbox_file="$test_dir/sandbox.json"
+local_node="$src_dir/src/node_main.exe"
+local_client="$src_dir/src/client_main.exe"
 
 source $src_dir/scripts/node_lib.inc.sh
 source $src_dir/scripts/client_lib.inc.sh

test/p2p/jbuild

@@ -0,0 +1,40 @@
+(jbuild_version 1)
+
+(executables
+ ((names (test_p2p_connection
+          test_p2p_connection_pool
+          test_p2p_io_scheduler))
+  (libraries (minutils utils test_lib node_net))
+  (flags (:standard -linkall
+                    -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))))
+
+(alias
+ ((name buildtest)
+  (deps (test_p2p_connection.exe
+         test_p2p_connection_pool.exe
+         test_p2p_io_scheduler.exe))))
+
+(alias
+ ((name runtest_p2p_connection)
+  (action (run ${exe:test_p2p_connection.exe} -v))))
+
+(alias
+ ((name runtest_p2p_connection_pool)
+  (action (run ${exe:test_p2p_connection_pool.exe} --clients 10 --repeat 5 -v))))
+
+(alias
+ ((name runtest_p2p_io_scheduler)
+  (action (run ${exe:test_p2p_io_scheduler.exe}
+               	   --delay 5 --clients 8
+                   --max-upload-speed 262144 ;; 1 << 18 = 256kB
+                   --max-download-speed 1048576 ;; 1 << 20 = 1MB
+                   ))))
+
+(alias
+ ((name runtest)
+  (deps ((alias runtest_p2p_connection)
+         (alias runtest_p2p_connection_pool)
+         (alias runtest_p2p_io_scheduler)))))

test/proto_alpha/jbuild

@@ -0,0 +1,62 @@
+(jbuild_version 1)
+
+(executables
+ ((names (test_endorsement
+          ;; test_michelson_parser
+          test_origination
+          test_transaction
+          test_vote))
+  (libraries (test_lib
+              client_lib
+              client_embedded_genesis
+              client_embedded_alpha))
+  (flags (:standard -open Error_monad
+                    -open Hash
+                    -open Tezos_data
+                    -open Tezos_protocol_environment_alpha
+                    -open Tezos_embedded_raw_protocol_alpha
+                    -open Tezos_context
+                    -open Client_embedded_alpha))))
+
+(alias
+ ((name buildtest)
+  (deps (test_endorsement.exe
+         ;; test_michelson_parser.exe
+         test_origination.exe
+         test_transaction.exe
+         test_vote.exe))))
+
+(alias
+ ((name runtest_endorsement)
+  (deps (../../src/node_main.exe
+         sandbox.json))
+  (locks  (/tcp-port/18100))
+  (action (run ${exe:test_endorsement.exe}))))
+
+(alias
+ ((name runtest_origination)
+  (deps (../../src/node_main.exe
+         sandbox.json))
+  (locks  (/tcp-port/18200))
+  (action (run ${exe:test_origination.exe}))))
+
+(alias
+ ((name runtest_transaction)
+  (deps (../../src/node_main.exe
+         sandbox.json))
+  (locks  (/tcp-port/18300))
+  (action (run ${exe:test_transaction.exe}))))
+
+(alias
+ ((name runtest_vote)
+  (deps (../../src/node_main.exe
+         sandbox-vote.json))
+  (locks  (/tcp-port/18400))
+  (action (run ${exe:test_vote.exe}))))
+
+(alias
+ ((name runtest)
+  (deps ((alias runtest_endorsement)
+         (alias runtest_origination)
+         (alias runtest_transaction)
+         (alias runtest_vote)))))

test/shell/jbuild

@@ -0,0 +1,41 @@
+(jbuild_version 1)
+
+(executables
+ ((names (test_context
+          test_state
+          test_store))
+  (libraries (minutils
+              utils
+              test_lib
+              node_shell
+              tezos_embedded_protocol_demo
+              tezos_embedded_protocol_alpha
+              tezos_embedded_protocol_genesis))
+  (flags (:standard -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))))
+
+(alias
+ ((name buildtest)
+  (deps (test_context.exe
+         test_state.exe
+         test_store.exe))))
+
+(alias
+ ((name runtest_context)
+  (action (run ${exe:test_context.exe}))))
+
+(alias
+ ((name runtest_state)
+  (action (run ${exe:test_state.exe}))))
+
+(alias
+ ((name runtest_store)
+  (action (run ${exe:test_store.exe}))))
+
+(alias
+ ((name runtest)
+  (deps ((alias runtest_context)
+         (alias runtest_state)
+         (alias runtest_store)))))

test/test_contracts.sh

@@ -6,7 +6,7 @@ set -o pipefail
 test_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)")"
 source $test_dir/lib/test_lib.inc.sh
 
-start_node 1
+start_node 2
 activate_alpha
 
 key1=foo

test/utils/jbuild

@@ -0,0 +1,49 @@
+(jbuild_version 1)
+
+(executables
+ ((names (test_data_encoding
+          test_lwt_pipe
+          test_merkle
+          test_stream_data_encoding
+          test_utils))
+  (libraries (minutils utils test_lib))
+  (flags (:standard -open Error_monad
+                    -open Hash
+                    -open Utils
+                    -open Tezos_data))))
+
+(alias
+  ((name buildtest)
+   (deps (test_data_encoding.exe
+          test_lwt_pipe.exe
+          test_merkle.exe
+          test_stream_data_encoding.exe
+          test_utils.exe))))
+
+(alias
+ ((name runtest_data_encoding)
+  (action (run ${exe:test_data_encoding.exe}))))
+
+(alias
+ ((name runtest_lwt_pipe)
+  (action (run ${exe:test_lwt_pipe.exe}))))
+
+(alias
+ ((name runtest_merkle)
+  (action (run ${exe:test_merkle.exe}))))
+
+(alias
+ ((name runtest_stream_data_encoding)
+  (action (run ${exe:test_stream_data_encoding.exe}))))
+
+(alias
+ ((name runtest_utils)
+  (action (run ${exe:test_utils.exe}))))
+
+(alias
+ ((name runtest)
+  (deps ((alias runtest_data_encoding)
+         (alias runtest_lwt_pipe)
+         (alias runtest_merkle)
+         (alias runtest_stream_data_encoding)
+         (alias runtest_utils)))))

tezos.opam

@@ -0,0 +1,43 @@
+opam-version: "1.2"
+version: "dev"
+maintainer: "contact@tezos.com"
+authors: [ "Tezos devteam" ]
+homepage: "https://www.tezos.com/"
+bug-reports: "https://gitlab.com/tezos/tezos/issues"
+dev-repo: "https://gitlab.com/tezos/tezos.git"
+license: "unreleased"
+depends: [
+  "ocamlfind" {build}
+  "jbuilder" {build}
+  "base-bigarray"
+  "base-threads"
+  "conf-libev"
+  "calendar"
+  "cohttp" { >= "0.21" }
+  "conduit"
+  "ezjsonm" { >= "0.5.0" }
+  "git"
+  "git-unix"
+  "irmin" { >= "1.3" }
+  "irmin-unix" { >= "1.3" }
+  "lwt" { >= "3.0.0" }
+  "lwt_ssl"
+  "menhir"
+  "mtime" { >= "1.0.0" }
+  "ocp-ocamlres" { >= "dev" }
+  "ocplib-endian"
+  "ocplib-json-typed"
+  "ocplib-resto" { >= "dev" }
+  "reactiveData"
+  "sodium" { >= "0.3.0" }
+  "magic-mime"
+  "leveldb" { >= "1.1.2" }
+  "kaputt" # { test }
+  "bisect_ppx" # { test }
+]
+build: [
+  [ "jbuilder" "build" "-p" name "-j" jobs ]
+]
+build-test: [
+  [ "jbuilder" "runtest" ]
+]