From 644d90be6db8663f7c7425058e6cdcdb2997de40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Fri, 6 Sep 2019 04:02:18 +0200 Subject: [PATCH] Fix CI and Makefile (WIP) --- .gitlab-ci.yml | 93 +++++++++++----------------- Makefile | 38 +++++++++++- docker/Dockerfile | 9 ++- makefile | 25 -------- scripts/build_ligo_local.sh | 10 ++- scripts/debug_show_versions.sh | 8 +++ scripts/install_build_environment.sh | 3 + scripts/install_opam.sh | 19 +++++- scripts/install_vendors_deps.sh | 12 ++++ scripts/setup_dev_switch.sh | 7 ++- scripts/setup_switch.sh | 15 +++++ scripts/test_ligo.sh | 5 +- src/README_INSTALL | 20 ------ src/parser/pascaligo/.gitlab-ci.yml | 21 ------- src/test/coase_tests.ml | 16 ++--- 15 files changed, 161 insertions(+), 140 deletions(-) delete mode 100644 makefile create mode 100755 scripts/debug_show_versions.sh create mode 100755 scripts/install_vendors_deps.sh create mode 100755 scripts/setup_switch.sh delete mode 100644 src/README_INSTALL delete mode 100644 src/parser/pascaligo/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fb22a317..6b5a86e70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,34 +9,21 @@ stages: - test .website_build: &website_build - stage: build_and_deploy_website + stage: test # TODO DODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODO restore to build_and_deploy_website image: node:8 before_script: - scripts/install_native_dependencies.sh - # TODO: these things are moved to scripts in other branches. - - wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O opam-2.0.1-x86_64-linux - - cp opam-2.0.1-x86_64-linux /usr/local/bin/opam - - chmod +x /usr/local/bin/opam + - scripts/install_opam.sh # TODO: or scripts/install_build_environment.sh ? - export PATH="/usr/local/bin${PATH:+:}${PATH:-}" - - # Initialise opam - - printf '' | opam init --bare - eval $(opam config env) - - # Create switch - - printf '' | opam switch create toto ocaml-base-compiler.4.06.1 + - scripts/setup_switch.sh - eval $(opam config env) - - # Show versions and current switch - - echo "$PATH" - - opam --version - - printf '' | ocaml - - opam switch + - scripts/debug_show_versions.sh || true # install deps for internal documentation - opam install -y odoc - - vendors/opam-repository-tools/rewrite-local-opam-repository.sh - - opam repository add localrepo "file://$PWD/vendors/ligo-opam-repository-local-generated/" + # - vendors/opam-repository-tools/rewrite-local-opam-repository.sh + # - opam repository add localrepo "file://$PWD/vendors/ligo-opam-repository-local-generated/" - opam install -y --build-test --deps-only ./src/ - dune build -p ligo # TODO: also try instead from time to time: @@ -75,44 +62,36 @@ stages: before_script: # Install dependencies # rsync is needed by opam to sync a package installed from a local directory with the copy in ~/.opam - - apt-get update -qq - scripts/install_native_dependencies.sh - - scripts/install_opam.sh + - scripts/install_opam.sh # TODO: or scripts/install_build_environment.sh ? - export PATH="/usr/local/bin${PATH:+:}${PATH:-}" - - # Initialise opam, create switch, load opam environment variables - - printf '' | opam init --bare - - printf '' | opam switch create ligo-switch ocaml-base-compiler.4.06.1 + - echo "$PATH"; which ocaml || true - eval $(opam config env) - - # Show versions and current switch - - echo "$PATH" - - opam --version - - printf '' | ocaml - - opam switch + - echo "$PATH"; which ocaml || true + - scripts/setup_switch.sh + - echo "$PATH"; which ocaml || true + - eval $(opam config env) + - echo "$PATH"; which ocaml || true + - scripts/debug_show_versions.sh || true local-dune-job: <<: *before_script stage: test script: - - scripts/setup_ligo_opam_repository.sh - - opam install -y --build-test --deps-only ./src/ - - dune build -p ligo - # TODO: also try instead from time to time: - #- (cd ./src/; dune build -p ligo) + - scripts/install_vendors_deps.sh + - scripts/build_ligo_local.sh - dune build @ligo-test -# artifacts: -# paths: -# - src/ligo/bin/cli.ml -local-repo-job: - <<: *before_script - stage: test - script: - - vendors/opam-repository-tools/rewrite-local-opam-repository.sh - - opam repository add localrepo "file://$PWD/vendors/ligo-opam-repository-local-generated/" - #--build-test - - opam install -y ligo +# TODO: uncomment this + +# TODO +# local-repo-job: +# <<: *before_script +# stage: test +# script: +# - scripts/install_vendors_deps.sh +# # TODO: also try from time to time with --build-test +# - opam install -y ligo remote-repo-job: <<: *before_script @@ -130,19 +109,21 @@ remote-repo-job: only: - master -# Run a docker build without publishing to the registry -build-current-docker-image: - stage: build_docker - <<: *docker - <<: *docker_build - except: - - master - - dev +# TODO: uncomment this + +# # Run a docker build without publishing to the registry +# build-current-docker-image: +# stage: test # TODO DODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODO restore to build_docker +# <<: *docker +# <<: *docker_build +# except: +# - master +# - dev # When a MR/PR is merged to dev # take the previous build and publish it to Docker Hub build-and-publish-latest-docker-image: - stage: build_and_deploy_docker + stage: test # TODO DODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODODO restore to build_and_deploy_docker <<: *docker <<: *docker_build after_script: diff --git a/Makefile b/Makefile index f70b9412a..21dd72cc7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,37 @@ +.ONESHELL: + +all: test + +# Use install-deps instead of 'install' because usually 'make install' adds a +# binary to the system path and we don't want to confuse users +install-deps: +# Install ligo/tezos specific system-level dependencies + sudo scripts/install_native_dependencies.sh + scripts/install_build_environment.sh # TODO: or scripts/install_opam.sh ? + build-deps: - scripts/install_native_dependencies.sh - scripts/install_opam.sh + echo aa + export PATH="/usr/local/bin$${PATH:+:}$${PATH:-}" +# Create opam dev switch locally for use with Ligo, add merlin/etc + if [ -n "`opam switch show | grep -P ".+/ligo"`" ]; + then :; else scripts/setup_dev_switch.sh; + fi + echo bb + eval $$(opam config env) + echo cc +# Install OCaml build dependencies for Ligo + scripts/install_vendors_deps.sh + echo dd + scripts/install_ligo_with_dependencies.sh # TODO: rename & cleanup + echo ee + +build: build-deps + export PATH="/usr/local/bin$${PATH:+:}$${PATH:-}" + eval $$(opam config env) +# Build Ligo for local dev use + scripts/build_ligo_local.sh + +test: build + export PATH="/usr/local/bin$${PATH:+:}$${PATH:-}" + eval $$(opam config env) + scripts/test_ligo.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index dbc051aee..17a80641a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,12 +19,17 @@ WORKDIR /ligo # Install required native dependencies RUN sh scripts/install_native_dependencies.sh -# Setup a custom opam repository where ligo is published -RUN sh scripts/setup_ligo_opam_repository.sh +# Install OPAM +# TODO: or scripts/install_build_environment.sh ? +RUN sh scripts/install_opam.sh + +# Creat opam switch +RUN sh scripts/setup_switch.sh RUN opam update # Install ligo +RUN sh scripts/install_vendors_deps.sh RUN sh scripts/install_ligo_with_dependencies.sh # Use the ligo binary as a default command diff --git a/makefile b/makefile deleted file mode 100644 index 069943f5b..000000000 --- a/makefile +++ /dev/null @@ -1,25 +0,0 @@ -# Use install-deps instead of 'install' because usually 'make install' adds a -# binary to the system path and we don't want to confuse users -install-deps: -# Install ligo/tezos specific system-level dependencies - sudo scripts/install_native_dependencies.sh - -build-deps: -# Create opam dev switch locally for use with Ligo, add merlin/etc - if [ -n "`opam switch show | grep -P ".+/ligo"`" ]; - then exit; else scripts/setup_dev_switch.sh; - fi -# Set up the local ligo opam repository so that it can be built - if [ -n "`opam repo list --safe | grep -P "ligo-opam-repository"`" ]; - then exit; else scripts/setup_ligo_opam_repository.sh; - fi -# Install OCaml build dependencies for Ligo - scripts/install_ligo_with_dependencies.sh - -build: build-deps -# Build Ligo for local dev use - scripts/build_ligo_local.sh - -.ONESHELL: -test: build - scripts/test_ligo.sh diff --git a/scripts/build_ligo_local.sh b/scripts/build_ligo_local.sh index 85fcb6892..8ccfd8658 100755 --- a/scripts/build_ligo_local.sh +++ b/scripts/build_ligo_local.sh @@ -1,2 +1,8 @@ -eval $(opam env) -dune build -p ligo +#!/bin/sh +set -e + +eval $(opam config env) +dune build src # TODO: make it work with -p ligo + +# TODO: also try instead from time to time: +#- (cd ./src/; dune build -p ligo) diff --git a/scripts/debug_show_versions.sh b/scripts/debug_show_versions.sh new file mode 100755 index 000000000..1c744a566 --- /dev/null +++ b/scripts/debug_show_versions.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +echo "$PATH" +opam --version +printf '' | ocaml +opam switch + diff --git a/scripts/install_build_environment.sh b/scripts/install_build_environment.sh index 4d5e36400..958f855b1 100755 --- a/scripts/install_build_environment.sh +++ b/scripts/install_build_environment.sh @@ -1,3 +1,6 @@ +#!/bin/sh +set -e + # This script installs opam for the user. It should NOT be included in any makefiles/etc. if [ -n "`which opam`" ] diff --git a/scripts/install_opam.sh b/scripts/install_opam.sh index 1a89f6a9b..eb9c1c43f 100755 --- a/scripts/install_opam.sh +++ b/scripts/install_opam.sh @@ -1,10 +1,27 @@ #!/bin/sh set -e +set -x # TODO: this has many different modes of failure (file temp.opam-2.0.1-x86_64-linux.download-in-progress already exists, /usr/local/bin/opam already exists and is a directory or hard link, …) # Try to improve these aspects. -wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O temp.opam-2.0.1-x86_64-linux.download-in-progress +if command -v wget >/dev/null 2>&1; then + wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O temp.opam-2.0.1-x86_64-linux.download-in-progress +else + curl -L https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux --output temp.opam-2.0.1-x86_64-linux.download-in-progress +fi + +# debug +ls +apt -y install hexdump || true +apt -y install xxd || true +(cat temp.opam-2.0.1-x86_64-linux.download-in-progress | xxd | head -n 30) || true + cp -i temp.opam-2.0.1-x86_64-linux.download-in-progress /usr/local/bin/opam chmod +x /usr/local/bin/opam rm temp.opam-2.0.1-x86_64-linux.download-in-progress + +which opam || true + + +opam init -a --bare diff --git a/scripts/install_vendors_deps.sh b/scripts/install_vendors_deps.sh new file mode 100755 index 000000000..9d2035b45 --- /dev/null +++ b/scripts/install_vendors_deps.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +# Install local dependencies +opam install -y ./vendors/ligo-utils/simple-utils +opam install -y ./vendors/ligo-utils/tezos-protocol-alpha +opam install -y ./vendors/ligo-utils/tezos-protocol-alpha-parameters +opam install -y ./vendors/ligo-utils/memory-proto-alpha +opam install -y ./vendors/ligo-utils/tezos-utils/michelson-parser +opam install -y ./vendors/ligo-utils/tezos-utils +opam install -y ./vendors/ligo-utils/proto-alpha-utils +opam install -y getopt ppx_deriving menhir diff --git a/scripts/setup_dev_switch.sh b/scripts/setup_dev_switch.sh index 5b9ea1dca..88662c79a 100755 --- a/scripts/setup_dev_switch.sh +++ b/scripts/setup_dev_switch.sh @@ -1,4 +1,7 @@ -opam switch create . ocaml-base-compiler.4.06.1 -eval $(opam env) +#!/bin/sh +set -e + +"$(dirname "$0")"/setup_switch.sh + opam install -y ocp-indent tuareg merlin alcotest-lwt crowbar opam -y user-setup install diff --git a/scripts/setup_switch.sh b/scripts/setup_switch.sh new file mode 100755 index 000000000..070e720a7 --- /dev/null +++ b/scripts/setup_switch.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -e +set -x + +printf '' | opam switch create . 4.07.1 # toto ocaml-base-compiler.4.06.1 +eval $(opam config env) + +# Add Tezos opam repository +opam repo add tezos-opam-repository https://gitlab.com/nomadic-labs/tezos-opam-repository.git + +# TODO: move this to install_vendor_deps.sh +# Pin the versions of some dependencies +opam pin -y zarith 1.7 +opam pin -y ipaddr 3.1.0 +opam pin -y macaddr 3.1.0 diff --git a/scripts/test_ligo.sh b/scripts/test_ligo.sh index d13e26d3d..9fd6320e7 100755 --- a/scripts/test_ligo.sh +++ b/scripts/test_ligo.sh @@ -1,2 +1,5 @@ -eval $(opam env) +#!/bin/sh +set -e + +eval $(opam config env) dune build @ligo-test diff --git a/src/README_INSTALL b/src/README_INSTALL deleted file mode 100644 index 431f6ac8d..000000000 --- a/src/README_INSTALL +++ /dev/null @@ -1,20 +0,0 @@ -switch=titi -cd src/ligo -sudo apt -y install libev-dev libhidapi-dev -opam init -eval $(opam env) -opam switch create $switch ocaml-base-compiler.4.06.1 -eval $(opam env --switch=$switch --set-switch) -opam repository add new-tezos https://gitlab.com/ligolang/new-tezos-opam-repository.git - -# si une build a déjà été tentée, il vaut mieux git add tout ce qui est utile et git clean -dfx pour supprimer tout le reste (dune 1.7 crée des fichiers non compatibles avec dune 1.6) -opam install -y ocplib-endian alcotest - -(cd ligo-parser && opam install -y .) -eval $(opam env) -(cd ligo-helpers && opam install -y .) -eval $(opam env) -(opam install -y .) -eval $(opam env) -opam install merlin ocp-indent ledit -opam user-setup install diff --git a/src/parser/pascaligo/.gitlab-ci.yml b/src/parser/pascaligo/.gitlab-ci.yml deleted file mode 100644 index a64366465..000000000 --- a/src/parser/pascaligo/.gitlab-ci.yml +++ /dev/null @@ -1,21 +0,0 @@ -before_script: - - apt-get update -qq - - apt-get -y -qq install libhidapi-dev libcap-dev bubblewrap - - wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O opam-2.0.1-x86_64-linux - - cp opam-2.0.1-x86_64-linux /usr/local/bin/opam - - chmod +x /usr/local/bin/opam - - export PATH="/usr/local/bin${PATH:+:}${PATH:-}" - - echo "$PATH" - - printf '' | opam init - - eval $(opam config env) - - opam repository add tezos-opam-repository https://gitlab.com/ligolang/tezos-opam-repository.git - - eval $(opam config env) - - opam --version - - printf '' | ocaml - -default-job: - script: - - opam install -y --working-dir . - artifacts: - paths: - - Parser.exe diff --git a/src/test/coase_tests.ml b/src/test/coase_tests.ml index 0db0e53f8..1931b9857 100644 --- a/src/test/coase_tests.ml +++ b/src/test/coase_tests.ml @@ -75,13 +75,13 @@ let (first_owner , first_contract) = let open Proto_alpha_utils.Memory_proto_alpha in let id = List.nth dummy_environment.identities 0 in let kt = id.implicit_contract in - Alpha_context.Contract.to_b58check kt , kt + Protocol.Alpha_context.Contract.to_b58check kt , kt let second_owner = let open Proto_alpha_utils.Memory_proto_alpha in let id = List.nth dummy_environment.identities 1 in let kt = id.implicit_contract in - Alpha_context.Contract.to_b58check kt + Protocol.Alpha_context.Contract.to_b58check kt let basic a b cards next_id = let card_patterns = List.map card_pattern_ez [ @@ -113,13 +113,13 @@ let buy () = let%bind () = let%bind amount = trace_option (simple_error "getting amount for run") @@ - Memory_proto_alpha.Alpha_context.Tez.of_mutez @@ Int64.of_int 10000000000 in + Memory_proto_alpha.Protocol.Alpha_context.Tez.of_mutez @@ Int64.of_int 10000000000 in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount () in expect_eq_n_pos_small ~options program "buy_single" make_input make_expected in let%bind () = let%bind amount = trace_option (simple_error "getting amount for run") @@ - Memory_proto_alpha.Alpha_context.Tez.of_mutez @@ Int64.of_int 0 in + Memory_proto_alpha.Protocol.Alpha_context.Tez.of_mutez @@ Int64.of_int 0 in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount () in trace_strong (simple_error "could buy without money") @@ Assert.assert_fail @@ -152,13 +152,13 @@ let dispatch_buy () = let%bind () = let%bind amount = trace_option (simple_error "getting amount for run") @@ - Memory_proto_alpha.Alpha_context.Tez.of_mutez @@ Int64.of_int 10000000000 in + Memory_proto_alpha.Protocol.Alpha_context.Tez.of_mutez @@ Int64.of_int 10000000000 in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount () in expect_eq_n_pos_small ~options program "main" make_input make_expected in let%bind () = let%bind amount = trace_option (simple_error "getting amount for run") @@ - Memory_proto_alpha.Alpha_context.Tez.of_mutez @@ Int64.of_int 0 in + Memory_proto_alpha.Protocol.Alpha_context.Tez.of_mutez @@ Int64.of_int 0 in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount () in trace_strong (simple_error "could buy without money") @@ Assert.assert_fail @@ -190,7 +190,7 @@ let transfer () = e_pair ops storage in let%bind () = - let amount = Memory_proto_alpha.Alpha_context.Tez.zero in + let amount = Memory_proto_alpha.Protocol.Alpha_context.Tez.zero in let payer = first_contract in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount ~payer () in expect_eq_n_strict_pos_small ~options program "transfer_single" make_input make_expected in @@ -220,7 +220,7 @@ let sell () = Ast_simplified.Misc.assert_value_eq (expected_storage , storage) in let%bind () = - let amount = Memory_proto_alpha.Alpha_context.Tez.zero in + let amount = Memory_proto_alpha.Protocol.Alpha_context.Tez.zero in let payer = first_contract in let options = Proto_alpha_utils.Memory_proto_alpha.make_options ~amount ~payer () in expect_n_strict_pos_small ~options program "sell_single" make_input make_expecter in