diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..cd60a33be --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +.git + +tezos-node +tezos-protocol-compiler +tezos-client +tezos-webclient +tezos-attacker + +src/Makefile.local + +src/webclient_static.ml +src/.depend + +src/node/updater/environment_gen +src/node/updater/proto_environment.mli +src/compiler/embedded_cmis.ml + +src/proto/**/_tzbuild +src/proto/register_client_*.ml +src/client/embedded/**/_tzbuild + +src/client/embedded/demo/.depend + +src/client/embedded/bootstrap/.depend +src/client/embedded/bootstrap/concrete_lexer.ml +src/client/embedded/bootstrap/concrete_parser.ml +src/client/embedded/bootstrap/concrete_parser.mli +src/client/embedded/bootstrap/webclient_proto_static.ml +src/client/embedded/bootstrap/main.byte +src/client/embedded/bootstrap/webclient_static/main.js +src/client/embedded/bootstrap/webclient/browser/main.byte +src/client/embedded/bootstrap/webclient/static/main.js +src/client/embedded/bootstrap/webclient/webclient_proto_static.ml + +test/.depend +test/reports + +test/test-store +test/test-state +test/test-context +test/test-basic +test/test-data-encoding +test/test-p2p-io-scheduler +test/test-p2p-connection +test/test-p2p-connection-pool +test/LOG + +**/*~ +**/\#*\# + +**/*.[oa] +**/*.so +**/*~ +**/*.cm[iaoxt] +**/*.cmti +**/*.cmxa +**/*.cmxs +**/*.cmp +**/*.mli.deps +**/*.ml.deps +**/*.mli.deps.byte +**/*.ml.deps.byte + +**/bisect*.out + +**/*.rej +**/*.orig diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87b7a28df..cb640e4d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,10 @@ variables: - ocaml_image: alpine_ocaml-4.03.0 - build_image_name: ocp_tezos_build + image_name: ocp_tezos + image_id: ${CI_BUILD_REF_NAME} + build_image_name: ${image_name}_build_deps build_image_id: ${CI_BUILD_REF} - build_image: ${build_image_name}:${ocaml_image} -image: ${build_image}_${build_image_id} +image: ${build_image_name}:${build_image_id} stages: - build_deps @@ -21,8 +21,8 @@ build_deps: tags: - docker_builder script: - - ./scripts/create_docker_builder.sh - ${build_image_name} ${ocaml_image} "_${build_image_id}" + - ./scripts/create_build_deps_docker_image.sh + ${build_image_name} ${build_image_id} build: stage: build @@ -137,15 +137,29 @@ test:p2p-connection-pool: - build - build:test -expurge: - stage: expurge +publish:docker: + stage: publish + image: ocp:docker + tags: + - docker_builder + script: + - ./scripts/create_docker_image.sh ${image_name} ${build_image_id} + - docker tag ${image_name}:${build_image_id} + "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - docker push "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + - docker rmi "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + +publish:github: + image: ocaml/opam + stage: publish tags: - tezos_builder only: - master@tezos/tezos script: - echo "${CI_KH}" > ~/.ssh/known_hosts - - echo "${CI_PK_GITLAB}" > ~/.ssh/id_rsa + - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa - chmod 400 ~/.ssh/id_rsa - rm -fr .git/refs/original - git filter-branch --prune-empty --index-filter @@ -162,23 +176,7 @@ expurge: export GIT_AUTHOR_EMAIL="contact@tezos.com" ; fi' HEAD - - git tag ${CI_BUILD_REF}_expurged - - git push git@gitlab.ocamlpro.com:${CI_PROJECT_PATH}.git - -f --tags HEAD:master-expurged - -publish:github: - image: ocaml/opam:${ocaml_image} - stage: publish - tags: - - tezos_builder - only: - - master@tezos/tezos - script: - - echo "${CI_KH}" > ~/.ssh/known_hosts - - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa - - chmod 400 ~/.ssh/id_rsa - - git reset ${CI_BUILD_REF}_expurged - - git push git@github.com:tezos/tezos.git -f HEAD:master + - git push git@github.com:OCamlPro/tezos.git -f HEAD:master cleanup: stage: cleanup @@ -186,7 +184,7 @@ cleanup: tags: - docker_builder script: - - docker tag ${build_image}_${build_image_id} - ${build_image}_${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME} - - docker rmi ${build_image}_${build_image_id} + - docker tag ${build_image_name}:${build_image_id} + ${build_image_name}:${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME} + - docker rmi ${build_image_name}:${build_image_id} when: always diff --git a/Makefile b/Makefile index dcdf44df4..6522e1c51 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,16 @@ clean: ${MAKE} -C src clean ${MAKE} -C test clean +partial-clean: + ${MAKE} -C src partial-clean + ${MAKE} -C test clean + .PHONY: test test: ${MAKE} -C test build-deps: @./scripts/install_build_deps.sh all + +docker-image: + @./scripts/create_docker_image.sh diff --git a/scripts/Dockerfile.binaries.in b/scripts/Dockerfile.binaries.in new file mode 100644 index 000000000..8831fb436 --- /dev/null +++ b/scripts/Dockerfile.binaries.in @@ -0,0 +1,24 @@ +FROM alpine:$alpine_version + +LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux" + +RUN apk update && \ + apk upgrade && \ + apk add sudo bash libssl1.0 libsodium gmp && \ + rm -f /var/cache/apk/* && \ + adduser -S tezos && \ + echo 'tezos ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/tezos && \ + chmod 440 /etc/sudoers.d/tezos && \ + chown root:root /etc/sudoers.d/tezos && \ + sed -i 's/^Defaults.*requiretty//g' /etc/sudoers +USER tezos + +COPY . /home/tezos +WORKDIR /home/tezos + +RUN sudo chown root:root bin/* && \ + sudo chmod a+rx bin/* && \ + sudo mv bin/* /usr/local/bin && \ + rmdir bin + +ENTRYPOINT [ "/bin/bash" ] diff --git a/scripts/Dockerfile.build_bin.in b/scripts/Dockerfile.build_bin.in new file mode 100644 index 000000000..cb903902f --- /dev/null +++ b/scripts/Dockerfile.build_bin.in @@ -0,0 +1,7 @@ +FROM $base_name:$base_version + +COPY . /home/opam/tezos +RUN sudo chown -R opam /home/opam/tezos && \ + opam config exec -- make -C tezos -j4 && \ + mkdir bin && \ + cp tezos/tezos-* bin diff --git a/scripts/Dockerfile.build_deps.in b/scripts/Dockerfile.build_deps.in new file mode 100644 index 000000000..030511316 --- /dev/null +++ b/scripts/Dockerfile.build_deps.in @@ -0,0 +1,8 @@ +FROM ocaml/opam:alpine-$alpine_version_ocaml-$ocaml_version + +COPY scripts/install_build_deps.sh scripts/version.sh scripts/ +COPY src/tezos-deps.opam src/ +RUN sudo apk add libsodium-dev && \ + opam config exec -- ./scripts/install_build_deps.sh all \ + rm -fr ~/.opam/log/ \ + "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi diff --git a/scripts/create_build_deps_docker_image.sh b/scripts/create_build_deps_docker_image.sh new file mode 100755 index 000000000..144db6b2f --- /dev/null +++ b/scripts/create_build_deps_docker_image.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +set -e +set -x + +script_dir="$(dirname "$(readlink -f "$0")")" +src_dir="$(dirname "$script_dir")" +cd "$src_dir" + +. scripts/version.sh +image_name="${1:-tezos_build_deps}" +image_version="${2:-latest}" + +sed scripts/Dockerfile.build_deps.in \ + -e 's/$alpine_version/'"$alpine_version"'/g' \ + -e 's/$ocaml_version/'"$ocaml_version"'/g' > Dockerfile + +docker build --pull -t "$image_name:$image_version" . + +rm Dockerfile diff --git a/scripts/create_docker_builder.sh b/scripts/create_docker_builder.sh deleted file mode 100755 index c89f6328d..000000000 --- a/scripts/create_docker_builder.sh +++ /dev/null @@ -1,32 +0,0 @@ -#! /bin/sh - -set -x - -dir=$(mktemp -d) -cur_dir="$(dirname "$(readlink -f "$0")")" - -image_name=${1:=tezos_build} -ocaml_version=${2:=alpine_ocaml-4.03.0} -image_version=$3 - -docker pull ocaml/opam:${ocaml_version} - -cp ${cur_dir}/install_build_deps.sh ${dir} -cp ${cur_dir}/../src/tezos-deps.opam ${dir} -cat > ${dir}/Dockerfile < Dockerfile + +echo Building tezos... +tmp_image="$(docker build -q .)" +tmp_container="$(docker run -dit "$tmp_image" true)" + +docker cp "$tmp_container":/home/opam/bin/ bin + +echo Building minimal docker image... +sed scripts/Dockerfile.binaries.in \ + -e 's/$alpine_version/'"$alpine_version"'/g' > Dockerfile +docker build -q -t "$image_name:$image_version" . diff --git a/scripts/install_build_deps.sh b/scripts/install_build_deps.sh index 646a2f4c6..c3562b864 100755 --- a/scripts/install_build_deps.sh +++ b/scripts/install_build_deps.sh @@ -1,39 +1,40 @@ -#!/bin/sh +#! /bin/sh -if ! [ -f 'src/tezos-deps.opam' ]; then - echo - echo " Please run from the project's root directory. Aborting." - echo - exit 1 -fi +script_dir="$(dirname "$(readlink -f "$0")")" +src_dir="$(dirname "$script_dir")" + +. "$script_dir/version.sh" -ocaml_version=4.03.0 if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then - echo - echo " Unexpected compiler version ($(ocaml -vnum))" - echo " You should use ocaml-$ocaml_version." - echo - exit 1 + echo ; + echo " Unexpected compiler version ($(ocaml -vnum))"; + echo " You should use ocaml-$ocaml_version."; + echo ; + exit 1; fi cmd="$1" if [ -z "$cmd" ]; then cmd=all; fi -case "$cmd" in +pin=false +depext=false +install=false + +case $cmd in pin) - pin=yes - ;; + pin=true + ;; depext) - depext=yes - ;; + depext=true + ;; install) - install=yes - ;; + install=true + ;; all) - pin=yes - depext=yes - install=yes - ;; + pin=true + depext=true + install=true + ;; *) echo "Unknown command '$cmd'." echo "Usage: $0 [pin|depext|install|all|]" @@ -43,7 +44,7 @@ esac set -e set -x -if ! [ -z "$pin" ]; then +if "$pin"; then opam pin --yes remove --no-action --dev-repo ocplib-resto || true opam pin --yes add --no-action --dev-repo sodium opam pin --yes add --no-action --dev-repo ocp-ocamlres @@ -56,15 +57,15 @@ if ! [ -z "$pin" ]; then opam pin --yes add --no-action tezos-deps src fi -if ! [ -z "$depext" ]; then - ## In our CI, this rule is executed as user 'root' - ## The other rules are executed as user 'opam'. +if "$depext"; then opam list --installed depext || opam install depext - opam depext tezos-deps + opam depext $DEPEXTOPT tezos-deps fi -if ! [ -z "$install" ]; then - opam install tezos-deps - ## This seems broken in the current opam-repo (2016-12-09) - ## opam install --build-test tezos-deps +if "$install"; then + if opam list --installed tezos-deps ; then + opam upgrade $(opam list -s --required-by tezos-deps | grep -ve '^ocaml *$') + else + opam install tezos-deps + fi fi diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100644 index 000000000..8830b85dc --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +alpine_version=3.5 +ocaml_version=4.03.0 diff --git a/src/Makefile b/src/Makefile index 2617fc8af..758de0561 100644 --- a/src/Makefile +++ b/src/Makefile @@ -65,7 +65,7 @@ node/updater/proto_environment.cmi: \ @echo OCAMLOPT ${TARGET} $@ @$(OCAMLOPT) -nopervasives -nostdlib -opaque -I tmp -I node/updater -c $< -clean:: +partial-clean:: rm -f node/updater/proto_environment.mli rm -f node/updater/environment_gen @@ -91,7 +91,7 @@ compiler/embedded_cmis.cmx: compiler/embedded_cmis.cmi compiler/embedded_cmis.ml: ${EMBEDDED_PROTOCOL_LIB_CMIS} @echo OCAMLRES ${TARGET} $(notdir $@) @$(OCAMLRES) -format ocaml -o $@ $^ -clean:: +partial-clean:: rm -f compiler/embedded_cmis.ml rm -rf tmp @@ -368,7 +368,7 @@ ${TZNODE}: minutils.cmxa utils.cmxa compiler.cmxa node.cmxa ${EMBEDDED_NODE_PROT @echo LINK $(notdir $@) @${OCAMLOPT} -linkall -linkpkg ${OCAMLFLAGS} -o $@ $^ -clean:: +partial-clean:: -rm -f proto/*.cm* proto/*.a ## Embedded protocol modules @@ -399,8 +399,6 @@ proto/client_embedded_proto_%.cmxa: \ $(addprefix -I , ${CLIENT_PROTO_INCLUDES}) \ $@ proto/$* - - clean:: rm -f ${TZNODE} @@ -529,7 +527,7 @@ client/embedded/webclient_%.cmx: \ $$(shell find client/embedded/%/webclient/static/) @$(MAKE) -C client/embedded/$* ../webclient_$*.cmx -clean:: +partial-clean:: -for d in $$(ls -d client/embedded/*/) ; do make clean -C $$d ; done -rm -f client/embedded/*.cm* client/embedded/*.o @@ -601,8 +599,9 @@ clean:: ## Cleaning -.PHONY: clean -clean:: +.PHONY: clean partial-clean +clean:: partial-clean +partial-clean:: -find \( -name \*.cm\* -or -name \*.cmp -or -name \*.out -or -name \*~ -or -name \*.o -or -name \*.a \) -delete ## Dependencies @@ -652,6 +651,6 @@ compiler/tezos_compiler.cmo compiler/tezos_compiler.cmx: \ @echo OCAMLDEP ${TARGET} $(notdir $^) @$(OCAMLDEP) $(INCLUDES) $^ > $@ -clean:: +partial-clean:: -rm -f .depend -find \( -name \*.mli.deps -or -name \*.ml.deps \) -delete