diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..cd60a33be --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +.git + +tezos-node +tezos-protocol-compiler +tezos-client +tezos-webclient +tezos-attacker + +src/Makefile.local + +src/webclient_static.ml +src/.depend + +src/node/updater/environment_gen +src/node/updater/proto_environment.mli +src/compiler/embedded_cmis.ml + +src/proto/**/_tzbuild +src/proto/register_client_*.ml +src/client/embedded/**/_tzbuild + +src/client/embedded/demo/.depend + +src/client/embedded/bootstrap/.depend +src/client/embedded/bootstrap/concrete_lexer.ml +src/client/embedded/bootstrap/concrete_parser.ml +src/client/embedded/bootstrap/concrete_parser.mli +src/client/embedded/bootstrap/webclient_proto_static.ml +src/client/embedded/bootstrap/main.byte +src/client/embedded/bootstrap/webclient_static/main.js +src/client/embedded/bootstrap/webclient/browser/main.byte +src/client/embedded/bootstrap/webclient/static/main.js +src/client/embedded/bootstrap/webclient/webclient_proto_static.ml + +test/.depend +test/reports + +test/test-store +test/test-state +test/test-context +test/test-basic +test/test-data-encoding +test/test-p2p-io-scheduler +test/test-p2p-connection +test/test-p2p-connection-pool +test/LOG + +**/*~ +**/\#*\# + +**/*.[oa] +**/*.so +**/*~ +**/*.cm[iaoxt] +**/*.cmti +**/*.cmxa +**/*.cmxs +**/*.cmp +**/*.mli.deps +**/*.ml.deps +**/*.mli.deps.byte +**/*.ml.deps.byte + +**/bisect*.out + +**/*.rej +**/*.orig diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87b7a28df..cb640e4d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,10 @@ variables: - ocaml_image: alpine_ocaml-4.03.0 - build_image_name: ocp_tezos_build + image_name: ocp_tezos + image_id: ${CI_BUILD_REF_NAME} + build_image_name: ${image_name}_build_deps build_image_id: ${CI_BUILD_REF} - build_image: ${build_image_name}:${ocaml_image} -image: ${build_image}_${build_image_id} +image: ${build_image_name}:${build_image_id} stages: - build_deps @@ -21,8 +21,8 @@ build_deps: tags: - docker_builder script: - - ./scripts/create_docker_builder.sh - ${build_image_name} ${ocaml_image} "_${build_image_id}" + - ./scripts/create_build_deps_docker_image.sh + ${build_image_name} ${build_image_id} build: stage: build @@ -137,15 +137,29 @@ test:p2p-connection-pool: - build - build:test -expurge: - stage: expurge +publish:docker: + stage: publish + image: ocp:docker + tags: + - docker_builder + script: + - ./scripts/create_docker_image.sh ${image_name} ${build_image_id} + - docker tag ${image_name}:${build_image_id} + "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - docker push "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + - docker rmi "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME" + +publish:github: + image: ocaml/opam + stage: publish tags: - tezos_builder only: - master@tezos/tezos script: - echo "${CI_KH}" > ~/.ssh/known_hosts - - echo "${CI_PK_GITLAB}" > ~/.ssh/id_rsa + - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa - chmod 400 ~/.ssh/id_rsa - rm -fr .git/refs/original - git filter-branch --prune-empty --index-filter @@ -162,23 +176,7 @@ expurge: export GIT_AUTHOR_EMAIL="contact@tezos.com" ; fi' HEAD - - git tag ${CI_BUILD_REF}_expurged - - git push git@gitlab.ocamlpro.com:${CI_PROJECT_PATH}.git - -f --tags HEAD:master-expurged - -publish:github: - image: ocaml/opam:${ocaml_image} - stage: publish - tags: - - tezos_builder - only: - - master@tezos/tezos - script: - - echo "${CI_KH}" > ~/.ssh/known_hosts - - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa - - chmod 400 ~/.ssh/id_rsa - - git reset ${CI_BUILD_REF}_expurged - - git push git@github.com:tezos/tezos.git -f HEAD:master + - git push git@github.com:OCamlPro/tezos.git -f HEAD:master cleanup: stage: cleanup @@ -186,7 +184,7 @@ cleanup: tags: - docker_builder script: - - docker tag ${build_image}_${build_image_id} - ${build_image}_${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME} - - docker rmi ${build_image}_${build_image_id} + - docker tag ${build_image_name}:${build_image_id} + ${build_image_name}:${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME} + - docker rmi ${build_image_name}:${build_image_id} when: always diff --git a/Makefile b/Makefile index dcdf44df4..6522e1c51 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,16 @@ clean: ${MAKE} -C src clean ${MAKE} -C test clean +partial-clean: + ${MAKE} -C src partial-clean + ${MAKE} -C test clean + .PHONY: test test: ${MAKE} -C test build-deps: @./scripts/install_build_deps.sh all + +docker-image: + @./scripts/create_docker_image.sh diff --git a/scripts/Dockerfile.binaries.in b/scripts/Dockerfile.binaries.in index 66eb58b87..8831fb436 100644 --- a/scripts/Dockerfile.binaries.in +++ b/scripts/Dockerfile.binaries.in @@ -1,20 +1,24 @@ -FROM alpine:3.4 -# FIXME: I'm currently guessing the version of alpine that the opam image we've -# built the tezos binaries on is based on. if that becomes newer (e.g. alpine -# 3.5), the binaries built on that version of alpine could (will) depend on -# library symbols that won't be in alpine 3.4 +FROM alpine:$alpine_version -LABEL distro_style="apk" distro="alpine" distro_long="alpine-3.4" arch="x86_64" operatingsystem="linux" +LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux" RUN apk update && \ apk upgrade && \ - apk add sudo bash libsodium gmp && \ + apk add sudo bash libssl1.0 libsodium gmp && \ + rm -f /var/cache/apk/* && \ adduser -S tezos && \ echo 'tezos ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/tezos && \ chmod 440 /etc/sudoers.d/tezos && \ chown root:root /etc/sudoers.d/tezos && \ - sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers + sed -i 's/^Defaults.*requiretty//g' /etc/sudoers USER tezos -ADD built-bin /usr/local/bin +COPY . /home/tezos WORKDIR /home/tezos + +RUN sudo chown root:root bin/* && \ + sudo chmod a+rx bin/* && \ + sudo mv bin/* /usr/local/bin && \ + rmdir bin + +ENTRYPOINT [ "/bin/bash" ] diff --git a/scripts/Dockerfile.build_bin.in b/scripts/Dockerfile.build_bin.in index dfbc24954..cb903902f 100644 --- a/scripts/Dockerfile.build_bin.in +++ b/scripts/Dockerfile.build_bin.in @@ -1,13 +1,7 @@ -FROM tezos_build:$base_image +FROM $base_name:$base_version -ADD tezos /home/opam/tezos -ENV HOME /home/opam -WORKDIR $HOME/tezos -RUN sudo HOME="$HOME" opam config exec -- make clean && \ - sudo rm -fr ~/.opam/log && \ - sudo chown -R opam /home/opam -RUN opam config exec -- make -RUN mkdir -p ~/bin && \ - (cp tezos-client tezos-node tezos-protocol-compiler tezos-webclient ~/bin || true) - -WORKDIR $HOME/bin +COPY . /home/opam/tezos +RUN sudo chown -R opam /home/opam/tezos && \ + opam config exec -- make -C tezos -j4 && \ + mkdir bin && \ + cp tezos/tezos-* bin diff --git a/scripts/Dockerfile.build_deps.in b/scripts/Dockerfile.build_deps.in index db976b328..030511316 100644 --- a/scripts/Dockerfile.build_deps.in +++ b/scripts/Dockerfile.build_deps.in @@ -1,11 +1,8 @@ -FROM ocaml/opam:$base_image -COPY install_build_deps.sh /tmp -COPY tezos-deps.opam /tmp/src/tezos-deps.opam -WORKDIR /tmp -RUN opam config exec -- ./install_build_deps.sh pin && rm -fr ~/.opam/log/ -USER root -ENV HOME /home/opam -RUN opam config exec -- ./install_build_deps.sh depext && rm -fr ~/.opam/log/ -RUN apk add libsodium-dev -USER opam -RUN opam config exec -- ./install_build_deps.sh all && rm -fr ~/.opam/log/ +FROM ocaml/opam:alpine-$alpine_version_ocaml-$ocaml_version + +COPY scripts/install_build_deps.sh scripts/version.sh scripts/ +COPY src/tezos-deps.opam src/ +RUN sudo apk add libsodium-dev && \ + opam config exec -- ./scripts/install_build_deps.sh all \ + rm -fr ~/.opam/log/ \ + "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi diff --git a/scripts/create_build_deps_docker_image.sh b/scripts/create_build_deps_docker_image.sh new file mode 100755 index 000000000..144db6b2f --- /dev/null +++ b/scripts/create_build_deps_docker_image.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +set -e +set -x + +script_dir="$(dirname "$(readlink -f "$0")")" +src_dir="$(dirname "$script_dir")" +cd "$src_dir" + +. scripts/version.sh +image_name="${1:-tezos_build_deps}" +image_version="${2:-latest}" + +sed scripts/Dockerfile.build_deps.in \ + -e 's/$alpine_version/'"$alpine_version"'/g' \ + -e 's/$ocaml_version/'"$ocaml_version"'/g' > Dockerfile + +docker build --pull -t "$image_name:$image_version" . + +rm Dockerfile diff --git a/scripts/create_docker_builder.sh b/scripts/create_docker_builder.sh deleted file mode 100755 index 0154dc966..000000000 --- a/scripts/create_docker_builder.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh - -set -x -set -e - -dir="$(mktemp -d)" -src_dir="$(dirname "$(readlink -f "$0")")" - -image_name="${1:-tezos_build}" -base_image="${2:-alpine_ocaml-4.03.0}" -image_version="$3" - -docker pull ocaml/opam:"$base_image" - -cd "$dir" - -cp "$src_dir"/install_build_deps.sh "$dir" -cp "$src_dir"/../src/tezos-deps.opam "$dir" - -cp "$src_dir"/Dockerfile.build_deps.in "$dir" -sed Dockerfile.build_deps.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_deps - -docker build -f Dockerfile.build_deps -t "$image_name:$base_image$image_version" "$dir" diff --git a/scripts/create_docker_image.sh b/scripts/create_docker_image.sh new file mode 100755 index 000000000..eab3e1ee1 --- /dev/null +++ b/scripts/create_docker_image.sh @@ -0,0 +1,41 @@ +#! /bin/sh + +set -e +set -x + +script_dir="$(dirname "$(readlink -f "$0")")" +src_dir="$(dirname "$script_dir")" +cd "$src_dir" + +. scripts/version.sh +image_name="${1:-tezos}" +image_version="${2:-latest}" +build_deps_image_name="$image_name"_build_deps + +echo Building dependencies... +./scripts/create_build_deps_docker_image.sh \ + "$build_deps_image_name" "$image_version" + +cleanup () { + set +e + echo Cleaning up... + [ -z "$tmp_container" ] || docker rm "$tmp_container" + [ -z "$tmp_image" ] || docker rmi "$tmp_image" + rm -rf Dockerfile bin +} +trap cleanup EXIT INT + +sed scripts/Dockerfile.build_bin.in \ + -e 's/$base_name/'"$build_deps_image_name"'/g' \ + -e 's/$base_version/'"$image_version"'/g' > Dockerfile + +echo Building tezos... +tmp_image="$(docker build -q .)" +tmp_container="$(docker run -dit "$tmp_image" true)" + +docker cp "$tmp_container":/home/opam/bin/ bin + +echo Building minimal docker image... +sed scripts/Dockerfile.binaries.in \ + -e 's/$alpine_version/'"$alpine_version"'/g' > Dockerfile +docker build -q -t "$image_name:$image_version" . diff --git a/scripts/create_docker_with_binaries.sh b/scripts/create_docker_with_binaries.sh deleted file mode 100755 index 9f7ee2a2a..000000000 --- a/scripts/create_docker_with_binaries.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/sh - -set -x -set -e - -dir="$(mktemp -d)" -src_dir="$(dirname "$(readlink -f "$0")")" - -image_name="${1:-tezos_build}" -base_image="${2:-alpine_ocaml-4.03.0}" -image_version="$3" - -tezos_build_img="tezos_build:$base_image" - -if ! (docker images | grep -- "^tezos_build \+$base_image "); then - echo "Docker image not found: $tezos_build_img" >&2 - echo "Aborting" >&2 - exit 1 - fi - -cd "$dir" - -git clone "$src_dir"/.. "$dir"/tezos -rm -fr "$dir"/tezos/.git - -cp "$src_dir"/Dockerfile.build_bin.in "$dir" -sed Dockerfile.build_bin.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_bin - -docker build -f Dockerfile.build_bin -t "tezos_build_bin:$base_image$image_version" "$dir" - -mkdir -p "$dir"/built-bin -docker run -i --rm -v "$dir"/built-bin:/built-bin "tezos_build_bin:$base_image$image_version" /bin/bash << EOF -sudo cp -v /home/opam/bin/tezos-* /built-bin/ -sudo chown opam:nogroup /built-bin/tezos-* -sudo chmod a+rwx /built-bin/tezos-* -EOF - -cp "$src_dir"/Dockerfile.binaries.in "$dir" -sed Dockerfile.binaries.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.binaries - -docker build -f Dockerfile.binaries -t "tezos_binaries:$base_image$image_version" "$dir" diff --git a/scripts/install_build_deps.sh b/scripts/install_build_deps.sh index ba6694563..c3562b864 100755 --- a/scripts/install_build_deps.sh +++ b/scripts/install_build_deps.sh @@ -1,10 +1,14 @@ #! /bin/sh -OCAML_VERSION=4.03.0 -if [ "$(ocaml -vnum)" != "$OCAML_VERSION" ]; then +script_dir="$(dirname "$(readlink -f "$0")")" +src_dir="$(dirname "$script_dir")" + +. "$script_dir/version.sh" + +if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then echo ; echo " Unexpected compiler version ($(ocaml -vnum))"; - echo " You should use ocaml-$OCAML_VERSION."; + echo " You should use ocaml-$ocaml_version."; echo ; exit 1; fi diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100644 index 000000000..8830b85dc --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +alpine_version=3.5 +ocaml_version=4.03.0 diff --git a/src/Makefile b/src/Makefile index 2617fc8af..758de0561 100644 --- a/src/Makefile +++ b/src/Makefile @@ -65,7 +65,7 @@ node/updater/proto_environment.cmi: \ @echo OCAMLOPT ${TARGET} $@ @$(OCAMLOPT) -nopervasives -nostdlib -opaque -I tmp -I node/updater -c $< -clean:: +partial-clean:: rm -f node/updater/proto_environment.mli rm -f node/updater/environment_gen @@ -91,7 +91,7 @@ compiler/embedded_cmis.cmx: compiler/embedded_cmis.cmi compiler/embedded_cmis.ml: ${EMBEDDED_PROTOCOL_LIB_CMIS} @echo OCAMLRES ${TARGET} $(notdir $@) @$(OCAMLRES) -format ocaml -o $@ $^ -clean:: +partial-clean:: rm -f compiler/embedded_cmis.ml rm -rf tmp @@ -368,7 +368,7 @@ ${TZNODE}: minutils.cmxa utils.cmxa compiler.cmxa node.cmxa ${EMBEDDED_NODE_PROT @echo LINK $(notdir $@) @${OCAMLOPT} -linkall -linkpkg ${OCAMLFLAGS} -o $@ $^ -clean:: +partial-clean:: -rm -f proto/*.cm* proto/*.a ## Embedded protocol modules @@ -399,8 +399,6 @@ proto/client_embedded_proto_%.cmxa: \ $(addprefix -I , ${CLIENT_PROTO_INCLUDES}) \ $@ proto/$* - - clean:: rm -f ${TZNODE} @@ -529,7 +527,7 @@ client/embedded/webclient_%.cmx: \ $$(shell find client/embedded/%/webclient/static/) @$(MAKE) -C client/embedded/$* ../webclient_$*.cmx -clean:: +partial-clean:: -for d in $$(ls -d client/embedded/*/) ; do make clean -C $$d ; done -rm -f client/embedded/*.cm* client/embedded/*.o @@ -601,8 +599,9 @@ clean:: ## Cleaning -.PHONY: clean -clean:: +.PHONY: clean partial-clean +clean:: partial-clean +partial-clean:: -find \( -name \*.cm\* -or -name \*.cmp -or -name \*.out -or -name \*~ -or -name \*.o -or -name \*.a \) -delete ## Dependencies @@ -652,6 +651,6 @@ compiler/tezos_compiler.cmo compiler/tezos_compiler.cmx: \ @echo OCAMLDEP ${TARGET} $(notdir $^) @$(OCAMLDEP) $(INCLUDES) $^ > $@ -clean:: +partial-clean:: -rm -f .depend -find \( -name \*.mli.deps -or -name \*.ml.deps \) -delete