Docker: integration in CI

And minor fix:

- introduce configuration file for OCaml and alpine versions
- add missing `libssl1.0` in docker image
- remove the split call to `install_build_deps` (not required anymore)
- rename scripts

.dockerignore

@@ -0,0 +1,67 @@
+.git
+
+tezos-node
+tezos-protocol-compiler
+tezos-client
+tezos-webclient
+tezos-attacker
+
+src/Makefile.local
+
+src/webclient_static.ml
+src/.depend
+
+src/node/updater/environment_gen
+src/node/updater/proto_environment.mli
+src/compiler/embedded_cmis.ml
+
+src/proto/**/_tzbuild
+src/proto/register_client_*.ml
+src/client/embedded/**/_tzbuild
+
+src/client/embedded/demo/.depend
+
+src/client/embedded/bootstrap/.depend
+src/client/embedded/bootstrap/concrete_lexer.ml
+src/client/embedded/bootstrap/concrete_parser.ml
+src/client/embedded/bootstrap/concrete_parser.mli
+src/client/embedded/bootstrap/webclient_proto_static.ml
+src/client/embedded/bootstrap/main.byte
+src/client/embedded/bootstrap/webclient_static/main.js
+src/client/embedded/bootstrap/webclient/browser/main.byte
+src/client/embedded/bootstrap/webclient/static/main.js
+src/client/embedded/bootstrap/webclient/webclient_proto_static.ml
+
+test/.depend
+test/reports
+
+test/test-store
+test/test-state
+test/test-context
+test/test-basic
+test/test-data-encoding
+test/test-p2p-io-scheduler
+test/test-p2p-connection
+test/test-p2p-connection-pool
+test/LOG
+
+**/*~
+**/\#*\#
+
+**/*.[oa]
+**/*.so
+**/*~
+**/*.cm[iaoxt]
+**/*.cmti
+**/*.cmxa
+**/*.cmxs
+**/*.cmp
+**/*.mli.deps
+**/*.ml.deps
+**/*.mli.deps.byte
+**/*.ml.deps.byte
+
+**/bisect*.out
+
+**/*.rej
+**/*.orig

.gitlab-ci.yml

@@ -1,10 +1,10 @@
 variables:
-  ocaml_image: alpine_ocaml-4.03.0
+  image_name: ocp_tezos
-  build_image_name: ocp_tezos_build
+  image_id: ${CI_BUILD_REF_NAME}
+  build_image_name: ${image_name}_build_deps
   build_image_id: ${CI_BUILD_REF}
-  build_image: ${build_image_name}:${ocaml_image}
 
-image: ${build_image}_${build_image_id}
+image: ${build_image_name}:${build_image_id}
 
 stages:
   - build_deps
@@ -21,8 +21,8 @@ build_deps:
   tags:
     - docker_builder
   script:
-    - ./scripts/create_docker_builder.sh
+    - ./scripts/create_build_deps_docker_image.sh
-        ${build_image_name} ${ocaml_image} "_${build_image_id}"
+        ${build_image_name} ${build_image_id}
 
 build:
   stage: build
@@ -137,15 +137,29 @@ test:p2p-connection-pool:
     - build
     - build:test
 
-expurge:
+publish:docker:
-  stage: expurge
+  stage: publish
+  image: ocp:docker
+  tags:
+    - docker_builder
+  script:
+    - ./scripts/create_docker_image.sh ${image_name} ${build_image_id}
+    - docker tag ${image_name}:${build_image_id}
+        "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+    - docker push "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+    - docker rmi "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+
+publish:github:
+  image: ocaml/opam
+  stage: publish
   tags:
     - tezos_builder
   only:
     - master@tezos/tezos
   script:
     - echo "${CI_KH}" > ~/.ssh/known_hosts
-    - echo "${CI_PK_GITLAB}" > ~/.ssh/id_rsa
+    - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
     - chmod 400 ~/.ssh/id_rsa
     - rm -fr .git/refs/original
     - git filter-branch --prune-empty --index-filter
@@ -162,23 +176,7 @@ expurge:
                export GIT_AUTHOR_EMAIL="contact@tezos.com" ;
              fi'
          HEAD
-    - git tag ${CI_BUILD_REF}_expurged
+    - git push git@github.com:OCamlPro/tezos.git -f HEAD:master
-    - git push git@gitlab.ocamlpro.com:${CI_PROJECT_PATH}.git
-               -f --tags HEAD:master-expurged
-
-publish:github:
-  image: ocaml/opam:${ocaml_image}
-  stage: publish
-  tags:
-    - tezos_builder
-  only:
-    - master@tezos/tezos
-  script:
-    - echo "${CI_KH}" > ~/.ssh/known_hosts
-    - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
-    - chmod 400 ~/.ssh/id_rsa
-    - git reset ${CI_BUILD_REF}_expurged
-    - git push git@github.com:tezos/tezos.git -f HEAD:master
 
 cleanup:
   stage: cleanup
@@ -186,7 +184,7 @@ cleanup:
   tags:
     - docker_builder
   script:
-    - docker tag ${build_image}_${build_image_id}
+    - docker tag ${build_image_name}:${build_image_id}
-                 ${build_image}_${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
+                 ${build_image_name}:${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
-    - docker rmi ${build_image}_${build_image_id}
+    - docker rmi ${build_image_name}:${build_image_id}
   when: always

Makefile

@@ -6,9 +6,16 @@ clean:
 	${MAKE} -C src clean
 	${MAKE} -C test clean
 
+partial-clean:
+	${MAKE} -C src partial-clean
+	${MAKE} -C test clean
+
 .PHONY: test
 test:
 	${MAKE} -C test
 
 build-deps:
 	@./scripts/install_build_deps.sh all
+
+docker-image:
+	@./scripts/create_docker_image.sh

scripts/Dockerfile.binaries.in

@@ -1,20 +1,24 @@
-FROM alpine:3.4
+FROM alpine:$alpine_version
-# FIXME: I'm currently guessing the version of alpine that the opam image we've
-# built the tezos binaries on is based on. if that becomes newer (e.g. alpine
-# 3.5), the binaries built on that version of alpine could (will) depend on
-# library symbols that won't be in alpine 3.4
 
-LABEL distro_style="apk" distro="alpine" distro_long="alpine-3.4" arch="x86_64" operatingsystem="linux"
+LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux"
 
 RUN apk update && \
     apk upgrade && \
-    apk add sudo bash libsodium gmp && \
+    apk add sudo bash libssl1.0 libsodium gmp && \
+    rm -f /var/cache/apk/* && \
     adduser -S tezos && \
     echo 'tezos ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/tezos && \
     chmod 440 /etc/sudoers.d/tezos && \
     chown root:root /etc/sudoers.d/tezos && \
-    sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
+    sed -i 's/^Defaults.*requiretty//g' /etc/sudoers
 USER tezos
 
-ADD built-bin /usr/local/bin
+COPY . /home/tezos
 WORKDIR /home/tezos
+
+RUN sudo chown root:root bin/* && \
+    sudo chmod a+rx bin/* && \
+    sudo mv bin/* /usr/local/bin && \
+    rmdir bin
+
+ENTRYPOINT [ "/bin/bash" ]

scripts/Dockerfile.build_bin.in

@@ -1,13 +1,7 @@
-FROM tezos_build:$base_image
+FROM $base_name:$base_version
 
-ADD tezos /home/opam/tezos
+COPY . /home/opam/tezos
-ENV HOME /home/opam
+RUN sudo chown -R opam /home/opam/tezos && \
-WORKDIR $HOME/tezos
+    opam config exec -- make -C tezos -j4 && \
-RUN sudo HOME="$HOME" opam config exec -- make clean && \
+    mkdir bin && \
-    sudo rm -fr ~/.opam/log && \
+    cp tezos/tezos-* bin
-    sudo chown -R opam /home/opam
-RUN opam config exec -- make
-RUN mkdir -p ~/bin && \
-    (cp tezos-client tezos-node tezos-protocol-compiler tezos-webclient ~/bin || true)
-
-WORKDIR $HOME/bin

scripts/Dockerfile.build_deps.in

@@ -1,11 +1,8 @@
-FROM ocaml/opam:$base_image
+FROM ocaml/opam:alpine-$alpine_version_ocaml-$ocaml_version
-COPY install_build_deps.sh /tmp
+
-COPY tezos-deps.opam /tmp/src/tezos-deps.opam
+COPY scripts/install_build_deps.sh scripts/version.sh scripts/
-WORKDIR /tmp
+COPY src/tezos-deps.opam src/
-RUN opam config exec -- ./install_build_deps.sh pin    && rm -fr ~/.opam/log/
+RUN sudo apk add libsodium-dev && \
-USER root
+    opam config exec -- ./scripts/install_build_deps.sh all \
-ENV HOME /home/opam
+    rm -fr ~/.opam/log/ \
-RUN opam config exec -- ./install_build_deps.sh depext && rm -fr ~/.opam/log/
+           "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi
-RUN apk add libsodium-dev
-USER opam
-RUN opam config exec -- ./install_build_deps.sh all    && rm -fr ~/.opam/log/

scripts/create_build_deps_docker_image.sh

@@ -0,0 +1,20 @@
+#! /bin/sh
+
+set -e
+set -x
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+image_name="${1:-tezos_build_deps}"
+image_version="${2:-latest}"
+
+sed scripts/Dockerfile.build_deps.in \
+    -e 's/$alpine_version/'"$alpine_version"'/g' \
+    -e 's/$ocaml_version/'"$ocaml_version"'/g' > Dockerfile
+
+docker build --pull -t "$image_name:$image_version" .
+
+rm Dockerfile

scripts/create_docker_builder.sh

@@ -1,23 +0,0 @@
-#! /bin/sh
-
-set -x
-set -e
-
-dir="$(mktemp -d)"
-src_dir="$(dirname "$(readlink -f "$0")")"
-
-image_name="${1:-tezos_build}"
-base_image="${2:-alpine_ocaml-4.03.0}"
-image_version="$3"
-
-docker pull ocaml/opam:"$base_image"
-
-cd "$dir"
-
-cp "$src_dir"/install_build_deps.sh "$dir"
-cp "$src_dir"/../src/tezos-deps.opam "$dir"
-
-cp "$src_dir"/Dockerfile.build_deps.in "$dir"
-sed Dockerfile.build_deps.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_deps
-
-docker build -f Dockerfile.build_deps -t "$image_name:$base_image$image_version" "$dir"

scripts/create_docker_image.sh

@@ -0,0 +1,41 @@
+#! /bin/sh
+
+set -e
+set -x
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+image_name="${1:-tezos}"
+image_version="${2:-latest}"
+build_deps_image_name="$image_name"_build_deps
+
+echo Building dependencies...
+./scripts/create_build_deps_docker_image.sh \
+    "$build_deps_image_name" "$image_version"
+
+cleanup () {
+    set +e
+    echo Cleaning up...
+    [ -z "$tmp_container" ] || docker rm "$tmp_container"
+    [ -z "$tmp_image" ] || docker rmi "$tmp_image"
+    rm -rf Dockerfile bin
+}
+trap cleanup EXIT INT
+
+sed scripts/Dockerfile.build_bin.in \
+    -e 's/$base_name/'"$build_deps_image_name"'/g' \
+    -e 's/$base_version/'"$image_version"'/g'  > Dockerfile
+
+echo Building tezos...
+tmp_image="$(docker build -q .)"
+tmp_container="$(docker run -dit "$tmp_image" true)"
+
+docker cp "$tmp_container":/home/opam/bin/ bin
+
+echo Building minimal docker image...
+sed scripts/Dockerfile.binaries.in \
+    -e 's/$alpine_version/'"$alpine_version"'/g' > Dockerfile
+docker build -q -t "$image_name:$image_version" .

scripts/create_docker_with_binaries.sh

@@ -1,41 +0,0 @@
-#! /bin/sh
-
-set -x
-set -e
-
-dir="$(mktemp -d)"
-src_dir="$(dirname "$(readlink -f "$0")")"
-
-image_name="${1:-tezos_build}"
-base_image="${2:-alpine_ocaml-4.03.0}"
-image_version="$3"
-
-tezos_build_img="tezos_build:$base_image"
-
-if ! (docker images | grep -- "^tezos_build \+$base_image "); then
-  echo "Docker image not found: $tezos_build_img" >&2
-  echo "Aborting" >&2
-  exit 1
-  fi
-
-cd "$dir"
-
-git clone "$src_dir"/.. "$dir"/tezos
-rm -fr "$dir"/tezos/.git
-
-cp "$src_dir"/Dockerfile.build_bin.in "$dir"
-sed Dockerfile.build_bin.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_bin
-
-docker build -f Dockerfile.build_bin -t "tezos_build_bin:$base_image$image_version" "$dir"
-
-mkdir -p "$dir"/built-bin
-docker run -i --rm -v "$dir"/built-bin:/built-bin "tezos_build_bin:$base_image$image_version" /bin/bash << EOF
-sudo cp -v /home/opam/bin/tezos-* /built-bin/
-sudo chown opam:nogroup /built-bin/tezos-*
-sudo chmod a+rwx /built-bin/tezos-*
-EOF
-
-cp "$src_dir"/Dockerfile.binaries.in "$dir"
-sed Dockerfile.binaries.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.binaries
-
-docker build -f Dockerfile.binaries -t "tezos_binaries:$base_image$image_version" "$dir"

scripts/install_build_deps.sh

@@ -1,10 +1,14 @@
 #! /bin/sh
 
-OCAML_VERSION=4.03.0
+script_dir="$(dirname "$(readlink -f "$0")")"
-if [ "$(ocaml -vnum)" != "$OCAML_VERSION" ]; then
+src_dir="$(dirname "$script_dir")"
+
+. "$script_dir/version.sh"
+
+if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then
   echo ;
   echo "   Unexpected compiler version ($(ocaml -vnum))";
-  echo "   You should use ocaml-$OCAML_VERSION.";
+  echo "   You should use ocaml-$ocaml_version.";
   echo ;
   exit 1;
 fi

scripts/version.sh

@@ -0,0 +1,4 @@
+#! /bin/sh
+
+alpine_version=3.5
+ocaml_version=4.03.0

src/Makefile

@@ -65,7 +65,7 @@ node/updater/proto_environment.cmi: \
 	@echo OCAMLOPT ${TARGET} $@
 	@$(OCAMLOPT) -nopervasives -nostdlib -opaque -I tmp -I node/updater -c $<
 
-clean::
+partial-clean::
 	rm -f node/updater/proto_environment.mli
 	rm -f node/updater/environment_gen
 
@@ -91,7 +91,7 @@ compiler/embedded_cmis.cmx: compiler/embedded_cmis.cmi
 compiler/embedded_cmis.ml: ${EMBEDDED_PROTOCOL_LIB_CMIS}
 	@echo OCAMLRES ${TARGET} $(notdir $@)
 	@$(OCAMLRES) -format ocaml -o $@ $^
-clean::
+partial-clean::
 	rm -f compiler/embedded_cmis.ml
 	rm -rf tmp
 
@@ -368,7 +368,7 @@ ${TZNODE}: minutils.cmxa utils.cmxa compiler.cmxa node.cmxa ${EMBEDDED_NODE_PROT
 	@echo LINK $(notdir $@)
 	@${OCAMLOPT} -linkall -linkpkg ${OCAMLFLAGS} -o $@ $^
 
-clean::
+partial-clean::
 	-rm -f proto/*.cm* proto/*.a
 
 ## Embedded protocol modules
@@ -399,8 +399,6 @@ proto/client_embedded_proto_%.cmxa: \
 	   $(addprefix -I , ${CLIENT_PROTO_INCLUDES}) \
 	   $@ proto/$*
 
-
-
 clean::
 	rm -f ${TZNODE}
 
@@ -529,7 +527,7 @@ client/embedded/webclient_%.cmx: \
   $$(shell find client/embedded/%/webclient/static/)
 	@$(MAKE) -C client/embedded/$* ../webclient_$*.cmx
 
-clean::
+partial-clean::
 	-for d in $$(ls -d client/embedded/*/) ; do make clean -C $$d ; done
 	-rm -f client/embedded/*.cm* client/embedded/*.o
 
@@ -601,8 +599,9 @@ clean::
 
 ## Cleaning
 
-.PHONY: clean
+.PHONY: clean partial-clean
-clean::
+clean:: partial-clean
+partial-clean::
 	-find \( -name \*.cm\* -or -name \*.cmp -or -name \*.out -or -name \*~ -or -name \*.o -or -name \*.a \) -delete
 
 ## Dependencies
@@ -652,6 +651,6 @@ compiler/tezos_compiler.cmo compiler/tezos_compiler.cmx: \
 	@echo OCAMLDEP ${TARGET} $(notdir $^)
 	@$(OCAMLDEP) $(INCLUDES) $^ > $@
 
-clean::
+partial-clean::
 	-rm -f .depend
 	-find \( -name \*.mli.deps -or -name \*.ml.deps \) -delete