gitlab CI refactoring

- Docker files and build scripts
    + add git, bash and patch to the docker base definition (Dockerfile.base)
    + build leveldb once and for all
      (scripts/create_docker_image.leveldb_deps.sh)
    + add --depth 1 to all git clone calls
    + remove sudo, openssh and rsync from base images
    + add --no-cache to all apk calls
    + merge dockerfiles in the sh scripts

.dockerignore

@@ -1,13 +1,21 @@
 
+## /!\ /!\ Update .dockerignore accordingly /!\ /!\
+
+.DS_Store
+__pycache__
+
 _build
 **/*.install
 
 tezos-node
 tezos-protocol-compiler
 tezos-client
+tezos-admin-client
 
 scripts/opam-test-all.sh.DONE
 
+docs/introduction/readme.rst
+
 **/.merlin
 
 **/*~
@@ -18,6 +26,8 @@ scripts/opam-test-all.sh.DONE
 
 _opam
 
+## Not in .gitignore
+
 .git
 .gitignore
 .gitlab-ci.yml

.gitignore

@@ -1,9 +1,9 @@
 
+## /!\ /!\ Update .gitignore accordingly /!\ /!\
+
 .DS_Store
 __pycache__
 
-/docs/introduction/readme.rst
-
 _build
 *.install
 
@@ -14,6 +14,8 @@ _build
 
 /scripts/opam-test-all.sh.DONE
 
+/docs/introduction/readme.rst
+
 .merlin
 
 *~
@@ -23,4 +25,8 @@ _build
 *.orig
 
 /_opam
+
+## Not in .dockerignore
+
 /Dockerfile
+/_apk

.gitlab-ci.yml

@@ -1,3 +1,4 @@
+
 variables:
   public_docker_image: docker.io/tezos/tezos
 
@@ -22,9 +23,7 @@ stages:
     - docker info
     - docker login -u gitlab-ci-token -p "$CI_BUILD_TOKEN" "${CI_REGISTRY}"
   tags:
-    - safe_docker
+    - docker
-
-## Prepare docker image with precompiled OCaml/opam
 
 prepare:opam:
   <<: *dind_definition
@@ -34,30 +33,14 @@ prepare:opam:
   before_script:
     - . ./scripts/version.sh
   script:
-    - ./scripts/create_docker_image.alpine.sh
+    - mkdir -p ~/.docker
-    - mkdir ~/.docker || true
     - echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ;
+    - ./scripts/create_docker_image.opam.sh
     - docker push "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
     - docker tag "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
                  "tezos/opam:latest"
     - docker push "tezos/opam:latest"
 
-prepare:opam2:
-  <<: *dind_definition
-  stage: prepare
-  only:
-    - schedules
-  before_script:
-    - . ./scripts/version.sh
-  script:
-    - ./scripts/create_docker_image.alpine.opam2.sh
-    - mkdir ~/.docker || true
-    - echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ;
-    - docker push "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}"
-    - docker tag "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}"
-                 "tezos/opam2:latest"
-    - docker push "tezos/opam2:latest"
-
 ## Tezos
 
 build:
@@ -125,43 +108,36 @@ test:proto_alpha:transaction:
   <<: *test_definition
   script:
     - jbuilder build @test/proto_alpha/runtest_transaction
-  retry: 1
 
 test:proto_alpha:origination:
   <<: *test_definition
   script:
     - jbuilder build @test/proto_alpha/runtest_origination
-  retry: 1
 
 test:proto_alpha:endorsement:
   <<: *test_definition
   script:
     - jbuilder build @test/proto_alpha/runtest_endorsement
-  retry: 1
 
 test:proto_alpha:vote:
   <<: *test_definition
   script:
     - jbuilder build @test/proto_alpha/runtest_vote
-  retry: 1
 
 test:basic.sh:
   <<: *test_definition
   script:
     - jbuilder build @test/runtest_basic.sh
-  retry: 1
 
 test:contracts.sh:
   <<: *test_definition
   script:
     - jbuilder build @test/runtest_contracts.sh
-  retry: 1
 
 test:multinode.sh:
   <<: *test_definition
   script:
     - jbuilder build @test/runtest_multinode.sh
-  retry: 1
 
 test:proto:sandbox:
   <<: *test_definition

scripts/Dockerfile.alpine.in

@@ -1,46 +0,0 @@
-## Based on https://github.com/ocaml/opam-dockerfiles
-
-FROM alpine:$alpine_version
-
-RUN apk update && apk upgrade && \
-    apk add build-base snappy-dev abuild alpine-sdk \
-            openssh bash nano ncurses-dev rsync xz m4 \
-            opam aspcud \
-            gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \
-            libsodium-dev && \
-    rm -f /var/cache/apk/* && \
-    adduser -S opam && \
-    adduser opam abuild && \
-    echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
-    chmod 440 /etc/sudoers.d/opam && \
-    chown root:root /etc/sudoers.d/opam && \
-    sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
-
-USER opam
-
-WORKDIR /home/opam
-
-RUN mkdir .ssh && \
-    chmod 700 .ssh && \
-    git config --global user.email "docker@example.com" && \
-    git config --global user.name "Docker CI" && \
-    sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \
-    sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \
-    sudo -u opam sh -c "opam install -y depext travis-opam"
-
-COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD
-RUN sudo chown -R opam leveldb-1.18
-
-RUN sudo apk update && \
-    abuild-keygen -ai && \
-    cd leveldb-1.18 && \
-    abuild checksum && abuild -r && \
-    cd .. && \
-    sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \
-    sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \
-    rm -rf leveldb-1.18 packages && \
-    sudo rm -f /var/cache/apk/*
-
-ENTRYPOINT [ "opam", "config", "exec", "--" ]
-
-CMD [ "sh" ]

scripts/Dockerfile.alpine.opam2.in

@@ -1,60 +0,0 @@
-## Based on https://github.com/ocaml/opam-dockerfiles
-
-FROM alpine:$alpine_version
-
-RUN apk update && apk upgrade && \
-    apk add build-base snappy-dev abuild alpine-sdk \
-            openssh bash nano ncurses-dev rsync xz m4 \
-            ocaml \
-            gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \
-            libsodium-dev && \
-    rm -f /var/cache/apk/* && \
-    adduser -S opam && \
-    adduser opam abuild && \
-    echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
-    chmod 440 /etc/sudoers.d/opam && \
-    chown root:root /etc/sudoers.d/opam && \
-    sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
-
-USER opam
-
-WORKDIR /home/opam
-
-RUN mkdir ~/.ssh && \
-    chmod 700 ~/.ssh && \
-    git config --global user.email "docker@example.com" && \
-    git config --global user.name "Docker CI" && \
-    cd /home/opam && \
-    git clone -b "$opam_tag" git://github.com/ocaml/opam  && \
-    cd /home/opam/opam && \
-    ./configure --prefix /usr && \
-    make lib-ext && \
-    make opam && \
-    sudo cp -L opam /usr/bin/opam && \
-    cd /home/opam && \
-    sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \
-    cd opam-repository && \
-    sudo -u opam sh -c "cd /home/opam/opam-repository && opam admin upgrade && git checkout -b v2 && git add . && git commit -a -m 'opam admin upgrade'" && \
-    sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \
-    sudo -u opam sh -c "opam install -y depext travis-opam" && \
-    cd /home/opam/opam && \
-    make opam-installer && \
-    sudo cp -L opam-installer /usr/bin/opam-installer && \
-    rm -r /home/opam/opam
-
-COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD
-RUN sudo chown -R opam leveldb-1.18
-
-RUN sudo apk update && \
-    abuild-keygen -ai && \
-    cd leveldb-1.18 && \
-    abuild checksum && abuild -r && \
-    cd .. && \
-    sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \
-    sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \
-    rm -rf leveldb-1.18 packages && \
-    sudo rm -f /var/cache/apk/*
-
-ENTRYPOINT [ "opam", "config", "exec", "--" ]
-
-CMD [ "sh" ]

scripts/Dockerfile.build.in

@@ -1,8 +0,0 @@
-FROM $base_image
-
-COPY . tezos
-
-RUN sudo chown -R opam /home/opam/tezos && \
-    cd tezos && \
-    opam config exec -- jbuilder build @install && \
-    opam config exec -- jbuilder install

scripts/Dockerfile.build_deps.in

@@ -1,16 +0,0 @@
-FROM $base_image
-
-## /!\ /!\ /!\ /!\/!\ /!\ /!\
-## /!\ Don't forget fu update the variables 'dependencies' in
-## /!\ `create_docker_image.build_deps.sh` when modifyiing the
-## /!\  'COPY'edfiles.
-## /!\ /!\ /!\ /!\/!\ /!\ /!\
-
-$copy_files
-
-RUN sudo apk update && \
-    opam config exec -- ./tezos/scripts/install_build_deps.sh && \
-    opam install ocp-indent && \
-    sudo rm -f /var/cache/apk/* && \
-    rm -fr ~/.opam/log/ && \
-    rm -fr "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi

scripts/create_apk.leveldb.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+
+image_name="${1:-tezos/leveldb}"
+
+cat <<EOF > Dockerfile
+FROM andyshinn/alpine-abuild:v4
+
+ENV PACKAGER "Tezos <team@tezos.com>"
+WORKDIR /home/builder/
+RUN abuild-keygen -a -i
+COPY scripts/leveldb-$leveldb_version.APKBUILD APKBUILD
+RUN abuilder -r
+EOF
+
+echo
+echo "### Building leveldb..."
+echo
+
+docker build -t $image_name:$leveldb_version .
+rm Dockerfile
+
+mkdir -p _apk
+
+docker create --name tezos.leveldb.tmp $image_name:$leveldb_version
+docker cp -L tezos.leveldb.tmp:/etc/apk/keys _apk/
+docker cp -L tezos.leveldb.tmp:/packages _apk
+docker rm tezos.leveldb.tmp

scripts/create_docker_image.alpine.opam2.sh

@@ -1,34 +0,0 @@
-#! /bin/sh
-
-set -e
-
-script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
-src_dir="$(dirname "$script_dir")"
-cd "$src_dir"
-
-. scripts/version.sh
-image_name="${1:-tezos/opam2}"
-image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
-
-cleanup () {
-    set +e
-    echo Cleaning up...
-    rm -rf Dockerfile
-}
-trap cleanup EXIT INT
-
-#opam_tag=2.0.0-beta5
-opam_tag=master
-
-sed scripts/Dockerfile.alpine.opam2.in \
-    -e 's|$alpine_version|'"$alpine_version"'|g' \
-    -e 's|$ocaml_version|'"$ocaml_version"'|g' \
-    -e 's|$opam_tag|'"$opam_tag"'|g' > Dockerfile
-
-echo
-echo "### Building base image..."
-echo
-
-docker build --pull -t "$image_name:$image_version" .
-
-rm Dockerfile

scripts/create_docker_image.alpine.sh

@@ -1,30 +0,0 @@
-#! /bin/sh
-
-set -e
-
-script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
-src_dir="$(dirname "$script_dir")"
-cd "$src_dir"
-
-. scripts/version.sh
-image_name="${1:-tezos/opam}"
-image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
-
-cleanup () {
-    set +e
-    echo Cleaning up...
-    rm -rf Dockerfile
-}
-trap cleanup EXIT INT
-
-sed scripts/Dockerfile.alpine.in \
-    -e 's|$alpine_version|'"$alpine_version"'|g' \
-    -e 's|$ocaml_version|'"$ocaml_version"'|g' > Dockerfile
-
-echo
-echo "### Building base image..."
-echo
-
-docker build --pull -t "$image_name:$image_version" .
-
-rm Dockerfile

scripts/create_docker_image.build.sh

@@ -18,15 +18,30 @@ cleanup () {
 }
 trap cleanup EXIT INT
 
-sed -e 's|$base_image|'"$build_deps_image_name"'|g' \
+cat <<EOF > Dockerfile
-    scripts/Dockerfile.build.in > Dockerfile
+FROM $build_deps_image_name
+
+COPY . tezos
+USER root
+RUN chown -R opam /home/opam/tezos
+
+# build tezos and friends
+USER opam
+
+RUN cd tezos && \
+    opam config exec -- jbuilder build @install && \
+    opam config exec -- jbuilder install
+
+USER root
+
+ENTRYPOINT [ "/sbin/su-exec", "opam", "opam", "config", "exec", "--" ]
+EOF
 
 echo
 echo "### Building tezos..."
 echo
 
 docker build -t "$image_name:$image_version" .
-
 rm Dockerfile
 
 echo

scripts/create_docker_image.build_deps.sh

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 set -e
 
@@ -13,28 +13,40 @@ cached_image="${3:-}"
 
 base_image="tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
 if ! docker pull "$base_image" ; then
-    ./scripts/create_docker_image.alpine.sh
+    ./scripts/create_docker_image.alpine.opam2.sh
 fi
 
 cleanup () {
     set +e
     echo Cleaning up...
-    rm -rf Dockerfile
+    rm -f Dockerfile opams.tar.gz scripts.tar.gz
 }
 trap cleanup EXIT INT
 
-opam_files=$(find -name \*.opam | sort)
+dependencies="scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh"
-dependencies="$opam_files scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh Dockerfile"
+tar czvf scripts.tar.gz $dependencies
 
-for file in $dependencies; do
+opams=$(find -name \*.opam -type f)
-    if [ "$file" = Dockerfile ]; then continue; fi
+tar czvf opams.tar.gz $opams
-    copy_files="$copy_files\nCOPY $file ./tezos/$file"
-done
 
-sed -e 's|$base_image|'"$base_image"'|g' \
+cat <<EOF >Dockerfile
-    -e 's|$ocaml_version|'"$ocaml_version"'|g' \
+FROM $base_image
-    -e 's|$copy_files|'"$copy_files"'|g' \
+
-    scripts/Dockerfile.build_deps.in > Dockerfile
+# these two archives are created in the file
+# scripts/create_docker_image.build_deps.sh and removed
+# automatically after
+ADD opams.tar.gz tezos/
+ADD scripts.tar.gz tezos/
+
+USER opam
+
+RUN opam config exec -- ./tezos/scripts/install_build_deps.sh
+ENV OPAMYES=yes
+RUN opam config exec -- opam install ocp-indent && \
+  rm -fr ~/.opam/log/ && \
+  rm -fr "\$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi
+
+EOF
 
 ## Lookup for for prebuilt dependencies...
 dependencies_sha1=$(docker inspect --format="{{ .RootFS.Layers }}" --type=image $base_image | sha1sum - $dependencies | sha1sum | tr -d ' -')
@@ -54,12 +66,12 @@ if [ ! -z "$cached_image" ]; then
 fi
 
 echo
-echo "### Building tezos dependencies..."
+echo "### Building tezos dependencies... $image_name:$image_version"
 echo
 
-docker build -t "$image_name:$image_version" .
+docker build --pull -t "$image_name:$image_version" .
 
-rm Dockerfile
+rm -f Dockerfile opams.tar.gz scripts.tar.gz
 
 echo
 echo "### Succesfully build docker image: $image_name:$image_version"

scripts/create_docker_image.minimal.sh

@@ -18,10 +18,20 @@ cleanup () {
 }
 trap cleanup EXIT INT
 
-docker run -dit --rm  --volume $(pwd)/bin:/home/opam/bin "$build_image_name" \
+# assume $build_image_name has already been created
-       /bin/sh -c "sudo cp -L /home/opam/tezos/_build/install/default/bin/* /home/opam/bin"
+mkdir -p _docker_build_result
+docker create --name tmp1 $build_image_name
+docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-client _docker_build_result/
+docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-node _docker_build_result/
+docker rm tmp1
 
-ls bin
+# assume tezos/leveldb has already been created
+mkdir -p _docker_build_result/leveldb
+mkdir -p _docker_build_result/keys
+docker create --name tmp1 tezos/leveldb
+docker cp -L tmp1:/etc/apk/keys _docker_build_result/
+docker cp -L tmp1:/packages _docker_build_result/
+docker rm tmp1
 
 echo
 echo "### Building minimal docker image..."
@@ -35,4 +45,4 @@ echo
 echo "### Succesfully build docker image: $image_name:$image_version"
 echo
 
-rm -r bin
+rm -Rf _docker_build_result

scripts/create_docker_image.opam.sh

@@ -0,0 +1,88 @@
+#! /bin/sh
+
+set -e
+
+script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+
+image_name="${1:-tezos/opam}"
+image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
+
+cleanup () {
+    set +e
+    echo Cleaning up...
+    rm -rf Dockerfile
+}
+trap cleanup EXIT INT
+
+apk_keys="_apk/keys/"
+apk_packages="_apk/packages/home/x86_64/"
+
+if ! ( [ -f "$apk_packages/leveldb-$leveldb_version-r0.apk" ] && \
+       [ -f "$apk_packages/leveldb-dev-$leveldb_version-r0.apk" ] && \
+       [ -d "$apk_keys" ] ) ; then
+  ./scripts/create_apk.leveldb.sh
+fi
+
+opam_tag=2.0.0-rc
+
+cat <<EOF > Dockerfile
+FROM alpine:$alpine_version
+
+RUN apk update && \
+    apk upgrade && \
+    apk --no-cache add su-exec build-base \
+            bash ncurses-dev xz m4 git ca-certificates wget \
+            gmp-dev libev-dev libressl-dev pcre-dev perl zlib-dev libsodium-dev && \
+    update-ca-certificates && \
+    rm -f /var/cache/apk/* && \
+    adduser -S opam
+
+COPY $apk_keys /etc/apk/keys/
+COPY $apk_packages/leveldb-$leveldb_version-r0.apk .
+COPY $apk_packages/leveldb-dev-$leveldb_version-r0.apk .
+
+RUN apk --no-cache add leveldb-$leveldb_version-r0.apk \
+                       leveldb-dev-$leveldb_version-r0.apk && \
+    rm -f /var/cache/apk/* \
+          leveldb-$leveldb_version-r0.apk \
+          leveldb-dev-$leveldb_version-r0.apk
+
+RUN git clone --depth 1 -b "$opam_tag" git://github.com/ocaml/opam && \
+    cd opam && \
+    make cold CONFIGURE_ARGS="--prefix /usr" && \
+    make install && \
+    cd .. && \
+    rm -r opam
+
+USER opam
+
+WORKDIR /home/opam
+
+RUN mkdir ~/.ssh && \
+    chmod 700 ~/.ssh && \
+    git config --global user.email "tezos@example.com" && \
+    git config --global user.name "Tezos CI" && \
+    cd /home/opam && \
+    git clone --depth 1 -b master git://github.com/ocaml/opam-repository && \
+    cd opam-repository && \
+    cd /home/opam/opam-repository && \
+    opam admin upgrade && \
+    git checkout -b v2 && \
+    git add . && \
+    git commit -a -m 'opam admin upgrade' && \
+    opam init -a -y --comp $ocaml_version /home/opam/opam-repository && \
+    opam install -y depext
+
+EOF
+
+echo
+echo "### Building base image..."
+echo
+
+docker build --pull -t "$image_name:$image_version" .
+
+rm Dockerfile

scripts/version.sh

@@ -5,3 +5,4 @@
 
 alpine_version=3.6
 ocaml_version=4.04.2
+leveldb_version=1.18