From 99cdf5e5c2f3f1e3bf818b3de48fb8a445f9b05e Mon Sep 17 00:00:00 2001 From: Pietro Date: Tue, 16 Jan 2018 15:45:42 +0100 Subject: [PATCH] gitlab CI refactoring - Docker files and build scripts + add git, bash and patch to the docker base definition (Dockerfile.base) + build leveldb once and for all (scripts/create_docker_image.leveldb_deps.sh) + add --depth 1 to all git clone calls + remove sudo, openssh and rsync from base images + add --no-cache to all apk calls + merge dockerfiles in the sh scripts --- .dockerignore | 10 +++ .gitignore | 10 ++- .gitlab-ci.yml | 32 +------- scripts/Dockerfile.alpine.in | 46 ----------- scripts/Dockerfile.alpine.opam2.in | 60 -------------- scripts/Dockerfile.build.in | 8 -- scripts/Dockerfile.build_deps.in | 16 ---- scripts/create_apk.leveldb.sh | 35 ++++++++ scripts/create_docker_image.alpine.opam2.sh | 34 -------- scripts/create_docker_image.alpine.sh | 30 ------- scripts/create_docker_image.build.sh | 21 ++++- scripts/create_docker_image.build_deps.sh | 44 +++++++---- scripts/create_docker_image.minimal.sh | 18 ++++- scripts/create_docker_image.opam.sh | 88 +++++++++++++++++++++ scripts/version.sh | 1 + 15 files changed, 206 insertions(+), 247 deletions(-) delete mode 100644 scripts/Dockerfile.alpine.in delete mode 100644 scripts/Dockerfile.alpine.opam2.in delete mode 100644 scripts/Dockerfile.build.in delete mode 100644 scripts/Dockerfile.build_deps.in create mode 100755 scripts/create_apk.leveldb.sh delete mode 100755 scripts/create_docker_image.alpine.opam2.sh delete mode 100755 scripts/create_docker_image.alpine.sh create mode 100755 scripts/create_docker_image.opam.sh diff --git a/.dockerignore b/.dockerignore index 49311a465..c65a50708 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,21 @@ +## /!\ /!\ Update .dockerignore accordingly /!\ /!\ + +.DS_Store +__pycache__ + _build **/*.install tezos-node tezos-protocol-compiler tezos-client +tezos-admin-client scripts/opam-test-all.sh.DONE +docs/introduction/readme.rst + **/.merlin **/*~ @@ -18,6 +26,8 @@ scripts/opam-test-all.sh.DONE _opam +## Not in .gitignore + .git .gitignore .gitlab-ci.yml diff --git a/.gitignore b/.gitignore index eda2cd6d8..da7d913b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ +## /!\ /!\ Update .gitignore accordingly /!\ /!\ + .DS_Store __pycache__ -/docs/introduction/readme.rst - _build *.install @@ -14,6 +14,8 @@ _build /scripts/opam-test-all.sh.DONE +/docs/introduction/readme.rst + .merlin *~ @@ -23,4 +25,8 @@ _build *.orig /_opam + +## Not in .dockerignore + /Dockerfile +/_apk diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7a522ed6..f8ebeed5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,4 @@ + variables: public_docker_image: docker.io/tezos/tezos @@ -22,9 +23,7 @@ stages: - docker info - docker login -u gitlab-ci-token -p "$CI_BUILD_TOKEN" "${CI_REGISTRY}" tags: - - safe_docker - -## Prepare docker image with precompiled OCaml/opam + - docker prepare:opam: <<: *dind_definition @@ -34,30 +33,14 @@ prepare:opam: before_script: - . ./scripts/version.sh script: - - ./scripts/create_docker_image.alpine.sh - - mkdir ~/.docker || true + - mkdir -p ~/.docker - echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ; + - ./scripts/create_docker_image.opam.sh - docker push "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}" - docker tag "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}" "tezos/opam:latest" - docker push "tezos/opam:latest" -prepare:opam2: - <<: *dind_definition - stage: prepare - only: - - schedules - before_script: - - . ./scripts/version.sh - script: - - ./scripts/create_docker_image.alpine.opam2.sh - - mkdir ~/.docker || true - - echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ; - - docker push "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}" - - docker tag "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}" - "tezos/opam2:latest" - - docker push "tezos/opam2:latest" - ## Tezos build: @@ -125,43 +108,36 @@ test:proto_alpha:transaction: <<: *test_definition script: - jbuilder build @test/proto_alpha/runtest_transaction - retry: 1 test:proto_alpha:origination: <<: *test_definition script: - jbuilder build @test/proto_alpha/runtest_origination - retry: 1 test:proto_alpha:endorsement: <<: *test_definition script: - jbuilder build @test/proto_alpha/runtest_endorsement - retry: 1 test:proto_alpha:vote: <<: *test_definition script: - jbuilder build @test/proto_alpha/runtest_vote - retry: 1 test:basic.sh: <<: *test_definition script: - jbuilder build @test/runtest_basic.sh - retry: 1 test:contracts.sh: <<: *test_definition script: - jbuilder build @test/runtest_contracts.sh - retry: 1 test:multinode.sh: <<: *test_definition script: - jbuilder build @test/runtest_multinode.sh - retry: 1 test:proto:sandbox: <<: *test_definition diff --git a/scripts/Dockerfile.alpine.in b/scripts/Dockerfile.alpine.in deleted file mode 100644 index df547e1bb..000000000 --- a/scripts/Dockerfile.alpine.in +++ /dev/null @@ -1,46 +0,0 @@ -## Based on https://github.com/ocaml/opam-dockerfiles - -FROM alpine:$alpine_version - -RUN apk update && apk upgrade && \ - apk add build-base snappy-dev abuild alpine-sdk \ - openssh bash nano ncurses-dev rsync xz m4 \ - opam aspcud \ - gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \ - libsodium-dev && \ - rm -f /var/cache/apk/* && \ - adduser -S opam && \ - adduser opam abuild && \ - echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \ - chmod 440 /etc/sudoers.d/opam && \ - chown root:root /etc/sudoers.d/opam && \ - sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers - -USER opam - -WORKDIR /home/opam - -RUN mkdir .ssh && \ - chmod 700 .ssh && \ - git config --global user.email "docker@example.com" && \ - git config --global user.name "Docker CI" && \ - sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \ - sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \ - sudo -u opam sh -c "opam install -y depext travis-opam" - -COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD -RUN sudo chown -R opam leveldb-1.18 - -RUN sudo apk update && \ - abuild-keygen -ai && \ - cd leveldb-1.18 && \ - abuild checksum && abuild -r && \ - cd .. && \ - sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \ - sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \ - rm -rf leveldb-1.18 packages && \ - sudo rm -f /var/cache/apk/* - -ENTRYPOINT [ "opam", "config", "exec", "--" ] - -CMD [ "sh" ] diff --git a/scripts/Dockerfile.alpine.opam2.in b/scripts/Dockerfile.alpine.opam2.in deleted file mode 100644 index 3d8a92d86..000000000 --- a/scripts/Dockerfile.alpine.opam2.in +++ /dev/null @@ -1,60 +0,0 @@ -## Based on https://github.com/ocaml/opam-dockerfiles - -FROM alpine:$alpine_version - -RUN apk update && apk upgrade && \ - apk add build-base snappy-dev abuild alpine-sdk \ - openssh bash nano ncurses-dev rsync xz m4 \ - ocaml \ - gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \ - libsodium-dev && \ - rm -f /var/cache/apk/* && \ - adduser -S opam && \ - adduser opam abuild && \ - echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \ - chmod 440 /etc/sudoers.d/opam && \ - chown root:root /etc/sudoers.d/opam && \ - sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers - -USER opam - -WORKDIR /home/opam - -RUN mkdir ~/.ssh && \ - chmod 700 ~/.ssh && \ - git config --global user.email "docker@example.com" && \ - git config --global user.name "Docker CI" && \ - cd /home/opam && \ - git clone -b "$opam_tag" git://github.com/ocaml/opam && \ - cd /home/opam/opam && \ - ./configure --prefix /usr && \ - make lib-ext && \ - make opam && \ - sudo cp -L opam /usr/bin/opam && \ - cd /home/opam && \ - sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \ - cd opam-repository && \ - sudo -u opam sh -c "cd /home/opam/opam-repository && opam admin upgrade && git checkout -b v2 && git add . && git commit -a -m 'opam admin upgrade'" && \ - sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \ - sudo -u opam sh -c "opam install -y depext travis-opam" && \ - cd /home/opam/opam && \ - make opam-installer && \ - sudo cp -L opam-installer /usr/bin/opam-installer && \ - rm -r /home/opam/opam - -COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD -RUN sudo chown -R opam leveldb-1.18 - -RUN sudo apk update && \ - abuild-keygen -ai && \ - cd leveldb-1.18 && \ - abuild checksum && abuild -r && \ - cd .. && \ - sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \ - sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \ - rm -rf leveldb-1.18 packages && \ - sudo rm -f /var/cache/apk/* - -ENTRYPOINT [ "opam", "config", "exec", "--" ] - -CMD [ "sh" ] diff --git a/scripts/Dockerfile.build.in b/scripts/Dockerfile.build.in deleted file mode 100644 index e85c791fd..000000000 --- a/scripts/Dockerfile.build.in +++ /dev/null @@ -1,8 +0,0 @@ -FROM $base_image - -COPY . tezos - -RUN sudo chown -R opam /home/opam/tezos && \ - cd tezos && \ - opam config exec -- jbuilder build @install && \ - opam config exec -- jbuilder install diff --git a/scripts/Dockerfile.build_deps.in b/scripts/Dockerfile.build_deps.in deleted file mode 100644 index 39da49d9d..000000000 --- a/scripts/Dockerfile.build_deps.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM $base_image - -## /!\ /!\ /!\ /!\/!\ /!\ /!\ -## /!\ Don't forget fu update the variables 'dependencies' in -## /!\ `create_docker_image.build_deps.sh` when modifyiing the -## /!\ 'COPY'edfiles. -## /!\ /!\ /!\ /!\/!\ /!\ /!\ - -$copy_files - -RUN sudo apk update && \ - opam config exec -- ./tezos/scripts/install_build_deps.sh && \ - opam install ocp-indent && \ - sudo rm -f /var/cache/apk/* && \ - rm -fr ~/.opam/log/ && \ - rm -fr "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi diff --git a/scripts/create_apk.leveldb.sh b/scripts/create_apk.leveldb.sh new file mode 100755 index 000000000..727555e88 --- /dev/null +++ b/scripts/create_apk.leveldb.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +set -e + +script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" +src_dir="$(dirname "$script_dir")" +cd "$src_dir" + +. scripts/version.sh + +image_name="${1:-tezos/leveldb}" + +cat < Dockerfile +FROM andyshinn/alpine-abuild:v4 + +ENV PACKAGER "Tezos " +WORKDIR /home/builder/ +RUN abuild-keygen -a -i +COPY scripts/leveldb-$leveldb_version.APKBUILD APKBUILD +RUN abuilder -r +EOF + +echo +echo "### Building leveldb..." +echo + +docker build -t $image_name:$leveldb_version . +rm Dockerfile + +mkdir -p _apk + +docker create --name tezos.leveldb.tmp $image_name:$leveldb_version +docker cp -L tezos.leveldb.tmp:/etc/apk/keys _apk/ +docker cp -L tezos.leveldb.tmp:/packages _apk +docker rm tezos.leveldb.tmp diff --git a/scripts/create_docker_image.alpine.opam2.sh b/scripts/create_docker_image.alpine.opam2.sh deleted file mode 100755 index dc12dbb9c..000000000 --- a/scripts/create_docker_image.alpine.opam2.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/sh - -set -e - -script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" -src_dir="$(dirname "$script_dir")" -cd "$src_dir" - -. scripts/version.sh -image_name="${1:-tezos/opam2}" -image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}" - -cleanup () { - set +e - echo Cleaning up... - rm -rf Dockerfile -} -trap cleanup EXIT INT - -#opam_tag=2.0.0-beta5 -opam_tag=master - -sed scripts/Dockerfile.alpine.opam2.in \ - -e 's|$alpine_version|'"$alpine_version"'|g' \ - -e 's|$ocaml_version|'"$ocaml_version"'|g' \ - -e 's|$opam_tag|'"$opam_tag"'|g' > Dockerfile - -echo -echo "### Building base image..." -echo - -docker build --pull -t "$image_name:$image_version" . - -rm Dockerfile diff --git a/scripts/create_docker_image.alpine.sh b/scripts/create_docker_image.alpine.sh deleted file mode 100755 index bf8acc82a..000000000 --- a/scripts/create_docker_image.alpine.sh +++ /dev/null @@ -1,30 +0,0 @@ -#! /bin/sh - -set -e - -script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" -src_dir="$(dirname "$script_dir")" -cd "$src_dir" - -. scripts/version.sh -image_name="${1:-tezos/opam}" -image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}" - -cleanup () { - set +e - echo Cleaning up... - rm -rf Dockerfile -} -trap cleanup EXIT INT - -sed scripts/Dockerfile.alpine.in \ - -e 's|$alpine_version|'"$alpine_version"'|g' \ - -e 's|$ocaml_version|'"$ocaml_version"'|g' > Dockerfile - -echo -echo "### Building base image..." -echo - -docker build --pull -t "$image_name:$image_version" . - -rm Dockerfile diff --git a/scripts/create_docker_image.build.sh b/scripts/create_docker_image.build.sh index 8920de8ce..823c54876 100755 --- a/scripts/create_docker_image.build.sh +++ b/scripts/create_docker_image.build.sh @@ -18,15 +18,30 @@ cleanup () { } trap cleanup EXIT INT -sed -e 's|$base_image|'"$build_deps_image_name"'|g' \ - scripts/Dockerfile.build.in > Dockerfile +cat < Dockerfile +FROM $build_deps_image_name + +COPY . tezos +USER root +RUN chown -R opam /home/opam/tezos + +# build tezos and friends +USER opam + +RUN cd tezos && \ + opam config exec -- jbuilder build @install && \ + opam config exec -- jbuilder install + +USER root + +ENTRYPOINT [ "/sbin/su-exec", "opam", "opam", "config", "exec", "--" ] +EOF echo echo "### Building tezos..." echo docker build -t "$image_name:$image_version" . - rm Dockerfile echo diff --git a/scripts/create_docker_image.build_deps.sh b/scripts/create_docker_image.build_deps.sh index 8352a7b41..21d2b52ee 100755 --- a/scripts/create_docker_image.build_deps.sh +++ b/scripts/create_docker_image.build_deps.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh set -e @@ -13,28 +13,40 @@ cached_image="${3:-}" base_image="tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}" if ! docker pull "$base_image" ; then - ./scripts/create_docker_image.alpine.sh + ./scripts/create_docker_image.alpine.opam2.sh fi cleanup () { set +e echo Cleaning up... - rm -rf Dockerfile + rm -f Dockerfile opams.tar.gz scripts.tar.gz } trap cleanup EXIT INT -opam_files=$(find -name \*.opam | sort) -dependencies="$opam_files scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh Dockerfile" +dependencies="scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh" +tar czvf scripts.tar.gz $dependencies -for file in $dependencies; do - if [ "$file" = Dockerfile ]; then continue; fi - copy_files="$copy_files\nCOPY $file ./tezos/$file" -done +opams=$(find -name \*.opam -type f) +tar czvf opams.tar.gz $opams -sed -e 's|$base_image|'"$base_image"'|g' \ - -e 's|$ocaml_version|'"$ocaml_version"'|g' \ - -e 's|$copy_files|'"$copy_files"'|g' \ - scripts/Dockerfile.build_deps.in > Dockerfile +cat <Dockerfile +FROM $base_image + +# these two archives are created in the file +# scripts/create_docker_image.build_deps.sh and removed +# automatically after +ADD opams.tar.gz tezos/ +ADD scripts.tar.gz tezos/ + +USER opam + +RUN opam config exec -- ./tezos/scripts/install_build_deps.sh +ENV OPAMYES=yes +RUN opam config exec -- opam install ocp-indent && \ + rm -fr ~/.opam/log/ && \ + rm -fr "\$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi + +EOF ## Lookup for for prebuilt dependencies... dependencies_sha1=$(docker inspect --format="{{ .RootFS.Layers }}" --type=image $base_image | sha1sum - $dependencies | sha1sum | tr -d ' -') @@ -54,12 +66,12 @@ if [ ! -z "$cached_image" ]; then fi echo -echo "### Building tezos dependencies..." +echo "### Building tezos dependencies... $image_name:$image_version" echo -docker build -t "$image_name:$image_version" . +docker build --pull -t "$image_name:$image_version" . -rm Dockerfile +rm -f Dockerfile opams.tar.gz scripts.tar.gz echo echo "### Succesfully build docker image: $image_name:$image_version" diff --git a/scripts/create_docker_image.minimal.sh b/scripts/create_docker_image.minimal.sh index 2e9418f9d..87f7dff73 100755 --- a/scripts/create_docker_image.minimal.sh +++ b/scripts/create_docker_image.minimal.sh @@ -18,10 +18,20 @@ cleanup () { } trap cleanup EXIT INT -docker run -dit --rm --volume $(pwd)/bin:/home/opam/bin "$build_image_name" \ - /bin/sh -c "sudo cp -L /home/opam/tezos/_build/install/default/bin/* /home/opam/bin" +# assume $build_image_name has already been created +mkdir -p _docker_build_result +docker create --name tmp1 $build_image_name +docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-client _docker_build_result/ +docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-node _docker_build_result/ +docker rm tmp1 -ls bin +# assume tezos/leveldb has already been created +mkdir -p _docker_build_result/leveldb +mkdir -p _docker_build_result/keys +docker create --name tmp1 tezos/leveldb +docker cp -L tmp1:/etc/apk/keys _docker_build_result/ +docker cp -L tmp1:/packages _docker_build_result/ +docker rm tmp1 echo echo "### Building minimal docker image..." @@ -35,4 +45,4 @@ echo echo "### Succesfully build docker image: $image_name:$image_version" echo -rm -r bin +rm -Rf _docker_build_result diff --git a/scripts/create_docker_image.opam.sh b/scripts/create_docker_image.opam.sh new file mode 100755 index 000000000..9ba5336a8 --- /dev/null +++ b/scripts/create_docker_image.opam.sh @@ -0,0 +1,88 @@ +#! /bin/sh + +set -e + +script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" +src_dir="$(dirname "$script_dir")" +cd "$src_dir" + +. scripts/version.sh + +image_name="${1:-tezos/opam}" +image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}" + +cleanup () { + set +e + echo Cleaning up... + rm -rf Dockerfile +} +trap cleanup EXIT INT + +apk_keys="_apk/keys/" +apk_packages="_apk/packages/home/x86_64/" + +if ! ( [ -f "$apk_packages/leveldb-$leveldb_version-r0.apk" ] && \ + [ -f "$apk_packages/leveldb-dev-$leveldb_version-r0.apk" ] && \ + [ -d "$apk_keys" ] ) ; then + ./scripts/create_apk.leveldb.sh +fi + +opam_tag=2.0.0-rc + +cat < Dockerfile +FROM alpine:$alpine_version + +RUN apk update && \ + apk upgrade && \ + apk --no-cache add su-exec build-base \ + bash ncurses-dev xz m4 git ca-certificates wget \ + gmp-dev libev-dev libressl-dev pcre-dev perl zlib-dev libsodium-dev && \ + update-ca-certificates && \ + rm -f /var/cache/apk/* && \ + adduser -S opam + +COPY $apk_keys /etc/apk/keys/ +COPY $apk_packages/leveldb-$leveldb_version-r0.apk . +COPY $apk_packages/leveldb-dev-$leveldb_version-r0.apk . + +RUN apk --no-cache add leveldb-$leveldb_version-r0.apk \ + leveldb-dev-$leveldb_version-r0.apk && \ + rm -f /var/cache/apk/* \ + leveldb-$leveldb_version-r0.apk \ + leveldb-dev-$leveldb_version-r0.apk + +RUN git clone --depth 1 -b "$opam_tag" git://github.com/ocaml/opam && \ + cd opam && \ + make cold CONFIGURE_ARGS="--prefix /usr" && \ + make install && \ + cd .. && \ + rm -r opam + +USER opam + +WORKDIR /home/opam + +RUN mkdir ~/.ssh && \ + chmod 700 ~/.ssh && \ + git config --global user.email "tezos@example.com" && \ + git config --global user.name "Tezos CI" && \ + cd /home/opam && \ + git clone --depth 1 -b master git://github.com/ocaml/opam-repository && \ + cd opam-repository && \ + cd /home/opam/opam-repository && \ + opam admin upgrade && \ + git checkout -b v2 && \ + git add . && \ + git commit -a -m 'opam admin upgrade' && \ + opam init -a -y --comp $ocaml_version /home/opam/opam-repository && \ + opam install -y depext + +EOF + +echo +echo "### Building base image..." +echo + +docker build --pull -t "$image_name:$image_version" . + +rm Dockerfile diff --git a/scripts/version.sh b/scripts/version.sh index 5e6df0574..a81ae4d1b 100644 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -5,3 +5,4 @@ alpine_version=3.6 ocaml_version=4.04.2 +leveldb_version=1.18