gitlab CI refactoring

- Docker files and build scripts
    + add git, bash and patch to the docker base definition (Dockerfile.base)
    + build leveldb once and for all
      (scripts/create_docker_image.leveldb_deps.sh)
    + add --depth 1 to all git clone calls
    + remove sudo, openssh and rsync from base images
    + add --no-cache to all apk calls
    + merge dockerfiles in the sh scripts
This commit is contained in:
Pietro 2018-01-16 15:45:42 +01:00 committed by Grégoire Henry
parent 66fb1811ae
commit 99cdf5e5c2
15 changed files with 206 additions and 247 deletions

View File

@ -1,13 +1,21 @@
## /!\ /!\ Update .dockerignore accordingly /!\ /!\
.DS_Store
__pycache__
_build
**/*.install
tezos-node
tezos-protocol-compiler
tezos-client
tezos-admin-client
scripts/opam-test-all.sh.DONE
docs/introduction/readme.rst
**/.merlin
**/*~
@ -18,6 +26,8 @@ scripts/opam-test-all.sh.DONE
_opam
## Not in .gitignore
.git
.gitignore
.gitlab-ci.yml

10
.gitignore vendored
View File

@ -1,9 +1,9 @@
## /!\ /!\ Update .gitignore accordingly /!\ /!\
.DS_Store
__pycache__
/docs/introduction/readme.rst
_build
*.install
@ -14,6 +14,8 @@ _build
/scripts/opam-test-all.sh.DONE
/docs/introduction/readme.rst
.merlin
*~
@ -23,4 +25,8 @@ _build
*.orig
/_opam
## Not in .dockerignore
/Dockerfile
/_apk

View File

@ -1,3 +1,4 @@
variables:
public_docker_image: docker.io/tezos/tezos
@ -22,9 +23,7 @@ stages:
- docker info
- docker login -u gitlab-ci-token -p "$CI_BUILD_TOKEN" "${CI_REGISTRY}"
tags:
- safe_docker
## Prepare docker image with precompiled OCaml/opam
- docker
prepare:opam:
<<: *dind_definition
@ -34,30 +33,14 @@ prepare:opam:
before_script:
- . ./scripts/version.sh
script:
- ./scripts/create_docker_image.alpine.sh
- mkdir ~/.docker || true
- mkdir -p ~/.docker
- echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ;
- ./scripts/create_docker_image.opam.sh
- docker push "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
- docker tag "tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
"tezos/opam:latest"
- docker push "tezos/opam:latest"
prepare:opam2:
<<: *dind_definition
stage: prepare
only:
- schedules
before_script:
- . ./scripts/version.sh
script:
- ./scripts/create_docker_image.alpine.opam2.sh
- mkdir ~/.docker || true
- echo "$CI_DOCKER_AUTH" > ~/.docker/config.json ;
- docker push "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}"
- docker tag "tezos/opam2:alpine-${alpine_version}_ocaml-${ocaml_version}"
"tezos/opam2:latest"
- docker push "tezos/opam2:latest"
## Tezos
build:
@ -125,43 +108,36 @@ test:proto_alpha:transaction:
<<: *test_definition
script:
- jbuilder build @test/proto_alpha/runtest_transaction
retry: 1
test:proto_alpha:origination:
<<: *test_definition
script:
- jbuilder build @test/proto_alpha/runtest_origination
retry: 1
test:proto_alpha:endorsement:
<<: *test_definition
script:
- jbuilder build @test/proto_alpha/runtest_endorsement
retry: 1
test:proto_alpha:vote:
<<: *test_definition
script:
- jbuilder build @test/proto_alpha/runtest_vote
retry: 1
test:basic.sh:
<<: *test_definition
script:
- jbuilder build @test/runtest_basic.sh
retry: 1
test:contracts.sh:
<<: *test_definition
script:
- jbuilder build @test/runtest_contracts.sh
retry: 1
test:multinode.sh:
<<: *test_definition
script:
- jbuilder build @test/runtest_multinode.sh
retry: 1
test:proto:sandbox:
<<: *test_definition

View File

@ -1,46 +0,0 @@
## Based on https://github.com/ocaml/opam-dockerfiles
FROM alpine:$alpine_version
RUN apk update && apk upgrade && \
apk add build-base snappy-dev abuild alpine-sdk \
openssh bash nano ncurses-dev rsync xz m4 \
opam aspcud \
gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \
libsodium-dev && \
rm -f /var/cache/apk/* && \
adduser -S opam && \
adduser opam abuild && \
echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
chmod 440 /etc/sudoers.d/opam && \
chown root:root /etc/sudoers.d/opam && \
sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
USER opam
WORKDIR /home/opam
RUN mkdir .ssh && \
chmod 700 .ssh && \
git config --global user.email "docker@example.com" && \
git config --global user.name "Docker CI" && \
sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \
sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \
sudo -u opam sh -c "opam install -y depext travis-opam"
COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD
RUN sudo chown -R opam leveldb-1.18
RUN sudo apk update && \
abuild-keygen -ai && \
cd leveldb-1.18 && \
abuild checksum && abuild -r && \
cd .. && \
sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \
sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \
rm -rf leveldb-1.18 packages && \
sudo rm -f /var/cache/apk/*
ENTRYPOINT [ "opam", "config", "exec", "--" ]
CMD [ "sh" ]

View File

@ -1,60 +0,0 @@
## Based on https://github.com/ocaml/opam-dockerfiles
FROM alpine:$alpine_version
RUN apk update && apk upgrade && \
apk add build-base snappy-dev abuild alpine-sdk \
openssh bash nano ncurses-dev rsync xz m4 \
ocaml \
gmp-dev libev-dev libressl-dev linux-headers pcre-dev perl zlib-dev \
libsodium-dev && \
rm -f /var/cache/apk/* && \
adduser -S opam && \
adduser opam abuild && \
echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
chmod 440 /etc/sudoers.d/opam && \
chown root:root /etc/sudoers.d/opam && \
sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
USER opam
WORKDIR /home/opam
RUN mkdir ~/.ssh && \
chmod 700 ~/.ssh && \
git config --global user.email "docker@example.com" && \
git config --global user.name "Docker CI" && \
cd /home/opam && \
git clone -b "$opam_tag" git://github.com/ocaml/opam && \
cd /home/opam/opam && \
./configure --prefix /usr && \
make lib-ext && \
make opam && \
sudo cp -L opam /usr/bin/opam && \
cd /home/opam && \
sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \
cd opam-repository && \
sudo -u opam sh -c "cd /home/opam/opam-repository && opam admin upgrade && git checkout -b v2 && git add . && git commit -a -m 'opam admin upgrade'" && \
sudo -u opam sh -c "opam init -a -y --comp $ocaml_version /home/opam/opam-repository" && \
sudo -u opam sh -c "opam install -y depext travis-opam" && \
cd /home/opam/opam && \
make opam-installer && \
sudo cp -L opam-installer /usr/bin/opam-installer && \
rm -r /home/opam/opam
COPY scripts/leveldb-1.18.APKBUILD /home/opam/leveldb-1.18/APKBUILD
RUN sudo chown -R opam leveldb-1.18
RUN sudo apk update && \
abuild-keygen -ai && \
cd leveldb-1.18 && \
abuild checksum && abuild -r && \
cd .. && \
sudo apk add packages/opam/x86_64/leveldb-1.18-r0.apk && \
sudo apk add packages/opam/x86_64/leveldb-dev-1.18-r0.apk && \
rm -rf leveldb-1.18 packages && \
sudo rm -f /var/cache/apk/*
ENTRYPOINT [ "opam", "config", "exec", "--" ]
CMD [ "sh" ]

View File

@ -1,8 +0,0 @@
FROM $base_image
COPY . tezos
RUN sudo chown -R opam /home/opam/tezos && \
cd tezos && \
opam config exec -- jbuilder build @install && \
opam config exec -- jbuilder install

View File

@ -1,16 +0,0 @@
FROM $base_image
## /!\ /!\ /!\ /!\/!\ /!\ /!\
## /!\ Don't forget fu update the variables 'dependencies' in
## /!\ `create_docker_image.build_deps.sh` when modifyiing the
## /!\ 'COPY'edfiles.
## /!\ /!\ /!\ /!\/!\ /!\ /!\
$copy_files
RUN sudo apk update && \
opam config exec -- ./tezos/scripts/install_build_deps.sh && \
opam install ocp-indent && \
sudo rm -f /var/cache/apk/* && \
rm -fr ~/.opam/log/ && \
rm -fr "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi

35
scripts/create_apk.leveldb.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/sh
set -e
script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos/leveldb}"
cat <<EOF > Dockerfile
FROM andyshinn/alpine-abuild:v4
ENV PACKAGER "Tezos <team@tezos.com>"
WORKDIR /home/builder/
RUN abuild-keygen -a -i
COPY scripts/leveldb-$leveldb_version.APKBUILD APKBUILD
RUN abuilder -r
EOF
echo
echo "### Building leveldb..."
echo
docker build -t $image_name:$leveldb_version .
rm Dockerfile
mkdir -p _apk
docker create --name tezos.leveldb.tmp $image_name:$leveldb_version
docker cp -L tezos.leveldb.tmp:/etc/apk/keys _apk/
docker cp -L tezos.leveldb.tmp:/packages _apk
docker rm tezos.leveldb.tmp

View File

@ -1,34 +0,0 @@
#! /bin/sh
set -e
script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos/opam2}"
image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
cleanup () {
set +e
echo Cleaning up...
rm -rf Dockerfile
}
trap cleanup EXIT INT
#opam_tag=2.0.0-beta5
opam_tag=master
sed scripts/Dockerfile.alpine.opam2.in \
-e 's|$alpine_version|'"$alpine_version"'|g' \
-e 's|$ocaml_version|'"$ocaml_version"'|g' \
-e 's|$opam_tag|'"$opam_tag"'|g' > Dockerfile
echo
echo "### Building base image..."
echo
docker build --pull -t "$image_name:$image_version" .
rm Dockerfile

View File

@ -1,30 +0,0 @@
#! /bin/sh
set -e
script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos/opam}"
image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
cleanup () {
set +e
echo Cleaning up...
rm -rf Dockerfile
}
trap cleanup EXIT INT
sed scripts/Dockerfile.alpine.in \
-e 's|$alpine_version|'"$alpine_version"'|g' \
-e 's|$ocaml_version|'"$ocaml_version"'|g' > Dockerfile
echo
echo "### Building base image..."
echo
docker build --pull -t "$image_name:$image_version" .
rm Dockerfile

View File

@ -18,15 +18,30 @@ cleanup () {
}
trap cleanup EXIT INT
sed -e 's|$base_image|'"$build_deps_image_name"'|g' \
scripts/Dockerfile.build.in > Dockerfile
cat <<EOF > Dockerfile
FROM $build_deps_image_name
COPY . tezos
USER root
RUN chown -R opam /home/opam/tezos
# build tezos and friends
USER opam
RUN cd tezos && \
opam config exec -- jbuilder build @install && \
opam config exec -- jbuilder install
USER root
ENTRYPOINT [ "/sbin/su-exec", "opam", "opam", "config", "exec", "--" ]
EOF
echo
echo "### Building tezos..."
echo
docker build -t "$image_name:$image_version" .
rm Dockerfile
echo

View File

@ -1,4 +1,4 @@
#! /bin/sh
#!/bin/sh
set -e
@ -13,28 +13,40 @@ cached_image="${3:-}"
base_image="tezos/opam:alpine-${alpine_version}_ocaml-${ocaml_version}"
if ! docker pull "$base_image" ; then
./scripts/create_docker_image.alpine.sh
./scripts/create_docker_image.alpine.opam2.sh
fi
cleanup () {
set +e
echo Cleaning up...
rm -rf Dockerfile
rm -f Dockerfile opams.tar.gz scripts.tar.gz
}
trap cleanup EXIT INT
opam_files=$(find -name \*.opam | sort)
dependencies="$opam_files scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh Dockerfile"
dependencies="scripts/install_build_deps.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh"
tar czvf scripts.tar.gz $dependencies
for file in $dependencies; do
if [ "$file" = Dockerfile ]; then continue; fi
copy_files="$copy_files\nCOPY $file ./tezos/$file"
done
opams=$(find -name \*.opam -type f)
tar czvf opams.tar.gz $opams
sed -e 's|$base_image|'"$base_image"'|g' \
-e 's|$ocaml_version|'"$ocaml_version"'|g' \
-e 's|$copy_files|'"$copy_files"'|g' \
scripts/Dockerfile.build_deps.in > Dockerfile
cat <<EOF >Dockerfile
FROM $base_image
# these two archives are created in the file
# scripts/create_docker_image.build_deps.sh and removed
# automatically after
ADD opams.tar.gz tezos/
ADD scripts.tar.gz tezos/
USER opam
RUN opam config exec -- ./tezos/scripts/install_build_deps.sh
ENV OPAMYES=yes
RUN opam config exec -- opam install ocp-indent && \
rm -fr ~/.opam/log/ && \
rm -fr "\$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi
EOF
## Lookup for for prebuilt dependencies...
dependencies_sha1=$(docker inspect --format="{{ .RootFS.Layers }}" --type=image $base_image | sha1sum - $dependencies | sha1sum | tr -d ' -')
@ -54,12 +66,12 @@ if [ ! -z "$cached_image" ]; then
fi
echo
echo "### Building tezos dependencies..."
echo "### Building tezos dependencies... $image_name:$image_version"
echo
docker build -t "$image_name:$image_version" .
docker build --pull -t "$image_name:$image_version" .
rm Dockerfile
rm -f Dockerfile opams.tar.gz scripts.tar.gz
echo
echo "### Succesfully build docker image: $image_name:$image_version"

View File

@ -18,10 +18,20 @@ cleanup () {
}
trap cleanup EXIT INT
docker run -dit --rm --volume $(pwd)/bin:/home/opam/bin "$build_image_name" \
/bin/sh -c "sudo cp -L /home/opam/tezos/_build/install/default/bin/* /home/opam/bin"
# assume $build_image_name has already been created
mkdir -p _docker_build_result
docker create --name tmp1 $build_image_name
docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-client _docker_build_result/
docker cp -L tmp1:/home/opam/tezos/_build/install/default/bin/tezos-node _docker_build_result/
docker rm tmp1
ls bin
# assume tezos/leveldb has already been created
mkdir -p _docker_build_result/leveldb
mkdir -p _docker_build_result/keys
docker create --name tmp1 tezos/leveldb
docker cp -L tmp1:/etc/apk/keys _docker_build_result/
docker cp -L tmp1:/packages _docker_build_result/
docker rm tmp1
echo
echo "### Building minimal docker image..."
@ -35,4 +45,4 @@ echo
echo "### Succesfully build docker image: $image_name:$image_version"
echo
rm -r bin
rm -Rf _docker_build_result

View File

@ -0,0 +1,88 @@
#! /bin/sh
set -e
script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos/opam}"
image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
cleanup () {
set +e
echo Cleaning up...
rm -rf Dockerfile
}
trap cleanup EXIT INT
apk_keys="_apk/keys/"
apk_packages="_apk/packages/home/x86_64/"
if ! ( [ -f "$apk_packages/leveldb-$leveldb_version-r0.apk" ] && \
[ -f "$apk_packages/leveldb-dev-$leveldb_version-r0.apk" ] && \
[ -d "$apk_keys" ] ) ; then
./scripts/create_apk.leveldb.sh
fi
opam_tag=2.0.0-rc
cat <<EOF > Dockerfile
FROM alpine:$alpine_version
RUN apk update && \
apk upgrade && \
apk --no-cache add su-exec build-base \
bash ncurses-dev xz m4 git ca-certificates wget \
gmp-dev libev-dev libressl-dev pcre-dev perl zlib-dev libsodium-dev && \
update-ca-certificates && \
rm -f /var/cache/apk/* && \
adduser -S opam
COPY $apk_keys /etc/apk/keys/
COPY $apk_packages/leveldb-$leveldb_version-r0.apk .
COPY $apk_packages/leveldb-dev-$leveldb_version-r0.apk .
RUN apk --no-cache add leveldb-$leveldb_version-r0.apk \
leveldb-dev-$leveldb_version-r0.apk && \
rm -f /var/cache/apk/* \
leveldb-$leveldb_version-r0.apk \
leveldb-dev-$leveldb_version-r0.apk
RUN git clone --depth 1 -b "$opam_tag" git://github.com/ocaml/opam && \
cd opam && \
make cold CONFIGURE_ARGS="--prefix /usr" && \
make install && \
cd .. && \
rm -r opam
USER opam
WORKDIR /home/opam
RUN mkdir ~/.ssh && \
chmod 700 ~/.ssh && \
git config --global user.email "tezos@example.com" && \
git config --global user.name "Tezos CI" && \
cd /home/opam && \
git clone --depth 1 -b master git://github.com/ocaml/opam-repository && \
cd opam-repository && \
cd /home/opam/opam-repository && \
opam admin upgrade && \
git checkout -b v2 && \
git add . && \
git commit -a -m 'opam admin upgrade' && \
opam init -a -y --comp $ocaml_version /home/opam/opam-repository && \
opam install -y depext
EOF
echo
echo "### Building base image..."
echo
docker build --pull -t "$image_name:$image_version" .
rm Dockerfile

View File

@ -5,3 +5,4 @@
alpine_version=3.6
ocaml_version=4.04.2
leveldb_version=1.18