balsoft/ligo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Docker: integration in CI

Browse Source 
And minor fix:

- introduce configuration file for OCaml and alpine versions
- add missing `libssl1.0` in docker image
- remove the split call to `install_build_deps` (not required anymore)
- rename scripts
This commit is contained in:
Grégoire Henry 2017-01-22 01:59:18 +01:00
parent 87254788b9
commit d87a6cd556
13 changed files with 208 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
.dockerignore Normal file
View File

@ -0,0 +1,67 @@
.git
tezos-node
tezos-protocol-compiler
tezos-client
tezos-webclient
tezos-attacker
src/Makefile.local
src/webclient_static.ml
src/.depend
src/node/updater/environment_gen
src/node/updater/proto_environment.mli
src/compiler/embedded_cmis.ml
src/proto/**/_tzbuild
src/proto/register_client_*.ml
src/client/embedded/**/_tzbuild
src/client/embedded/demo/.depend
src/client/embedded/bootstrap/.depend
src/client/embedded/bootstrap/concrete_lexer.ml
src/client/embedded/bootstrap/concrete_parser.ml
src/client/embedded/bootstrap/concrete_parser.mli
src/client/embedded/bootstrap/webclient_proto_static.ml
src/client/embedded/bootstrap/main.byte
src/client/embedded/bootstrap/webclient_static/main.js
src/client/embedded/bootstrap/webclient/browser/main.byte
src/client/embedded/bootstrap/webclient/static/main.js
src/client/embedded/bootstrap/webclient/webclient_proto_static.ml
test/.depend
test/reports
test/test-store
test/test-state
test/test-context
test/test-basic
test/test-data-encoding
test/test-p2p-io-scheduler
test/test-p2p-connection
test/test-p2p-connection-pool
test/LOG
**/*~
**/\#*\#
**/*.[oa]
**/*.so
**/*~
**/*.cm[iaoxt]
**/*.cmti
**/*.cmxa
**/*.cmxs
**/*.cmp
**/*.mli.deps
**/*.ml.deps
**/*.mli.deps.byte
**/*.ml.deps.byte
**/bisect*.out
**/*.rej
**/*.orig

56
.gitlab-ci.yml
View File

@ -1,10 +1,10 @@
variables:
ocaml_image: alpine_ocaml-4.03.0
build_image_name: ocp_tezos_build
image_name: ocp_tezos
image_id: ${CI_BUILD_REF_NAME}
build_image_name: ${image_name}_build_deps
build_image_id: ${CI_BUILD_REF}
build_image: ${build_image_name}:${ocaml_image}
image: ${build_image}_${build_image_id}
image: ${build_image_name}:${build_image_id}
stages:
- build_deps
@ -21,8 +21,8 @@ build_deps:
tags:
- docker_builder
script:
- ./scripts/create_docker_builder.sh
${build_image_name} ${ocaml_image} "_${build_image_id}"
- ./scripts/create_build_deps_docker_image.sh
${build_image_name} ${build_image_id}
build:
stage: build
@ -137,15 +137,29 @@ test:p2p-connection-pool:
- build
- build:test
expurge:
stage: expurge
publish:docker:
stage: publish
image: ocp:docker
tags:
- docker_builder
script:
- ./scripts/create_docker_image.sh ${image_name} ${build_image_id}
- docker tag ${image_name}:${build_image_id}
"$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker push "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
- docker rmi "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
publish:github:
image: ocaml/opam
stage: publish
tags:
- tezos_builder
only:
- master@tezos/tezos
script:
- echo "${CI_KH}" > ~/.ssh/known_hosts
- echo "${CI_PK_GITLAB}" > ~/.ssh/id_rsa
- echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
- chmod 400 ~/.ssh/id_rsa
- rm -fr .git/refs/original
- git filter-branch --prune-empty --index-filter
@ -162,23 +176,7 @@ expurge:
export GIT_AUTHOR_EMAIL="contact@tezos.com" ;
fi'
HEAD
- git tag ${CI_BUILD_REF}_expurged
- git push git@gitlab.ocamlpro.com:${CI_PROJECT_PATH}.git
-f --tags HEAD:master-expurged
publish:github:
image: ocaml/opam:${ocaml_image}
stage: publish
tags:
- tezos_builder
only:
- master@tezos/tezos
script:
- echo "${CI_KH}" > ~/.ssh/known_hosts
- echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
- chmod 400 ~/.ssh/id_rsa
- git reset ${CI_BUILD_REF}_expurged
- git push git@github.com:tezos/tezos.git -f HEAD:master
- git push git@github.com:OCamlPro/tezos.git -f HEAD:master
cleanup:
stage: cleanup
@ -186,7 +184,7 @@ cleanup:
tags:
- docker_builder
script:
- docker tag ${build_image}_${build_image_id}
${build_image}_${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
- docker rmi ${build_image}_${build_image_id}
- docker tag ${build_image_name}:${build_image_id}
${build_image_name}:${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
- docker rmi ${build_image_name}:${build_image_id}
when: always

7
Makefile
View File

@ -6,9 +6,16 @@ clean:
${MAKE} -C src clean
${MAKE} -C test clean
partial-clean:
${MAKE} -C src partial-clean
${MAKE} -C test clean
.PHONY: test
test:
${MAKE} -C test
build-deps:
@./scripts/install_build_deps.sh all
docker-image:
@./scripts/create_docker_image.sh

22
scripts/Dockerfile.binaries.in
View File

@ -1,20 +1,24 @@
FROM alpine:3.4
# FIXME: I'm currently guessing the version of alpine that the opam image we've
# built the tezos binaries on is based on. if that becomes newer (e.g. alpine
# 3.5), the binaries built on that version of alpine could (will) depend on
# library symbols that won't be in alpine 3.4
FROM alpine:$alpine_version
LABEL distro_style="apk" distro="alpine" distro_long="alpine-3.4" arch="x86_64" operatingsystem="linux"
LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux"
RUN apk update && \
apk upgrade && \
apk add sudo bash libsodium gmp && \
apk add sudo bash libssl1.0 libsodium gmp && \
rm -f /var/cache/apk/* && \
adduser -S tezos && \
echo 'tezos ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/tezos && \
chmod 440 /etc/sudoers.d/tezos && \
chown root:root /etc/sudoers.d/tezos && \
sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
sed -i 's/^Defaults.*requiretty//g' /etc/sudoers
USER tezos
ADD built-bin /usr/local/bin
COPY . /home/tezos
WORKDIR /home/tezos
RUN sudo chown root:root bin/* && \
sudo chmod a+rx bin/* && \
sudo mv bin/* /usr/local/bin && \
rmdir bin
ENTRYPOINT [ "/bin/bash" ]

18
scripts/Dockerfile.build_bin.in
View File

@ -1,13 +1,7 @@
FROM tezos_build:$base_image
FROM $base_name:$base_version
ADD tezos /home/opam/tezos
ENV HOME /home/opam
WORKDIR $HOME/tezos
RUN sudo HOME="$HOME" opam config exec -- make clean && \
sudo rm -fr ~/.opam/log && \
sudo chown -R opam /home/opam
RUN opam config exec -- make
RUN mkdir -p ~/bin && \
(cp tezos-client tezos-node tezos-protocol-compiler tezos-webclient ~/bin || true)
WORKDIR $HOME/bin
COPY . /home/opam/tezos
RUN sudo chown -R opam /home/opam/tezos && \
opam config exec -- make -C tezos -j4 && \
mkdir bin && \
cp tezos/tezos-* bin

19
scripts/Dockerfile.build_deps.in
View File

@ -1,11 +1,8 @@
FROM ocaml/opam:$base_image
COPY install_build_deps.sh /tmp
COPY tezos-deps.opam /tmp/src/tezos-deps.opam
WORKDIR /tmp
RUN opam config exec -- ./install_build_deps.sh pin && rm -fr ~/.opam/log/
USER root
ENV HOME /home/opam
RUN opam config exec -- ./install_build_deps.sh depext && rm -fr ~/.opam/log/
RUN apk add libsodium-dev
USER opam
RUN opam config exec -- ./install_build_deps.sh all && rm -fr ~/.opam/log/
FROM ocaml/opam:alpine-$alpine_version_ocaml-$ocaml_version
COPY scripts/install_build_deps.sh scripts/version.sh scripts/
COPY src/tezos-deps.opam src/
RUN sudo apk add libsodium-dev && \
opam config exec -- ./scripts/install_build_deps.sh all \
rm -fr ~/.opam/log/ \
"$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi

20
scripts/create_build_deps_docker_image.sh Executable file
View File

@ -0,0 +1,20 @@
#! /bin/sh
set -e
set -x
script_dir="$(dirname "$(readlink -f "$0")")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos_build_deps}"
image_version="${2:-latest}"
sed scripts/Dockerfile.build_deps.in \
-e 's/$alpine_version/'"$alpine_version"'/g' \
-e 's/$ocaml_version/'"$ocaml_version"'/g' > Dockerfile
docker build --pull -t "$image_name:$image_version" .
rm Dockerfile

23
scripts/create_docker_builder.sh
View File

@ -1,23 +0,0 @@
#! /bin/sh
set -x
set -e
dir="$(mktemp -d)"
src_dir="$(dirname "$(readlink -f "$0")")"
image_name="${1:-tezos_build}"
base_image="${2:-alpine_ocaml-4.03.0}"
image_version="$3"
docker pull ocaml/opam:"$base_image"
cd "$dir"
cp "$src_dir"/install_build_deps.sh "$dir"
cp "$src_dir"/../src/tezos-deps.opam "$dir"
cp "$src_dir"/Dockerfile.build_deps.in "$dir"
sed Dockerfile.build_deps.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_deps
docker build -f Dockerfile.build_deps -t "$image_name:$base_image$image_version" "$dir"

41
scripts/create_docker_image.sh Executable file
View File

@ -0,0 +1,41 @@
#! /bin/sh
set -e
set -x
script_dir="$(dirname "$(readlink -f "$0")")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. scripts/version.sh
image_name="${1:-tezos}"
image_version="${2:-latest}"
build_deps_image_name="$image_name"_build_deps
echo Building dependencies...
./scripts/create_build_deps_docker_image.sh \
"$build_deps_image_name" "$image_version"
cleanup () {
set +e
echo Cleaning up...
[ -z "$tmp_container" ] || docker rm "$tmp_container"
[ -z "$tmp_image" ] || docker rmi "$tmp_image"
rm -rf Dockerfile bin
}
trap cleanup EXIT INT
sed scripts/Dockerfile.build_bin.in \
-e 's/$base_name/'"$build_deps_image_name"'/g' \
-e 's/$base_version/'"$image_version"'/g' > Dockerfile
echo Building tezos...
tmp_image="$(docker build -q .)"
tmp_container="$(docker run -dit "$tmp_image" true)"
docker cp "$tmp_container":/home/opam/bin/ bin
echo Building minimal docker image...
sed scripts/Dockerfile.binaries.in \
-e 's/$alpine_version/'"$alpine_version"'/g' > Dockerfile
docker build -q -t "$image_name:$image_version" .

41
scripts/create_docker_with_binaries.sh
View File

@ -1,41 +0,0 @@
#! /bin/sh
set -x
set -e
dir="$(mktemp -d)"
src_dir="$(dirname "$(readlink -f "$0")")"
image_name="${1:-tezos_build}"
base_image="${2:-alpine_ocaml-4.03.0}"
image_version="$3"
tezos_build_img="tezos_build:$base_image"
if ! (docker images | grep -- "^tezos_build \+$base_image "); then
echo "Docker image not found: $tezos_build_img" >&2
echo "Aborting" >&2
exit 1
fi
cd "$dir"
git clone "$src_dir"/.. "$dir"/tezos
rm -fr "$dir"/tezos/.git
cp "$src_dir"/Dockerfile.build_bin.in "$dir"
sed Dockerfile.build_bin.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.build_bin
docker build -f Dockerfile.build_bin -t "tezos_build_bin:$base_image$image_version" "$dir"
mkdir -p "$dir"/built-bin
docker run -i --rm -v "$dir"/built-bin:/built-bin "tezos_build_bin:$base_image$image_version" /bin/bash << EOF
sudo cp -v /home/opam/bin/tezos-* /built-bin/
sudo chown opam:nogroup /built-bin/tezos-*
sudo chmod a+rwx /built-bin/tezos-*
EOF
cp "$src_dir"/Dockerfile.binaries.in "$dir"
sed Dockerfile.binaries.in -e 's/$base_image/'"$base_image"'/g' > Dockerfile.binaries
docker build -f Dockerfile.binaries -t "tezos_binaries:$base_image$image_version" "$dir"

10
scripts/install_build_deps.sh
View File

@ -1,10 +1,14 @@
#! /bin/sh
OCAML_VERSION=4.03.0
if [ "$(ocaml -vnum)" != "$OCAML_VERSION" ]; then
script_dir="$(dirname "$(readlink -f "$0")")"
src_dir="$(dirname "$script_dir")"
. "$script_dir/version.sh"
if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then
echo ;
echo " Unexpected compiler version ($(ocaml -vnum))";
echo " You should use ocaml-$OCAML_VERSION.";
echo " You should use ocaml-$ocaml_version.";
echo ;
exit 1;
fi

4
scripts/version.sh Normal file
View File

@ -0,0 +1,4 @@
#! /bin/sh
alpine_version=3.5
ocaml_version=4.03.0

17
src/Makefile
View File

@ -65,7 +65,7 @@ node/updater/proto_environment.cmi: \
@echo OCAMLOPT ${TARGET} $@
@$(OCAMLOPT) -nopervasives -nostdlib -opaque -I tmp -I node/updater -c $<
clean::
partial-clean::
rm -f node/updater/proto_environment.mli
rm -f node/updater/environment_gen
@ -91,7 +91,7 @@ compiler/embedded_cmis.cmx: compiler/embedded_cmis.cmi
compiler/embedded_cmis.ml: ${EMBEDDED_PROTOCOL_LIB_CMIS}
@echo OCAMLRES ${TARGET} $(notdir $@)
@$(OCAMLRES) -format ocaml -o $@ $^
clean::
partial-clean::
rm -f compiler/embedded_cmis.ml
rm -rf tmp
@ -368,7 +368,7 @@ ${TZNODE}: minutils.cmxa utils.cmxa compiler.cmxa node.cmxa ${EMBEDDED_NODE_PROT
@echo LINK $(notdir $@)
@${OCAMLOPT} -linkall -linkpkg ${OCAMLFLAGS} -o $@ $^
clean::
partial-clean::
-rm -f proto/*.cm* proto/*.a
## Embedded protocol modules
@ -399,8 +399,6 @@ proto/client_embedded_proto_%.cmxa: \
$(addprefix -I , ${CLIENT_PROTO_INCLUDES}) \
$@ proto/$*
clean::
rm -f ${TZNODE}
@ -529,7 +527,7 @@ client/embedded/webclient_%.cmx: \
$$(shell find client/embedded/%/webclient/static/)
@$(MAKE) -C client/embedded/$* ../webclient_$*.cmx
clean::
partial-clean::
-for d in $$(ls -d client/embedded/*/) ; do make clean -C $$d ; done
-rm -f client/embedded/*.cm* client/embedded/*.o
@ -601,8 +599,9 @@ clean::
## Cleaning
.PHONY: clean
clean::
.PHONY: clean partial-clean
clean:: partial-clean
partial-clean::
-find \( -name \*.cm\* -or -name \*.cmp -or -name \*.out -or -name \*~ -or -name \*.o -or -name \*.a \) -delete
## Dependencies
@ -652,6 +651,6 @@ compiler/tezos_compiler.cmo compiler/tezos_compiler.cmx: \
@echo OCAMLDEP ${TARGET} $(notdir $^)
@$(OCAMLDEP) $(INCLUDES) $^ > $@
clean::
partial-clean::
-rm -f .depend
-find \( -name \*.mli.deps -or -name \*.ml.deps \) -delete