Merge branch '88-better-docker-script' into 'master'

Resolve "Better docker script"

Closes #88

See merge request !130

.dockerignore

@@ -0,0 +1,67 @@
+.git
+
+tezos-node
+tezos-protocol-compiler
+tezos-client
+tezos-webclient
+tezos-attacker
+
+src/Makefile.local
+
+src/webclient_static.ml
+src/.depend
+
+src/node/updater/environment_gen
+src/node/updater/proto_environment.mli
+src/compiler/embedded_cmis.ml
+
+src/proto/**/_tzbuild
+src/proto/register_client_*.ml
+src/client/embedded/**/_tzbuild
+
+src/client/embedded/demo/.depend
+
+src/client/embedded/bootstrap/.depend
+src/client/embedded/bootstrap/concrete_lexer.ml
+src/client/embedded/bootstrap/concrete_parser.ml
+src/client/embedded/bootstrap/concrete_parser.mli
+src/client/embedded/bootstrap/webclient_proto_static.ml
+src/client/embedded/bootstrap/main.byte
+src/client/embedded/bootstrap/webclient_static/main.js
+src/client/embedded/bootstrap/webclient/browser/main.byte
+src/client/embedded/bootstrap/webclient/static/main.js
+src/client/embedded/bootstrap/webclient/webclient_proto_static.ml
+
+test/.depend
+test/reports
+
+test/test-store
+test/test-state
+test/test-context
+test/test-basic
+test/test-data-encoding
+test/test-p2p-io-scheduler
+test/test-p2p-connection
+test/test-p2p-connection-pool
+test/LOG
+
+**/*~
+**/\#*\#
+
+**/*.[oa]
+**/*.so
+**/*~
+**/*.cm[iaoxt]
+**/*.cmti
+**/*.cmxa
+**/*.cmxs
+**/*.cmp
+**/*.mli.deps
+**/*.ml.deps
+**/*.mli.deps.byte
+**/*.ml.deps.byte
+
+**/bisect*.out
+
+**/*.rej
+**/*.orig

.gitlab-ci.yml

@@ -1,10 +1,10 @@
 variables:
-  ocaml_image: alpine_ocaml-4.03.0
+  image_name: ocp_tezos
-  build_image_name: ocp_tezos_build
+  image_id: ${CI_BUILD_REF_NAME}
+  build_image_name: ${image_name}_build_deps
   build_image_id: ${CI_BUILD_REF}
-  build_image: ${build_image_name}:${ocaml_image}
 
-image: ${build_image}_${build_image_id}
+image: ${build_image_name}:${build_image_id}
 
 stages:
   - build_deps
@@ -21,8 +21,8 @@ build_deps:
   tags:
     - docker_builder
   script:
-    - ./scripts/create_docker_builder.sh
+    - ./scripts/create_build_deps_docker_image.sh
-        ${build_image_name} ${ocaml_image} "_${build_image_id}"
+        ${build_image_name} ${build_image_id}
 
 build:
   stage: build
@@ -137,15 +137,29 @@ test:p2p-connection-pool:
     - build
     - build:test
 
-expurge:
+publish:docker:
-  stage: expurge
+  stage: publish
+  image: ocp:docker
+  tags:
+    - docker_builder
+  script:
+    - ./scripts/create_docker_image.sh ${image_name} ${build_image_id}
+    - docker tag ${image_name}:${build_image_id}
+        "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+    - docker push "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+    - docker rmi "$CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME"
+
+publish:github:
+  image: ocaml/opam
+  stage: publish
   tags:
     - tezos_builder
   only:
     - master@tezos/tezos
   script:
     - echo "${CI_KH}" > ~/.ssh/known_hosts
-    - echo "${CI_PK_GITLAB}" > ~/.ssh/id_rsa
+    - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
     - chmod 400 ~/.ssh/id_rsa
     - rm -fr .git/refs/original
     - git filter-branch --prune-empty --index-filter
@@ -162,23 +176,7 @@ expurge:
                export GIT_AUTHOR_EMAIL="contact@tezos.com" ;
              fi'
          HEAD
-    - git tag ${CI_BUILD_REF}_expurged
+    - git push git@github.com:OCamlPro/tezos.git -f HEAD:master
-    - git push git@gitlab.ocamlpro.com:${CI_PROJECT_PATH}.git
-               -f --tags HEAD:master-expurged
-
-publish:github:
-  image: ocaml/opam:${ocaml_image}
-  stage: publish
-  tags:
-    - tezos_builder
-  only:
-    - master@tezos/tezos
-  script:
-    - echo "${CI_KH}" > ~/.ssh/known_hosts
-    - echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
-    - chmod 400 ~/.ssh/id_rsa
-    - git reset ${CI_BUILD_REF}_expurged
-    - git push git@github.com:tezos/tezos.git -f HEAD:master
 
 cleanup:
   stage: cleanup
@@ -186,7 +184,7 @@ cleanup:
   tags:
     - docker_builder
   script:
-    - docker tag ${build_image}_${build_image_id}
+    - docker tag ${build_image_name}:${build_image_id}
-                 ${build_image}_${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
+                 ${build_image_name}:${CI_PROJECT_NAMESPACE}_${CI_BUILD_REF_NAME}
-    - docker rmi ${build_image}_${build_image_id}
+    - docker rmi ${build_image_name}:${build_image_id}
   when: always

Makefile

@@ -6,9 +6,16 @@ clean:
 	${MAKE} -C src clean
 	${MAKE} -C test clean
 
+partial-clean:
+	${MAKE} -C src partial-clean
+	${MAKE} -C test clean
+
 .PHONY: test
 test:
 	${MAKE} -C test
 
 build-deps:
 	@./scripts/install_build_deps.sh all
+
+docker-image:
+	@./scripts/create_docker_image.sh

scripts/Dockerfile.binaries.in

@@ -0,0 +1,24 @@
+FROM alpine:$alpine_version
+
+LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux"
+
+RUN apk update && \
+    apk upgrade && \
+    apk add sudo bash libssl1.0 libsodium gmp && \
+    rm -f /var/cache/apk/* && \
+    adduser -S tezos && \
+    echo 'tezos ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/tezos && \
+    chmod 440 /etc/sudoers.d/tezos && \
+    chown root:root /etc/sudoers.d/tezos && \
+    sed -i 's/^Defaults.*requiretty//g' /etc/sudoers
+USER tezos
+
+COPY . /home/tezos
+WORKDIR /home/tezos
+
+RUN sudo chown root:root bin/* && \
+    sudo chmod a+rx bin/* && \
+    sudo mv bin/* /usr/local/bin && \
+    rmdir bin
+
+ENTRYPOINT [ "/bin/bash" ]

scripts/Dockerfile.build_bin.in

@@ -0,0 +1,7 @@
+FROM $base_name:$base_version
+
+COPY . /home/opam/tezos
+RUN sudo chown -R opam /home/opam/tezos && \
+    opam config exec -- make -C tezos -j4 && \
+    mkdir bin && \
+    cp tezos/tezos-* bin

scripts/Dockerfile.build_deps.in

@@ -0,0 +1,8 @@
+FROM ocaml/opam:alpine-$alpine_version_ocaml-$ocaml_version
+
+COPY scripts/install_build_deps.sh scripts/version.sh scripts/
+COPY src/tezos-deps.opam src/
+RUN sudo apk add libsodium-dev && \
+    opam config exec -- ./scripts/install_build_deps.sh all \
+    rm -fr ~/.opam/log/ \
+           "$(opam config exec -- ocamlfind query stdlib)"/topdirs.cmi

scripts/create_build_deps_docker_image.sh

@@ -0,0 +1,20 @@
+#! /bin/sh
+
+set -e
+set -x
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+image_name="${1:-tezos_build_deps}"
+image_version="${2:-latest}"
+
+sed scripts/Dockerfile.build_deps.in \
+    -e 's/$alpine_version/'"$alpine_version"'/g' \
+    -e 's/$ocaml_version/'"$ocaml_version"'/g' > Dockerfile
+
+docker build --pull -t "$image_name:$image_version" .
+
+rm Dockerfile

scripts/create_docker_builder.sh

@@ -1,32 +0,0 @@
-#! /bin/sh
-
-set -x
-
-dir=$(mktemp -d)
-cur_dir="$(dirname "$(readlink -f "$0")")"
-
-image_name=${1:=tezos_build}
-ocaml_version=${2:=alpine_ocaml-4.03.0}
-image_version=$3
-
-docker pull ocaml/opam:${ocaml_version}
-
-cp ${cur_dir}/install_build_deps.sh ${dir}
-cp ${cur_dir}/../src/tezos-deps.opam ${dir}
-cat > ${dir}/Dockerfile <<EOF
-FROM ocaml/opam:${ocaml_version}
-COPY install_build_deps.sh /tmp
-COPY tezos-deps.opam /tmp/src/tezos-deps.opam
-RUN cd /tmp && opam config exec -- ./install_build_deps.sh pin \
-            && rm -fr ~/.opam/log/
-USER root
-ENV HOME /home/opam
-RUN cd /tmp && opam config exec -- ./install_build_deps.sh depext \
-            && rm -fr ~/.opam/log/
-RUN apk add libsodium-dev
-USER opam
-RUN cd /tmp && opam config exec -- ./install_build_deps.sh install \
-            && rm -fr ~/.opam/log/
-EOF
-
-docker build -t ${image_name}:${ocaml_version}${image_version} ${dir}

scripts/create_docker_image.sh

@@ -0,0 +1,41 @@
+#! /bin/sh
+
+set -e
+set -x
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+src_dir="$(dirname "$script_dir")"
+cd "$src_dir"
+
+. scripts/version.sh
+image_name="${1:-tezos}"
+image_version="${2:-latest}"
+build_deps_image_name="$image_name"_build_deps
+
+echo Building dependencies...
+./scripts/create_build_deps_docker_image.sh \
+    "$build_deps_image_name" "$image_version"
+
+cleanup () {
+    set +e
+    echo Cleaning up...
+    [ -z "$tmp_container" ] || docker rm "$tmp_container"
+    [ -z "$tmp_image" ] || docker rmi "$tmp_image"
+    rm -rf Dockerfile bin
+}
+trap cleanup EXIT INT
+
+sed scripts/Dockerfile.build_bin.in \
+    -e 's/$base_name/'"$build_deps_image_name"'/g' \
+    -e 's/$base_version/'"$image_version"'/g'  > Dockerfile
+
+echo Building tezos...
+tmp_image="$(docker build -q .)"
+tmp_container="$(docker run -dit "$tmp_image" true)"
+
+docker cp "$tmp_container":/home/opam/bin/ bin
+
+echo Building minimal docker image...
+sed scripts/Dockerfile.binaries.in \
+    -e 's/$alpine_version/'"$alpine_version"'/g' > Dockerfile
+docker build -q -t "$image_name:$image_version" .

scripts/install_build_deps.sh

@@ -1,39 +1,40 @@
-#!/bin/sh
+#! /bin/sh
 
-if ! [ -f 'src/tezos-deps.opam' ]; then
+script_dir="$(dirname "$(readlink -f "$0")")"
-    echo
+src_dir="$(dirname "$script_dir")"
-    echo "   Please run from the project's root directory. Aborting."
+
-    echo
+. "$script_dir/version.sh"
-    exit 1
-fi
 
-ocaml_version=4.03.0
 if [ "$(ocaml -vnum)" != "$ocaml_version" ]; then
-    echo
+  echo ;
-    echo "   Unexpected compiler version ($(ocaml -vnum))"
+  echo "   Unexpected compiler version ($(ocaml -vnum))";
-    echo "   You should use ocaml-$ocaml_version."
+  echo "   You should use ocaml-$ocaml_version.";
-    echo
+  echo ;
-    exit 1
+  exit 1;
 fi
 
 cmd="$1"
 if [ -z "$cmd" ]; then cmd=all; fi
 
-case "$cmd" in
+pin=false
+depext=false
+install=false
+
+case $cmd in
     pin)
-        pin=yes
+	pin=true
-        ;;
+	;;
     depext)
-        depext=yes
+	depext=true
-        ;;
+	;;
     install)
-        install=yes
+	install=true
-        ;;
+	;;
     all)
-        pin=yes
+	pin=true
-        depext=yes
+	depext=true
-        install=yes
+	install=true
-        ;;
+	;;
     *)
         echo "Unknown command '$cmd'."
         echo "Usage: $0 [pin|depext|install|all|]"
@@ -43,7 +44,7 @@ esac
 set -e
 set -x
 
-if ! [ -z "$pin" ]; then
+if "$pin"; then
     opam pin --yes remove --no-action --dev-repo ocplib-resto || true
     opam pin --yes add --no-action --dev-repo sodium
     opam pin --yes add --no-action --dev-repo ocp-ocamlres
@@ -56,15 +57,15 @@ if ! [ -z "$pin" ]; then
     opam pin --yes add --no-action tezos-deps src
 fi
 
-if ! [ -z "$depext" ]; then
+if "$depext"; then
-    ## In our CI, this rule is executed as user 'root'
-    ## The other rules are executed as user 'opam'.
     opam list --installed depext || opam install depext
-    opam depext tezos-deps
+    opam depext $DEPEXTOPT tezos-deps
 fi
 
-if ! [ -z "$install" ]; then
+if "$install"; then
-    opam install tezos-deps
+    if opam list --installed tezos-deps ; then
-    ## This seems broken in the current opam-repo (2016-12-09)
+	opam upgrade $(opam list -s --required-by tezos-deps | grep -ve '^ocaml *$')
-    ## opam install --build-test tezos-deps
+    else
+	opam install tezos-deps
+    fi
 fi

scripts/version.sh

@@ -0,0 +1,4 @@
+#! /bin/sh
+
+alpine_version=3.5
+ocaml_version=4.03.0

src/Makefile

@@ -65,7 +65,7 @@ node/updater/proto_environment.cmi: \
 	@echo OCAMLOPT ${TARGET} $@
 	@$(OCAMLOPT) -nopervasives -nostdlib -opaque -I tmp -I node/updater -c $<
 
-clean::
+partial-clean::
 	rm -f node/updater/proto_environment.mli
 	rm -f node/updater/environment_gen
 
@@ -91,7 +91,7 @@ compiler/embedded_cmis.cmx: compiler/embedded_cmis.cmi
 compiler/embedded_cmis.ml: ${EMBEDDED_PROTOCOL_LIB_CMIS}
 	@echo OCAMLRES ${TARGET} $(notdir $@)
 	@$(OCAMLRES) -format ocaml -o $@ $^
-clean::
+partial-clean::
 	rm -f compiler/embedded_cmis.ml
 	rm -rf tmp
 
@@ -368,7 +368,7 @@ ${TZNODE}: minutils.cmxa utils.cmxa compiler.cmxa node.cmxa ${EMBEDDED_NODE_PROT
 	@echo LINK $(notdir $@)
 	@${OCAMLOPT} -linkall -linkpkg ${OCAMLFLAGS} -o $@ $^
 
-clean::
+partial-clean::
 	-rm -f proto/*.cm* proto/*.a
 
 ## Embedded protocol modules
@@ -399,8 +399,6 @@ proto/client_embedded_proto_%.cmxa: \
 	   $(addprefix -I , ${CLIENT_PROTO_INCLUDES}) \
 	   $@ proto/$*
 
-
-
 clean::
 	rm -f ${TZNODE}
 
@@ -529,7 +527,7 @@ client/embedded/webclient_%.cmx: \
   $$(shell find client/embedded/%/webclient/static/)
 	@$(MAKE) -C client/embedded/$* ../webclient_$*.cmx
 
-clean::
+partial-clean::
 	-for d in $$(ls -d client/embedded/*/) ; do make clean -C $$d ; done
 	-rm -f client/embedded/*.cm* client/embedded/*.o
 
@@ -601,8 +599,9 @@ clean::
 
 ## Cleaning
 
-.PHONY: clean
+.PHONY: clean partial-clean
-clean::
+clean:: partial-clean
+partial-clean::
 	-find \( -name \*.cm\* -or -name \*.cmp -or -name \*.out -or -name \*~ -or -name \*.o -or -name \*.a \) -delete
 
 ## Dependencies
@@ -652,6 +651,6 @@ compiler/tezos_compiler.cmo compiler/tezos_compiler.cmx: \
 	@echo OCAMLDEP ${TARGET} $(notdir $^)
 	@$(OCAMLDEP) $(INCLUDES) $^ > $@
 
-clean::
+partial-clean::
 	-rm -f .depend
 	-find \( -name \*.mli.deps -or -name \*.ml.deps \) -delete