Opam: update dependencies

.gitlab-ci.yml

@@ -1,6 +1,6 @@
 variables:
   ## Please update `scripts/version.sh` accordingly
-  build_deps_image_version: 63f06ab145e22aef4cc55b5b9557756d4c868843
+  build_deps_image_version: 4200cbebcae75562d79898a8e11e9f4dc237cc08
   build_deps_image_name: registry.gitlab.com/tezos/opam-repository
   public_docker_image_name: docker.io/${CI_PROJECT_PATH}
 
@@ -51,7 +51,6 @@ build:
   <<: *build_definition
   script:
     - make all
-    - make build-test
   artifacts:
     paths:
     - _build
@@ -168,6 +167,8 @@ test:documentation:
   <<: *test_definition
   script:
     - sudo apk add --no-cache py3-sphinx py3-sphinx_rtd_theme
+    - sudo pip3 uninstall 'idna' --yes ## Fix up dependencies in alpine:3.8
+    - sudo pip3 install 'idna<2.7'
     - sudo ln -s /usr/bin/sphinx-build-3 /usr/bin/sphinx-build
     - make doc-html
 

scripts/opam-check.sh

@@ -5,26 +5,33 @@ src_dir="$(dirname "$script_dir")"
 
 . "$script_dir"/version.sh
 
-. "$script_dir"/opam-pin.sh
+opams=$(find "$src_dir/vendors" "$src_dir/src" -name \*.opam -print)
 
 echo "## Checking installed dependencies..."
 echo
 
-if ! opam install $packages --deps-only --with-test --show-actions | grep "Nothing to do." > /dev/null 2>&1 ; then
+if ! opam install $opams --deps-only --with-test --show-actions | grep "Nothing to do." > /dev/null 2>&1 ; then
     echo
     echo 'Failure! Missing actions:'
     echo
     opam install $packages --deps-only --with-test --show-actions
     echo
-    echo 'Running `./scripts/update_opam_repo.sh`'
+    echo 'Failed! Please read the doc in `./scripts/update_opam_repo.sh` and act accordingly.'
-    echo
-    ./scripts/update_opam_repo.sh
-    echo
-    echo 'Result:'
     echo
+    exit 1
+fi
+
+echo '## Running `./scripts/update_opam_repo.sh`'
+echo
+./scripts/update_opam_repo.sh
+
+if [ -n "$(cat opam_repo.patch)" ] ; then
+
+    echo "##################################################"
     cat opam_repo.patch
-    echo
+    echo "##################################################"
-    echo 'Failed! Please run: `./scripts/update_opam_repo.sh` and follow the instructions.'
+
+    echo 'Failed! The variables `opam_repository_tag` and `full_opam_repository_tag` are not synchronized. Please read the doc in `./scripts/update_opam_repo.sh` and act accordingly.'
     echo
     exit 1
 fi

scripts/update_opam_repo.sh

@@ -1,5 +1,37 @@
 #! /bin/sh
 
+# Update the repository of opam packages used by tezos.  Tezos uses a
+# private, epurated, opam repository to store all its
+# dependencies. This is generated by the official opam repository
+# (branch 2.0.0) and then filtered using opam admin to include only
+# the cone of tezos dependencies.  This repository is then used to
+# create the based opam image used by the CI to compile tezos and to
+# generate the docker images.  From time to time, when it is necessary
+# to update a dependency, this repository should mannually
+# refreshed. This script takes care of generating a patch for the
+# private opam tezos repository. This patch must be applied manually
+# w.r.t. the master branch. The procedure is as follows :
+#
+# 1. Update the varialbe `full_opam_repository_tag` in `version.sh` to
+#    a commit hash from the 2.0.0 branch of the official
+#    opam-repository. All the required packages will be extracted from
+#    this snapshot fo the repo.
+#
+# 2. Run this script, it will generate a file `opam_repo.patch`
+#
+# 3. Review the patch.
+#
+# 4. In the tezos opam-repository, create a new branch from master and
+#    apply this patch. Push the patch and create a merge request. A
+#    new docker image with all the prebuilt dependencies will be
+#    created by the CI.
+#
+# 5. Update the variable `opam_repository_tag` in files
+#    `scripts/version.sh` and `.gitlab-ci.yml` with the hash of the
+#    newly created commit in `tezos/opam-repository`.
+#
+# 6. Enjoy your new dependencies
+
 set -e
 
 target="$(pwd)"/opam_repo.patch tmp_dir=$(mktemp -dt tezos_deps_opam.XXXXXXXX)
@@ -20,7 +52,8 @@ src_dir="$(dirname "$script_dir")"
 opams=$(find "$src_dir/vendors" "$src_dir/src" -name \*.opam -print)
 
 ## Full snapshot of the opam repository
-git clone "$opam_repository_url" -b full "$tmp_dir"
+git clone https://github.com/ocaml/opam-repository -b 2.0.0 "$tmp_dir"
+
 
 ## Adding the various tezos packages
 packages=
@@ -50,6 +83,8 @@ done
 
 ## Filtering unrequired packages
 cd $tmp_dir
+
+git reset --hard "$full_opam_repository_tag"
 opam admin filter --yes \
      --resolve $packages,ocaml,ocaml-base-compiler,odoc,opam-depext
 
@@ -69,6 +104,8 @@ done
 opam admin add-hashes sha256 sha512
 
 ## Generating the diff!
+git remote add tezos $opam_repository_url
+git fetch tezos
 git reset "$opam_repository_tag"
 git add packages
 git diff HEAD -- packages > "$target"

scripts/version.sh

@@ -4,9 +4,10 @@
 ## `lib.protocol-compiler/tezos-protocol-compiler.opam`
 
 ocaml_version=4.06.1
-opam_version=2.0.0~rc3
+opam_version=2.0.0
 
 ## Please update `.gitlab-ci.yml` accordingly
-opam_repository_tag=63f06ab145e22aef4cc55b5b9557756d4c868843
+opam_repository_tag=4200cbebcae75562d79898a8e11e9f4dc237cc08
+full_opam_repository_tag=2274027270cc192b88ecd7b9d4035dba66b2cffd
 opam_repository_url=https://gitlab.com/tezos/opam-repository.git
 opam_repository=$opam_repository_url\#$opam_repository_tag