From e66417992757ad7b0a67398945cae17090aeb019 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gr=C3=A9goire=20Henry?= <gregoire.henry@tezos.com>
Date: Tue, 23 Jan 2018 13:51:36 +0100
Subject: [PATCH] Refactor: CI: check opam packages

---
 .gitlab-ci.yml              | 159 +++++++++++++++++++-----------------
 scripts/check_opam_test.sh  |  36 ++++++++
 scripts/update_opam_test.sh |  30 +++++++
 src/lib_rpc/tezos-rpc.opam  |   1 +
 4 files changed, 151 insertions(+), 75 deletions(-)
 create mode 100755 scripts/check_opam_test.sh
 create mode 100755 scripts/update_opam_test.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3c12e6983..379422267 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -91,6 +91,11 @@ build:
   dependencies:
     - build
 
+test:opam:
+  <<: *test_definition
+  script:
+    - ./scripts/check_opam_test.sh "$CI_PROJECT_DIR/$CI_CONFIG_PATH"
+
 test:ocp-indent:
   <<: *test_definition
   script:
@@ -178,180 +183,184 @@ test:proto:sandbox:
   tags:
     - gitlab-org
 
+##BEGIN_OPAM##
+opam:00:tezos-stdlib:
+  <<: *opam_definition
+  variables:
+    package: tezos-stdlib
+
 opam:01:ocplib-resto:
   <<: *opam_definition
   variables:
     package: ocplib-resto
 
-opam:02:ocplib-resto-directory:
-  <<: *opam_definition
-  variables:
-    package: ocplib-resto-directory
-
-opam:03:ocplib-resto-cohttp:
-  <<: *opam_definition
-  variables:
-    package: ocplib-resto-cohttp
-
-opam:04:ocplib-resto-json:
-  <<: *opam_definition
-  variables:
-    package: ocplib-resto-json
-
-opam:05:ocplib-ezresto:
-  <<: *opam_definition
-  variables:
-    package: ocplib-ezresto
-
-opam:06:ocplib-ezresto-directory:
-  <<: *opam_definition
-  variables:
-    package: ocplib-ezresto-directory
-
-opam:07:tezos-stdlib:
-  <<: *opam_definition
-  variables:
-    package: tezos-stdlib
-
-opam:08:tezos-data-encoding:
+opam:02:tezos-data-encoding:
   <<: *opam_definition
   variables:
     package: tezos-data-encoding
 
-opam:09:tezos-error-monad:
+opam:03:ocplib-resto-directory:
+  <<: *opam_definition
+  variables:
+    package: ocplib-resto-directory
+
+opam:04:tezos-error-monad:
   <<: *opam_definition
   variables:
     package: tezos-error-monad
 
-opam:10:tezos-stdlib-lwt:
+opam:05:tezos-rpc:
+  <<: *opam_definition
+  variables:
+    package: tezos-rpc
+
+opam:06:tezos-stdlib-lwt:
   <<: *opam_definition
   variables:
     package: tezos-stdlib-lwt
 
-opam:11:tezos-crypto:
+opam:07:tezos-crypto:
   <<: *opam_definition
   variables:
     package: tezos-crypto
 
-opam:12:tezos-base:
+opam:08:tezos-base:
   <<: *opam_definition
   variables:
     package: tezos-base
 
-opam:13:tezos-node-p2p-base:
-  <<: *opam_definition
-  variables:
-    package: tezos-node-p2p-base
-
-opam:14:tezos-node-services:
-  <<: *opam_definition
-  variables:
-    package: tezos-node-services
-
-opam:15:tezos-protocol-environment-sigs:
+opam:09:tezos-protocol-environment-sigs:
   <<: *opam_definition
   variables:
     package: tezos-protocol-environment-sigs
 
-opam:16:irmin-leveldb:
+opam:10:irmin-leveldb:
   <<: *opam_definition
   variables:
     package: irmin-leveldb
 
-opam:17:tezos-micheline:
+opam:11:tezos-micheline:
   <<: *opam_definition
   variables:
     package: tezos-micheline
 
-opam:18:tezos-rpc-http:
-  <<: *opam_definition
-  variables:
-    package: tezos-rpc-http
-
-opam:19:tezos-protocol-compiler:
+opam:12:tezos-protocol-compiler:
   <<: *opam_definition
   variables:
     package: tezos-protocol-compiler
 
-opam:20:tezos-storage:
+opam:13:tezos-storage:
   <<: *opam_definition
   variables:
     package: tezos-storage
 
-opam:21:tezos-node-p2p:
+opam:14:ocplib-resto-cohttp:
   <<: *opam_definition
   variables:
-    package: tezos-node-p2p
+    package: ocplib-resto-cohttp
 
-opam:22:tezos-protocol-updater:
+opam:15:tezos-p2p:
+  <<: *opam_definition
+  variables:
+    package: tezos-p2p
+
+opam:16:tezos-protocol-updater:
   <<: *opam_definition
   variables:
     package: tezos-protocol-updater
 
-opam:23:tezos-node-shell:
+opam:17:tezos-rpc-http:
   <<: *opam_definition
   variables:
-    package: tezos-node-shell
+    package: tezos-rpc-http
 
-opam:24:tezos-embedded-protocol-alpha:
+opam:18:tezos-shell-services:
+  <<: *opam_definition
+  variables:
+    package: tezos-shell-services
+
+opam:19:tezos-shell:
+  <<: *opam_definition
+  variables:
+    package: tezos-shell
+
+opam:20:tezos-embedded-protocol-alpha:
   <<: *opam_definition
   variables:
     package: tezos-embedded-protocol-alpha
 
-opam:25:tezos-embedded-protocol-demo:
+opam:21:tezos-embedded-protocol-demo:
   <<: *opam_definition
   variables:
     package: tezos-embedded-protocol-demo
 
-opam:26:tezos-embedded-protocol-genesis:
+opam:22:tezos-embedded-protocol-genesis:
   <<: *opam_definition
   variables:
     package: tezos-embedded-protocol-genesis
 
-opam:27:tezos-client-base:
+opam:23:tezos-client-base:
   <<: *opam_definition
   variables:
     package: tezos-client-base
 
-opam:28:tezos-embedded-client-alpha:
+opam:24:ocplib-resto-json:
+  <<: *opam_definition
+  variables:
+    package: ocplib-resto-json
+
+opam:25:tezos-embedded-client-alpha:
   <<: *opam_definition
   variables:
     package: tezos-embedded-client-alpha
 
-opam:29:tezos-embedded-client-genesis:
+opam:26:tezos-embedded-client-genesis:
   <<: *opam_definition
   variables:
     package: tezos-embedded-client-genesis
 
-opam:30:tezos-protocol-demo:
+opam:27:ocplib-ezresto:
   <<: *opam_definition
   variables:
-    package: tezos-protocol-demo
+    package: ocplib-ezresto
 
-opam:31:tezos-client:
+opam:28:tezos-client:
   <<: *opam_definition
   variables:
     package: tezos-client
 
-opam:32:tezos-node:
+opam:29:tezos-node:
   <<: *opam_definition
   variables:
     package: tezos-node
 
-opam:33:tezos-protocol-alpha:
+opam:30:ocplib-ezresto-directory:
   <<: *opam_definition
   variables:
-    package: tezos-protocol-alpha
+    package: ocplib-ezresto-directory
 
-opam:34:tezos-protocol-genesis:
+opam:31:tezos-test:
+  <<: *opam_definition
+  variables:
+    package: tezos-test
+
+opam:32:tezos-protocol-genesis:
   <<: *opam_definition
   variables:
     package: tezos-protocol-genesis
 
-opam:35:tezos-test:
+opam:33:tezos-protocol-demo:
   <<: *opam_definition
   variables:
-    package: tezos-test
+    package: tezos-protocol-demo
+
+opam:34:tezos-protocol-alpha:
+  <<: *opam_definition
+  variables:
+    package: tezos-protocol-alpha
+
+
+##END_OPAM##
 
 ## Publishing (small) docker images with tezos binaries
 
diff --git a/scripts/check_opam_test.sh b/scripts/check_opam_test.sh
new file mode 100755
index 000000000..bd55a8b4c
--- /dev/null
+++ b/scripts/check_opam_test.sh
@@ -0,0 +1,36 @@
+#! /bin/sh
+
+set -e
+
+script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
+src_dir="$(dirname "$script_dir")"
+
+opams=$(find "$src_dir" -name \*.opam -print)
+
+yml="${1:-$src_dir/.gitlab-ci.yml}"
+
+missing=
+for opam in $opams; do
+    file=$(basename $opam)
+    package=${file%.opam}
+    if ! grep -qe "opam:..:$package:\$" "$yml"; then
+        missing=yes
+        echo "Missing test for package '$package'."
+    fi
+done
+
+tested=$(grep -e '^opam:..:tezos-.*:$' "$yml" | cut -d: -f3)
+for package in $tested; do
+    found=$(find "$src_dir" -name $package.opam | wc -l 2>&1)
+    if [ $found != 1 ] ; then
+        missing=yes
+        echo "Test for unknown package '$package'."
+    fi
+done
+
+if ! [ -z "$missing" ]; then
+    echo
+    echo "You should update .gitlab-ci.yml by running: ./scripts/update_opam_test.sh"
+    echo
+    exit 1
+fi
diff --git a/scripts/update_opam_test.sh b/scripts/update_opam_test.sh
new file mode 100755
index 000000000..09499b8ae
--- /dev/null
+++ b/scripts/update_opam_test.sh
@@ -0,0 +1,30 @@
+#! /bin/sh
+
+set -e
+
+script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
+src_dir="$(dirname "$script_dir")"
+
+. "$script_dir/opam-pin.sh"
+
+tmp=$(mktemp)
+
+sed -z 's/^\(.*##BEGIN_OPAM##\n\).*\(\n##END_OPAM##.*\)$/\1/' "$src_dir/.gitlab-ci.yml" > $tmp
+
+cpt=0
+for package in $packages; do
+    num=$(printf "%02d" $cpt)
+    cpt=$((cpt+1))
+    cat >> $tmp <<EOF
+opam:$num:$package:
+  <<: *opam_definition
+  variables:
+    package: $package
+
+EOF
+done
+
+sed -z 's/^\(.*##BEGIN_OPAM##\n\).*\(\n##END_OPAM##.*\)$/\2/' "$src_dir/.gitlab-ci.yml" >> $tmp
+
+mv $tmp "$src_dir/.gitlab-ci.yml"
+
diff --git a/src/lib_rpc/tezos-rpc.opam b/src/lib_rpc/tezos-rpc.opam
index d74a8f5c5..8f7b34e13 100644
--- a/src/lib_rpc/tezos-rpc.opam
+++ b/src/lib_rpc/tezos-rpc.opam
@@ -9,6 +9,7 @@ license: "unreleased"
 depends: [
   "ocamlfind" { build }
   "jbuilder" { build & >= "1.0+beta15" }
+  "tezos-error-monad"
   "tezos-data-encoding"
   "ocplib-resto"
   "ocplib-resto-directory"