From 78c6cbae68ba610502025ddb89b44ebb5175b45e Mon Sep 17 00:00:00 2001
From: Matej Sima <matej.sima@gmail.com>
Date: Mon, 12 Aug 2019 10:41:53 +0200
Subject: [PATCH] Add build & packaging scripts for multiplatform ligo binary
 distribution

---
 .dockerignore                                 |  6 ++
 .gitlab-ci.yml                                | 79 +++++++++++++------
 dist/.gitignore                               |  4 +
 .../distribution/debian/distribute.Dockerfile |  0
 docker/distribution/debian/package.Dockerfile | 32 ++++++++
 .../generic/build.Dockerfile}                 | 11 +--
 scripts/build_docker_image.sh                 |  3 +-
 scripts/distribution/generic/build.sh         | 11 +++
 scripts/distribution/generic/env_variables.sh | 12 +++
 scripts/distribution/generic/package.sh       | 20 +++++
 scripts/distribution/generic/parameters.sh    | 32 ++++++++
 11 files changed, 179 insertions(+), 31 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 dist/.gitignore
 create mode 100644 docker/distribution/debian/distribute.Dockerfile
 create mode 100644 docker/distribution/debian/package.Dockerfile
 rename docker/{Dockerfile => distribution/generic/build.Dockerfile} (81%)
 create mode 100755 scripts/distribution/generic/build.sh
 create mode 100755 scripts/distribution/generic/env_variables.sh
 create mode 100755 scripts/distribution/generic/package.sh
 create mode 100644 scripts/distribution/generic/parameters.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..ccf03db24
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,6 @@
+dist
+_opam
+_build
+docker
+gitlab-pages
+Makefile
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1fa3cd9ec..19e5d7f6a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,12 +1,27 @@
 # TODO: remove this as submodules aren't used anymore.
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
+  build_binary_script: "./scripts/distribution/generic/build.sh"
+  package_binary_script: "./scripts/distribution/generic/package.sh"
 
 stages:
   - build_docker
-  - build_and_deploy_docker
-  - build_and_deploy_website
   - test
+  - build_and_deploy_docker
+  - build_and_package_binaries
+  - build_and_deploy_website
+
+.build_binary: &build_binary
+  stage: build_and_package_binaries
+  script:
+    - $build_binary_script "$target_os_family" "$target_os" "$target_os_version"
+    - $package_binary_script "$target_os_family" "$target_os" "$target_os_version"
+  artifacts:
+    paths:
+      - dist/package/**/*
+  only:
+    - master
+    - dev
 
 .website_build: &website_build
   stage: build_and_deploy_website
@@ -71,30 +86,12 @@ local-dune-job:
     - scripts/build_ligo_local.sh
     - dune build @ligo-test
 
-remote-repo-job:
-  <<: *before_script
-  stage: test
-  script:
-    # Add repository
-    - opam repository add ligo-repository https://gitlab.com/ligolang/ligo.git
-    - eval $(opam config env)
-    - opam install -y ligo
-    # Used in the IDE
-    #- opam install -y user-setup
-    #- opam install -y merlin
-    #- opam install -y ocp-indent
-    #- opam user-setup install
-  only:
-    - master
-
-# TODO: uncomment this
-
 # Run a docker build without publishing to the registry
 build-current-docker-image:
   stage: build_docker
   <<: *docker
   script:
-    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
+    - sh scripts/build_docker_image.sh
     - sh scripts/test_cli.sh
   except:
     - master
@@ -106,13 +103,51 @@ build-and-publish-latest-docker-image:
   stage: build_and_deploy_docker
   <<: *docker
   script:
-    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
+    - sh scripts/build_docker_image.sh
     - sh scripts/test_cli.sh
     - docker login -u $LIGO_REGISTRY_USER -p $LIGO_REGISTRY_PASSWORD
     - docker push $LIGO_REGISTRY_IMAGE:next
   only:
     - dev
 
+# It'd be a good idea to generate those jobs dynamically,
+# based on desired targets
+build-and-package-debian-9:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "debian"
+    target_os_version: "9"
+  <<: *build_binary
+
+build-and-package-debian-10:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "debian"
+    target_os_version: "10"
+  <<: *build_binary
+
+build-and-package-ubuntu-18-04:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "ubuntu"
+    target_os_version: "18.04"
+  <<: *build_binary
+
+build-and-package-ubuntu-19-04:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "ubuntu"
+    target_os_version: "19.04"
+  <<: *build_binary
+
 # Pages are deployed from both master & dev, be careful not to override 'next'
 # in case something gets merged into 'dev' while releasing.
 pages:
diff --git a/dist/.gitignore b/dist/.gitignore
new file mode 100644
index 000000000..86d0cb272
--- /dev/null
+++ b/dist/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
diff --git a/docker/distribution/debian/distribute.Dockerfile b/docker/distribution/debian/distribute.Dockerfile
new file mode 100644
index 000000000..e69de29bb
diff --git a/docker/distribution/debian/package.Dockerfile b/docker/distribution/debian/package.Dockerfile
new file mode 100644
index 000000000..440496521
--- /dev/null
+++ b/docker/distribution/debian/package.Dockerfile
@@ -0,0 +1,32 @@
+ARG targetBuildImage
+FROM ${targetBuildImage}
+ARG version
+
+RUN apt-get update -qq
+RUN apt-get -y -qq install \
+    dput \
+    devscripts
+
+RUN mkdir /package && mkdir /package/bin && mkdir /package/DEBIAN && mkdir /package/dist
+# @TODO: make the binary path configurable
+RUN cp /home/opam/.opam/4.07/bin/ligo /package/bin/ligo
+
+# @TODO: inherit version (and other details) from the ligo opam package definition
+# In our case we're using the version field to name our package accordingly, 
+# however this is most likely not ideal
+# Also, the architecture field should not be 'all' but rather specific instead.
+RUN echo "Package: ligo\n\
+Version: $version\n\
+Architecture: all\n\
+Maintainer: info@ligolang.org\n\
+Depends: libev-dev, perl, pkg-config, libgmp-dev, libhidapi-dev, m4, libcap-dev, bubblewrap, rsync\n\
+Homepage: http://ligolang.org\n\
+Description: LIGO is a statically typed high-level smart-contract language that compiles down to Michelson." >> /package/DEBIAN/control
+
+RUN cd /package/dist && dpkg-deb --build /package .
+
+# Test the package
+# Optionally this could/should be done in a more pristine docker environment - in a separate image
+RUN apt-get update -qq
+RUN apt-get -y -f install "/package/dist/$(ls /package/dist)"
+RUN ligo --help
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/distribution/generic/build.Dockerfile
similarity index 81%
rename from docker/Dockerfile
rename to docker/distribution/generic/build.Dockerfile
index 057bd6436..e183fc683 100644
--- a/docker/Dockerfile
+++ b/docker/distribution/generic/build.Dockerfile
@@ -1,5 +1,7 @@
-# At the moment, this really means 4.07.1
-FROM ocaml/opam2:4.07
+ARG target
+FROM ocaml/opam2:${target}
+
+RUN opam switch 4.07 && eval $(opam env)
 
 USER root
 
@@ -10,7 +12,6 @@ USER root
 # because the currently checkout out version (from git) will be used
 # to build the image
 ADD . /ligo
-
 # Set the current working directory to /ligo for
 # the upcoming scripts
 WORKDIR /ligo
@@ -18,10 +19,6 @@ WORKDIR /ligo
 # Install required native dependencies
 RUN sh scripts/install_native_dependencies.sh
 
-# Install OPAM
-# TODO: or scripts/install_build_environment.sh ?
-RUN sh scripts/install_opam.sh
-
 # Add tezos repository
 RUN sh scripts/setup_repos.sh
 
diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh
index 273fa92c6..21e3a2bf5 100755
--- a/scripts/build_docker_image.sh
+++ b/scripts/build_docker_image.sh
@@ -1,4 +1,3 @@
 #!/bin/sh
 set -e
-
-docker build -t ligolang/ligo -f docker/Dockerfile .
+docker build --build-arg target="4.07" -t "${LIGO_REGISTRY_IMAGE_BUILD:-ligolang/ligo}:next" -f ./docker/distribution/generic/build.Dockerfile .
diff --git a/scripts/distribution/generic/build.sh b/scripts/distribution/generic/build.sh
new file mode 100755
index 000000000..7f4a89e9a
--- /dev/null
+++ b/scripts/distribution/generic/build.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+dockerfile_name="build"
+# Generic dockerfile
+dockerfile="./docker/distribution/generic/build.Dockerfile"
+. ./scripts/distribution/generic/parameters.sh
+
+echo "Building LIGO for $target"
+echo "Using Dockerfile: $dockerfile"
+echo "Tagging as: $tag_build\n"
+docker build --build-arg target="$target" -t "$tag_build" -f "$dockerfile" .
\ No newline at end of file
diff --git a/scripts/distribution/generic/env_variables.sh b/scripts/distribution/generic/env_variables.sh
new file mode 100755
index 000000000..7279e4cbe
--- /dev/null
+++ b/scripts/distribution/generic/env_variables.sh
@@ -0,0 +1,12 @@
+# This file is a substitute for env variables configured in the CI
+# in case you want to run the "CI scripts" on your own
+# You can load the following variables using:
+# source ./scripts/distribution/generic/env_variables.sh
+
+export LIGO_REGISTRY_IMAGE_BASE_NAME="ligolang/ligo"
+# packages build locally are tagget by the 'short' commit hash,
+# instead of the build/job/pipeline ID as in the CI to avoid possible confusion
+# ligo_incrementing-id_commit-hash
+export CI_JOB_ID="0"
+export CI_COMMIT_SHORT_SHA="$(git rev-parse --short HEAD)"
+export LIGO_DIST_DIR="./dist"
diff --git a/scripts/distribution/generic/package.sh b/scripts/distribution/generic/package.sh
new file mode 100755
index 000000000..b00b61841
--- /dev/null
+++ b/scripts/distribution/generic/package.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+dockerfile_name="package"
+dockerfile=""
+. ./scripts/distribution/generic/parameters.sh
+
+if [ -n "$dockerfile" ]; then
+    echo "Packaging LIGO for $target"
+    echo "Using Dockerfile: $dockerfile"
+    echo "Using pre-built image: $tag_build"
+    echo "Version: $version\n"
+
+    # Build the package
+    docker build --build-arg targetBuildImage="$tag_build" --build-arg version="$version" -t "$tag_package" -f "$dockerfile" .
+    # Copy the package to host's (our own) file system
+    mkdir -p "$PWD/dist/package/$target"
+    docker run -v $PWD:$PWD "$tag_package" cp -r /package/dist/. "$PWD/dist/package/$target"
+else
+    echo "Dockerfile not found for target: $target"
+fi
\ No newline at end of file
diff --git a/scripts/distribution/generic/parameters.sh b/scripts/distribution/generic/parameters.sh
new file mode 100644
index 000000000..436b48bf0
--- /dev/null
+++ b/scripts/distribution/generic/parameters.sh
@@ -0,0 +1,32 @@
+# This script accepts three arguments, os family, os and it's version,
+# which are subsequently used to fetch the respective docker
+# image from the ocaml/infrastructure project.
+#
+# https://github.com/ocaml/infrastructure/wiki/Containers#selecting-linux-distributions
+target_os_family=$1
+target_os=$2
+target_os_version=$3
+
+# Variables configured at the CI level
+dist="$LIGO_DIST_DIR"
+version="$(echo $CI_JOB_ID)-$(echo $CI_COMMIT_SHORT_SHA)"
+ci_job_id="$CI_JOB_ID"
+
+# Image names for building & packaging
+target="$target_os-$target_os_version"
+tag_build="$LIGO_REGISTRY_IMAGE_BASE_NAME-build-$target:$version"
+tag_package="$LIGO_REGISTRY_IMAGE_BASE_NAME-package-$target:$version"
+tag_package_dockerized="$LIGO_REGISTRY_IMAGE_BASE_NAME-$target:$version"
+
+
+# Check if there's a specific dockerfile override for 
+# the current target_os_family (e.g. debian-ish distros) or target_os (ubuntu, xubuntu, ...) and use it if there is one
+target_os_family_specific_dockerfile="./docker/distribution/$target_os_family/$dockerfile_name.Dockerfile"
+if test -f "$target_os_family_specific_dockerfile"; then
+    dockerfile="$target_os_family_specific_dockerfile"
+fi
+
+target_os_specific_dockerfile="./docker/distribution/$target_os_family/$target_os/$dockerfile_name.Dockerfile"
+if test -f "$target_os_specific_dockerfile"; then
+    dockerfile="$target_os_specific_dockerfile"
+fi
\ No newline at end of file