Merge branch 'feature/LIGO-27-improve-packaging-and-distribution' into 'dev'

Add build & packaging scripts for multiplatform ligo binary distribution

See merge request ligolang/ligo!114

.dockerignore

@@ -0,0 +1,6 @@
+dist
+_opam
+_build
+docker
+gitlab-pages
+Makefile

.gitlab-ci.yml

@@ -1,12 +1,27 @@
 # TODO: remove this as submodules aren't used anymore.
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
+  build_binary_script: "./scripts/distribution/generic/build.sh"
+  package_binary_script: "./scripts/distribution/generic/package.sh"
 
 stages:
   - build_docker
-  - build_and_deploy_docker
-  - build_and_deploy_website
   - test
+  - build_and_deploy_docker
+  - build_and_package_binaries
+  - build_and_deploy_website
+
+.build_binary: &build_binary
+  stage: build_and_package_binaries
+  script:
+    - $build_binary_script "$target_os_family" "$target_os" "$target_os_version"
+    - $package_binary_script "$target_os_family" "$target_os" "$target_os_version"
+  artifacts:
+    paths:
+      - dist/package/**/*
+  only:
+    - master
+    - dev
 
 .website_build: &website_build
   stage: build_and_deploy_website
@@ -28,6 +43,9 @@ stages:
     # build with odoc
     - dune build @doc
 
+    # copy .deb packages into website
+    - find dist -name \*.deb -exec sh -c 'cp {} gitlab-pages/website/static/deb/ligo_$(basename $(dirname {})).deb' \;
+
     # npm
     - cd gitlab-pages/website
     - npm install
@@ -71,30 +89,12 @@ local-dune-job:
     - scripts/build_ligo_local.sh
     - dune build @ligo-test
 
-remote-repo-job:
-  <<: *before_script
-  stage: test
-  script:
-    # Add repository
-    - opam repository add ligo-repository https://gitlab.com/ligolang/ligo.git
-    - eval $(opam config env)
-    - opam install -y ligo
-    # Used in the IDE
-    #- opam install -y user-setup
-    #- opam install -y merlin
-    #- opam install -y ocp-indent
-    #- opam user-setup install
-  only:
-    - master
-
-# TODO: uncomment this
-
 # Run a docker build without publishing to the registry
 build-current-docker-image:
   stage: build_docker
   <<: *docker
   script:
-    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
+    - sh scripts/build_docker_image.sh
     - sh scripts/test_cli.sh
   except:
     - master
@@ -106,13 +106,51 @@ build-and-publish-latest-docker-image:
   stage: build_and_deploy_docker
   <<: *docker
   script:
-    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
+    - sh scripts/build_docker_image.sh
     - sh scripts/test_cli.sh
     - docker login -u $LIGO_REGISTRY_USER -p $LIGO_REGISTRY_PASSWORD
     - docker push $LIGO_REGISTRY_IMAGE:next
   only:
     - dev
 
+# It'd be a good idea to generate those jobs dynamically,
+# based on desired targets
+build-and-package-debian-9:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "debian"
+    target_os_version: "9"
+  <<: *build_binary
+
+build-and-package-debian-10:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "debian"
+    target_os_version: "10"
+  <<: *build_binary
+
+build-and-package-ubuntu-18-04:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "ubuntu"
+    target_os_version: "18.04"
+  <<: *build_binary
+
+build-and-package-ubuntu-19-04:
+  <<: *docker
+  stage: build_and_package_binaries
+  variables: 
+    target_os_family: "debian"
+    target_os: "ubuntu"
+    target_os_version: "19.04"
+  <<: *build_binary
+
 # Pages are deployed from both master & dev, be careful not to override 'next'
 # in case something gets merged into 'dev' while releasing.
 pages:

dist/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

docker/distribution/debian/package.Dockerfile

@@ -0,0 +1,32 @@
+ARG targetBuildImage
+FROM ${targetBuildImage}
+ARG version
+
+RUN apt-get update -qq
+RUN apt-get -y -qq install \
+    dput \
+    devscripts
+
+RUN mkdir /package && mkdir /package/bin && mkdir /package/DEBIAN && mkdir /package/dist
+# @TODO: make the binary path configurable
+RUN cp /home/opam/.opam/4.07/bin/ligo /package/bin/ligo
+
+# @TODO: inherit version (and other details) from the ligo opam package definition
+# In our case we're using the version field to name our package accordingly, 
+# however this is most likely not ideal
+# Also, the architecture field should not be 'all' but rather specific instead.
+RUN echo "Package: ligo\n\
+Version: $version\n\
+Architecture: all\n\
+Maintainer: info@ligolang.org\n\
+Depends: libev-dev, perl, pkg-config, libgmp-dev, libhidapi-dev, m4, libcap-dev, bubblewrap, rsync\n\
+Homepage: http://ligolang.org\n\
+Description: LIGO is a statically typed high-level smart-contract language that compiles down to Michelson." >> /package/DEBIAN/control
+
+RUN cd /package/dist && dpkg-deb --build /package .
+
+# Test the package
+# Optionally this could/should be done in a more pristine docker environment - in a separate image
+RUN apt-get update -qq
+RUN apt-get -y -f install "/package/dist/$(ls /package/dist)"
+RUN ligo --help

docker/distribution/generic/build.Dockerfile

@@ -1,5 +1,7 @@
-# At the moment, this really means 4.07.1
-FROM ocaml/opam2:4.07
+ARG target
+FROM ocaml/opam2:${target}
+
+RUN opam switch 4.07 && eval $(opam env)
 
 USER root
 
@@ -10,7 +12,6 @@ USER root
 # because the currently checkout out version (from git) will be used
 # to build the image
 ADD . /ligo
-
 # Set the current working directory to /ligo for
 # the upcoming scripts
 WORKDIR /ligo
@@ -18,10 +19,6 @@ WORKDIR /ligo
 # Install required native dependencies
 RUN sh scripts/install_native_dependencies.sh
 
-# Install OPAM
-# TODO: or scripts/install_build_environment.sh ?
-RUN sh scripts/install_opam.sh
-
 # Add tezos repository
 RUN sh scripts/setup_repos.sh
 

gitlab-pages/docs/setup/installation.md

@@ -3,7 +3,7 @@ id: installation
 title: Installation
 ---
 
-There are currently two ways to get started with Ligo, both of those will allow you to use the Ligo CLI with your contracts. You can choose to use either a Docker image, or to compile & build the Ligo CLI yourself.
+There are currently two ways to get started with Ligo. You can choose to either use a Docker image, or to install packages for your Debian Linux distribution.
 
 ## Dockerized installation (recommended)
 
@@ -40,6 +40,12 @@ ligo --help
 ```
 
 
-## Manual installation (advanced)
+## Debian Linux package installation
+
+We have produced .deb packages for a few Debian Linuxes. They will install a global `ligo` executable. You can install them in the usual way.
+
+- [Ubuntu 18.04](/deb/ligo_ubuntu-18.04.deb)
+- [Ubuntu 19.04](/deb/ligo_ubuntu-19.04.deb)
+- [Debian 9](/deb/ligo_debian-9.deb)
+- [Debian 10](/deb/ligo_debian-10.deb)
 
-For now, please refer to the steps described in the [Dockerfile](https://gitlab.com/ligolang/ligo/blob/master/docker/Dockerfile).

gitlab-pages/website/static/deb/.gitignore

@@ -0,0 +1 @@
+*

scripts/build_docker_image.sh

@@ -1,4 +1,3 @@
 #!/bin/sh
 set -e
-
-docker build -t ligolang/ligo -f docker/Dockerfile .
+docker build --build-arg target="4.07" -t "${LIGO_REGISTRY_IMAGE_BUILD:-ligolang/ligo}:next" -f ./docker/distribution/generic/build.Dockerfile .

scripts/distribution/generic/build.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+dockerfile_name="build"
+# Generic dockerfile
+dockerfile="./docker/distribution/generic/build.Dockerfile"
+. ./scripts/distribution/generic/parameters.sh
+
+echo "Building LIGO for $target"
+echo "Using Dockerfile: $dockerfile"
+echo "Tagging as: $tag_build\n"
+docker build --build-arg target="$target" -t "$tag_build" -f "$dockerfile" .

scripts/distribution/generic/env_variables.sh

@@ -0,0 +1,12 @@
+# This file is a substitute for env variables configured in the CI
+# in case you want to run the "CI scripts" on your own
+# You can load the following variables using:
+# source ./scripts/distribution/generic/env_variables.sh
+
+export LIGO_REGISTRY_IMAGE_BASE_NAME="ligolang/ligo"
+# packages build locally are tagget by the 'short' commit hash,
+# instead of the build/job/pipeline ID as in the CI to avoid possible confusion
+# ligo_incrementing-id_commit-hash
+export CI_JOB_ID="0"
+export CI_COMMIT_SHORT_SHA="$(git rev-parse --short HEAD)"
+export LIGO_DIST_DIR="./dist"

scripts/distribution/generic/package.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+dockerfile_name="package"
+dockerfile=""
+. ./scripts/distribution/generic/parameters.sh
+
+if [ -n "$dockerfile" ]; then
+    echo "Packaging LIGO for $target"
+    echo "Using Dockerfile: $dockerfile"
+    echo "Using pre-built image: $tag_build"
+    echo "Version: $version\n"
+
+    # Build the package
+    docker build --build-arg targetBuildImage="$tag_build" --build-arg version="$version" -t "$tag_package" -f "$dockerfile" .
+    # Copy the package to host's (our own) file system
+    mkdir -p "$PWD/dist/package/$target"
+    docker run --entrypoint '/bin/sh' -v $PWD:$PWD "$tag_package" -c "cp -r /package/dist/. $PWD/dist/package/$target"
+else
+    echo "Dockerfile not found for target: $target"
+fi

scripts/distribution/generic/parameters.sh

@@ -0,0 +1,32 @@
+# This script accepts three arguments, os family, os and it's version,
+# which are subsequently used to fetch the respective docker
+# image from the ocaml/infrastructure project.
+#
+# https://github.com/ocaml/infrastructure/wiki/Containers#selecting-linux-distributions
+target_os_family=$1
+target_os=$2
+target_os_version=$3
+
+# Variables configured at the CI level
+dist="$LIGO_DIST_DIR"
+version="$(echo $CI_JOB_ID)-$(echo $CI_COMMIT_SHORT_SHA)"
+ci_job_id="$CI_JOB_ID"
+
+# Image names for building & packaging
+target="$target_os-$target_os_version"
+tag_build="$LIGO_REGISTRY_IMAGE_BASE_NAME-build-$target:$version"
+tag_package="$LIGO_REGISTRY_IMAGE_BASE_NAME-package-$target:$version"
+tag_package_dockerized="$LIGO_REGISTRY_IMAGE_BASE_NAME-$target:$version"
+
+
+# Check if there's a specific dockerfile override for 
+# the current target_os_family (e.g. debian-ish distros) or target_os (ubuntu, xubuntu, ...) and use it if there is one
+target_os_family_specific_dockerfile="./docker/distribution/$target_os_family/$dockerfile_name.Dockerfile"
+if test -f "$target_os_family_specific_dockerfile"; then
+    dockerfile="$target_os_family_specific_dockerfile"
+fi
+
+target_os_specific_dockerfile="./docker/distribution/$target_os_family/$target_os/$dockerfile_name.Dockerfile"
+if test -f "$target_os_specific_dockerfile"; then
+    dockerfile="$target_os_specific_dockerfile"
+fi