diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ccf03db24 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +dist +_opam +_build +docker +gitlab-pages +Makefile \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fa3cd9ec..19e5d7f6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,27 @@ # TODO: remove this as submodules aren't used anymore. variables: GIT_SUBMODULE_STRATEGY: recursive + build_binary_script: "./scripts/distribution/generic/build.sh" + package_binary_script: "./scripts/distribution/generic/package.sh" stages: - build_docker - - build_and_deploy_docker - - build_and_deploy_website - test + - build_and_deploy_docker + - build_and_package_binaries + - build_and_deploy_website + +.build_binary: &build_binary + stage: build_and_package_binaries + script: + - $build_binary_script "$target_os_family" "$target_os" "$target_os_version" + - $package_binary_script "$target_os_family" "$target_os" "$target_os_version" + artifacts: + paths: + - dist/package/**/* + only: + - master + - dev .website_build: &website_build stage: build_and_deploy_website @@ -71,30 +86,12 @@ local-dune-job: - scripts/build_ligo_local.sh - dune build @ligo-test -remote-repo-job: - <<: *before_script - stage: test - script: - # Add repository - - opam repository add ligo-repository https://gitlab.com/ligolang/ligo.git - - eval $(opam config env) - - opam install -y ligo - # Used in the IDE - #- opam install -y user-setup - #- opam install -y merlin - #- opam install -y ocp-indent - #- opam user-setup install - only: - - master - -# TODO: uncomment this - # Run a docker build without publishing to the registry build-current-docker-image: stage: build_docker <<: *docker script: - - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile . + - sh scripts/build_docker_image.sh - sh scripts/test_cli.sh except: - master @@ -106,13 +103,51 @@ build-and-publish-latest-docker-image: stage: build_and_deploy_docker <<: *docker script: - - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile . + - sh scripts/build_docker_image.sh - sh scripts/test_cli.sh - docker login -u $LIGO_REGISTRY_USER -p $LIGO_REGISTRY_PASSWORD - docker push $LIGO_REGISTRY_IMAGE:next only: - dev +# It'd be a good idea to generate those jobs dynamically, +# based on desired targets +build-and-package-debian-9: + <<: *docker + stage: build_and_package_binaries + variables: + target_os_family: "debian" + target_os: "debian" + target_os_version: "9" + <<: *build_binary + +build-and-package-debian-10: + <<: *docker + stage: build_and_package_binaries + variables: + target_os_family: "debian" + target_os: "debian" + target_os_version: "10" + <<: *build_binary + +build-and-package-ubuntu-18-04: + <<: *docker + stage: build_and_package_binaries + variables: + target_os_family: "debian" + target_os: "ubuntu" + target_os_version: "18.04" + <<: *build_binary + +build-and-package-ubuntu-19-04: + <<: *docker + stage: build_and_package_binaries + variables: + target_os_family: "debian" + target_os: "ubuntu" + target_os_version: "19.04" + <<: *build_binary + # Pages are deployed from both master & dev, be careful not to override 'next' # in case something gets merged into 'dev' while releasing. pages: diff --git a/dist/.gitignore b/dist/.gitignore new file mode 100644 index 000000000..86d0cb272 --- /dev/null +++ b/dist/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/docker/distribution/debian/distribute.Dockerfile b/docker/distribution/debian/distribute.Dockerfile new file mode 100644 index 000000000..e69de29bb diff --git a/docker/distribution/debian/package.Dockerfile b/docker/distribution/debian/package.Dockerfile new file mode 100644 index 000000000..440496521 --- /dev/null +++ b/docker/distribution/debian/package.Dockerfile @@ -0,0 +1,32 @@ +ARG targetBuildImage +FROM ${targetBuildImage} +ARG version + +RUN apt-get update -qq +RUN apt-get -y -qq install \ + dput \ + devscripts + +RUN mkdir /package && mkdir /package/bin && mkdir /package/DEBIAN && mkdir /package/dist +# @TODO: make the binary path configurable +RUN cp /home/opam/.opam/4.07/bin/ligo /package/bin/ligo + +# @TODO: inherit version (and other details) from the ligo opam package definition +# In our case we're using the version field to name our package accordingly, +# however this is most likely not ideal +# Also, the architecture field should not be 'all' but rather specific instead. +RUN echo "Package: ligo\n\ +Version: $version\n\ +Architecture: all\n\ +Maintainer: info@ligolang.org\n\ +Depends: libev-dev, perl, pkg-config, libgmp-dev, libhidapi-dev, m4, libcap-dev, bubblewrap, rsync\n\ +Homepage: http://ligolang.org\n\ +Description: LIGO is a statically typed high-level smart-contract language that compiles down to Michelson." >> /package/DEBIAN/control + +RUN cd /package/dist && dpkg-deb --build /package . + +# Test the package +# Optionally this could/should be done in a more pristine docker environment - in a separate image +RUN apt-get update -qq +RUN apt-get -y -f install "/package/dist/$(ls /package/dist)" +RUN ligo --help \ No newline at end of file diff --git a/docker/Dockerfile b/docker/distribution/generic/build.Dockerfile similarity index 81% rename from docker/Dockerfile rename to docker/distribution/generic/build.Dockerfile index 057bd6436..e183fc683 100644 --- a/docker/Dockerfile +++ b/docker/distribution/generic/build.Dockerfile @@ -1,5 +1,7 @@ -# At the moment, this really means 4.07.1 -FROM ocaml/opam2:4.07 +ARG target +FROM ocaml/opam2:${target} + +RUN opam switch 4.07 && eval $(opam env) USER root @@ -10,7 +12,6 @@ USER root # because the currently checkout out version (from git) will be used # to build the image ADD . /ligo - # Set the current working directory to /ligo for # the upcoming scripts WORKDIR /ligo @@ -18,10 +19,6 @@ WORKDIR /ligo # Install required native dependencies RUN sh scripts/install_native_dependencies.sh -# Install OPAM -# TODO: or scripts/install_build_environment.sh ? -RUN sh scripts/install_opam.sh - # Add tezos repository RUN sh scripts/setup_repos.sh diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 273fa92c6..21e3a2bf5 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -1,4 +1,3 @@ #!/bin/sh set -e - -docker build -t ligolang/ligo -f docker/Dockerfile . +docker build --build-arg target="4.07" -t "${LIGO_REGISTRY_IMAGE_BUILD:-ligolang/ligo}:next" -f ./docker/distribution/generic/build.Dockerfile . diff --git a/scripts/distribution/generic/build.sh b/scripts/distribution/generic/build.sh new file mode 100755 index 000000000..7f4a89e9a --- /dev/null +++ b/scripts/distribution/generic/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +dockerfile_name="build" +# Generic dockerfile +dockerfile="./docker/distribution/generic/build.Dockerfile" +. ./scripts/distribution/generic/parameters.sh + +echo "Building LIGO for $target" +echo "Using Dockerfile: $dockerfile" +echo "Tagging as: $tag_build\n" +docker build --build-arg target="$target" -t "$tag_build" -f "$dockerfile" . \ No newline at end of file diff --git a/scripts/distribution/generic/env_variables.sh b/scripts/distribution/generic/env_variables.sh new file mode 100755 index 000000000..7279e4cbe --- /dev/null +++ b/scripts/distribution/generic/env_variables.sh @@ -0,0 +1,12 @@ +# This file is a substitute for env variables configured in the CI +# in case you want to run the "CI scripts" on your own +# You can load the following variables using: +# source ./scripts/distribution/generic/env_variables.sh + +export LIGO_REGISTRY_IMAGE_BASE_NAME="ligolang/ligo" +# packages build locally are tagget by the 'short' commit hash, +# instead of the build/job/pipeline ID as in the CI to avoid possible confusion +# ligo_incrementing-id_commit-hash +export CI_JOB_ID="0" +export CI_COMMIT_SHORT_SHA="$(git rev-parse --short HEAD)" +export LIGO_DIST_DIR="./dist" diff --git a/scripts/distribution/generic/package.sh b/scripts/distribution/generic/package.sh new file mode 100755 index 000000000..b00b61841 --- /dev/null +++ b/scripts/distribution/generic/package.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +dockerfile_name="package" +dockerfile="" +. ./scripts/distribution/generic/parameters.sh + +if [ -n "$dockerfile" ]; then + echo "Packaging LIGO for $target" + echo "Using Dockerfile: $dockerfile" + echo "Using pre-built image: $tag_build" + echo "Version: $version\n" + + # Build the package + docker build --build-arg targetBuildImage="$tag_build" --build-arg version="$version" -t "$tag_package" -f "$dockerfile" . + # Copy the package to host's (our own) file system + mkdir -p "$PWD/dist/package/$target" + docker run -v $PWD:$PWD "$tag_package" cp -r /package/dist/. "$PWD/dist/package/$target" +else + echo "Dockerfile not found for target: $target" +fi \ No newline at end of file diff --git a/scripts/distribution/generic/parameters.sh b/scripts/distribution/generic/parameters.sh new file mode 100644 index 000000000..436b48bf0 --- /dev/null +++ b/scripts/distribution/generic/parameters.sh @@ -0,0 +1,32 @@ +# This script accepts three arguments, os family, os and it's version, +# which are subsequently used to fetch the respective docker +# image from the ocaml/infrastructure project. +# +# https://github.com/ocaml/infrastructure/wiki/Containers#selecting-linux-distributions +target_os_family=$1 +target_os=$2 +target_os_version=$3 + +# Variables configured at the CI level +dist="$LIGO_DIST_DIR" +version="$(echo $CI_JOB_ID)-$(echo $CI_COMMIT_SHORT_SHA)" +ci_job_id="$CI_JOB_ID" + +# Image names for building & packaging +target="$target_os-$target_os_version" +tag_build="$LIGO_REGISTRY_IMAGE_BASE_NAME-build-$target:$version" +tag_package="$LIGO_REGISTRY_IMAGE_BASE_NAME-package-$target:$version" +tag_package_dockerized="$LIGO_REGISTRY_IMAGE_BASE_NAME-$target:$version" + + +# Check if there's a specific dockerfile override for +# the current target_os_family (e.g. debian-ish distros) or target_os (ubuntu, xubuntu, ...) and use it if there is one +target_os_family_specific_dockerfile="./docker/distribution/$target_os_family/$dockerfile_name.Dockerfile" +if test -f "$target_os_family_specific_dockerfile"; then + dockerfile="$target_os_family_specific_dockerfile" +fi + +target_os_specific_dockerfile="./docker/distribution/$target_os_family/$target_os/$dockerfile_name.Dockerfile" +if test -f "$target_os_specific_dockerfile"; then + dockerfile="$target_os_specific_dockerfile" +fi \ No newline at end of file