From edb3fce6429338711ce45990ddfe4212232cbad6 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Wed, 25 Mar 2020 21:14:18 +0300 Subject: [PATCH] Add nix as a build system * Dynamically linked executables (ligo-bin) * Statically linked executables (ligo-static) * Docker (ligo-docker) * deb package (ligo-deb) * webide (ligo-editor) * webide docker (ligo-editor-docker) * website (ligo-website) --- .gitlab-ci.yml | 164 ++++++++++++++++++ docker/distribution/generic/build.Dockerfile | 4 + gitlab-pages/website/package-lock.json | 17 +- gitlab-pages/website/package.json | 2 + nix/default.nix | 9 + nix/docker.nix | 10 ++ nix/ligo-editor.nix | 71 ++++++++ nix/ligo-website.nix | 18 ++ nix/mac-overlay.nix | 9 + nix/nodejs-overlay.nix | 5 + nix/ocaml-overlay.nix | 140 +++++++++++++++ nix/packageDeb.nix | 46 +++++ nix/pkgs.nix | 33 ++++ nix/sources.json | 87 ++++++++++ nix/sources.nix | 134 ++++++++++++++ nix/static-overlay.nix | 24 +++ nix/static.patch | 12 ++ scripts/distribution/generic/env_variables.sh | 2 + scripts/distribution/generic/parameters.sh | 2 + src/bin/cli.ml | 7 +- src/bin/dune | 2 +- src/stages/adt_generator/dune | 3 +- src/test/adt_generator/dune | 1 - .../webide/packages/client/public/index.html | 5 +- .../webide/packages/e2e/test/common-utils.js | 2 + vendors/Preprocessor/Preprocessor.opam | 2 +- vendors/ligo-utils/simple-utils/messages.sh | 2 +- 27 files changed, 794 insertions(+), 19 deletions(-) create mode 100644 nix/default.nix create mode 100644 nix/docker.nix create mode 100644 nix/ligo-editor.nix create mode 100644 nix/ligo-website.nix create mode 100644 nix/mac-overlay.nix create mode 100644 nix/nodejs-overlay.nix create mode 100644 nix/ocaml-overlay.nix create mode 100644 nix/packageDeb.nix create mode 100644 nix/pkgs.nix create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100644 nix/static-overlay.nix create mode 100644 nix/static.patch diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 767e9aa73..34c499512 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,8 @@ stages: - ide-build - ide-e2e-test - ide-deploy + - nix + - nix-push # TODO provide sensible CI for master dont-merge-to-master: @@ -42,6 +44,7 @@ dont-merge-to-master: - build-and-package-ubuntu-18-04 - build-and-package-ubuntu-19-10 before_script: + - export COMMIT_DATE="$(git show --no-patch --format=%ci)" - export TERM=dumb - scripts/install_native_dependencies.sh - scripts/install_opam.sh # TODO: or scripts/install_build_environment.sh ? @@ -85,6 +88,7 @@ dont-merge-to-master: before_script: # Install dependencies # rsync is needed by opam to sync a package installed from a local directory with the copy in ~/.opam + - export COMMIT_DATE="$(git show --no-patch --format=%ci)" - export TERM=dumb - scripts/install_native_dependencies.sh - scripts/install_opam.sh # TODO: or scripts/install_build_environment.sh ? @@ -117,6 +121,7 @@ build-current-docker-image: - build-and-package-debian-10 <<: *docker script: + - export COMMIT_DATE="$(git show --no-patch --format=%ci)" - sh scripts/build_docker_image.sh next - sh scripts/test_cli.sh only: @@ -281,3 +286,162 @@ deploy-handoff: rules: - if: '$CI_COMMIT_REF_NAME == "dev"' when: always + + +##### The following jobs will replace the ones above! ##### +# TODO: add jobs for deploying the website, build a docker image and deploy it + +.prepare_nix: &prepare_nix + image: nixos/nix:latest + before_script: + - nix-env -f channel:nixos-unstable -iA gnutar gitMinimal cachix + - export COMMIT_DATE="$(git show --no-patch --format=%ci)" + - echo "sandbox = true" > /etc/nix/nix.conf + # A temporary caching solution + - cachix use balsoft + # TODO Don't upload everything, use a post-build-hook to only upload what can't be substituted + - cachix push -w balsoft & + +# The binary produced is useless by itself +binary-nix: + stage: nix + <<: *prepare_nix + only: + - merge_requests + - dev + - /^.*-run-dev$/ + script: + - nix-build nix -A ligo-bin + +doc-nix: + stage: nix + <<: *prepare_nix + only: + - merge_requests + - dev + - /^.*-run-dev$/ + script: + - nix-build nix -A ligo-doc + - cp -Lr result/share/doc result-doc + artifacts: + paths: + - result-doc + +test-nix: + stage: nix + <<: *prepare_nix + only: + - merge_requests + - dev + - /^.*-run-dev$/ + script: + - nix-build nix -A ligo-coverage + - cp -Lr result/share/coverage result-coverage + artifacts: + paths: + - result-coverage + +# FIXME For some reason, e2e tests can't build on CI. +.webide-e2e-nix: + stage: nix + <<: *prepare_nix + rules: + - changes: + - tools/webide/** + when: always + - if: '$CI_COMMIT_REF_NAME =~ /^(dev|.*-run-dev)$/ && $CI_PROJECT_PATH == "ligolang/ligo"' + when: always + script: + - nix-build nix -A ligo-editor.e2e + +docker-nix: + stage: nix + only: + - merge_requests + - dev + - /^.*-run-dev$/ + <<: *prepare_nix + script: + - nix-build nix -A ligo-docker + - cp -L result ligo.tar.gz + artifacts: + paths: + - ligo.tar.gz + +docker-push-nix: + stage: nix-push + <<: *docker + dependencies: + - docker-nix + needs: + - docker-nix + rules: + # Only deploy docker when from the dev branch AND on the canonical ligolang/ligo repository + - if: '$CI_COMMIT_REF_NAME =~ /^(dev|.*-run-dev)$/ && $CI_PROJECT_PATH == "ligolang/ligo"' + when: always + script: + - echo ${LIGO_REGISTRY_PASSWORD} | docker login -u ${LIGO_REGISTRY_USER} --password-stdin + - docker load -i=./ligo.tar.gz + - export LIGO_REGISTRY_FULL_NAME=${LIGO_REGISTRY_IMAGE_BUILD:-ligolang/ligo}:$(if test "$CI_COMMIT_REF_NAME" = "dev"; then echo next-nix; else echo next-attempt-nix; fi) + - docker tag ligo "${LIGO_REGISTRY_FULL_NAME}" + - docker push "${LIGO_REGISTRY_FULL_NAME}" + +webide-docker-nix: + stage: nix + only: + - merge_requests + - dev + - /^.*-run-dev$/ + <<: *prepare_nix + script: + - nix-build nix -A ligo-editor-docker + - cp -L result webide.tar.gz + artifacts: + paths: + - webide.tar.gz + + +webide-push-nix: + stage: nix-push + <<: *docker + dependencies: + - webide-docker-nix + needs: + - webide-docker-nix + rules: + # Only deploy docker when from the dev branch AND on the canonical ligolang/ligo repository + - if: '$CI_COMMIT_REF_NAME =~ /^(dev|.*-run-dev)$/ && $CI_PROJECT_PATH == "ligolang/ligo"' + when: always + script: + - echo "${CI_BUILD_TOKEN}" | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com + - docker load -i=./webide.tar.gz + - docker tag ligo-editor "${WEBIDE_IMAGE_NAME}:nix${CI_COMMIT_SHORT_SHA}" + - docker push "${WEBIDE_IMAGE_NAME}:nix${CI_COMMIT_SHORT_SHA}" + +static-binary-nix: + stage: nix + <<: *prepare_nix + only: + - dev + - /^.*-run-dev$/ + script: + - nix-build nix -A ligo-static + # Check that the binary is truly static and has 0 dependencies + - test $(nix-store -q --references ./result | wc -l) -eq 0 + - cp -Lr result/bin result-static + artifacts: + paths: + - result-static + +website-nix: + stage: nix + <<: *prepare_nix + only: + - dev + - /^.*-run-dev$/ + script: + - nix-build nix -A ligo-website + - cp -Lr result/ result-website + artifacts: + paths: + - result-website diff --git a/docker/distribution/generic/build.Dockerfile b/docker/distribution/generic/build.Dockerfile index ba01c043c..a9630c79d 100644 --- a/docker/distribution/generic/build.Dockerfile +++ b/docker/distribution/generic/build.Dockerfile @@ -2,7 +2,11 @@ ARG target FROM ocaml/opam2:${target} ARG ci_job_id +ARG ci_commit_sha +ARG commit_date ENV CI_JOB_ID=$ci_job_id +ENV CI_COMMIT_SHA=$ci_commit_sha +ENV COMMIT_DATE=$commit_date RUN opam switch 4.07 && eval $(opam env) diff --git a/gitlab-pages/website/package-lock.json b/gitlab-pages/website/package-lock.json index 43d25ce3e..b42d4b706 100644 --- a/gitlab-pages/website/package-lock.json +++ b/gitlab-pages/website/package-lock.json @@ -1,6 +1,8 @@ { - "requires": true, + "name": "ligo-website", + "version": "0.0.1", "lockfileVersion": 1, + "requires": true, "dependencies": { "@ligolang/docusaurus-theme-compact-ligo-ide": { "version": "file:../../tools/compact-webide/packages/docusaurus-theme-compact-ligo-ide", @@ -1660,7 +1662,12 @@ "debug": { "version": "2.6.9", "requires": { - "ms": "2.0.0" + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" } }, "decamelize": { @@ -2017,10 +2024,8 @@ "fill-range": { "version": "4.0.0", "requires": { - "extend-shallow": "^2.0.1", - "is-number": "^3.0.0", - "repeat-string": "^1.6.1", - "to-regex-range": "^2.1.0" + "@docusaurus/types": "^2.0.0-alpha.50", + "sitemap": "^3.2.2" } }, "find-up": { diff --git a/gitlab-pages/website/package.json b/gitlab-pages/website/package.json index 8fa752c2e..75ac51078 100644 --- a/gitlab-pages/website/package.json +++ b/gitlab-pages/website/package.json @@ -1,4 +1,6 @@ { + "name": "ligo-website", + "version": "0.0.1", "scripts": { "start": "docusaurus start --host 0.0.0.0 --port 3000", "build": "docusaurus build", diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 000000000..cf7a81e2f --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,9 @@ +{ sources ? import ./sources.nix }@args: +let pkgs = import ./pkgs.nix args; +in { + inherit (pkgs) + ligo ligo-tests ligo-doc ligo-coverage + ligo-bin ligo-static ligo-docker ligo-deb + ligo-editor ligo-editor-docker + ligo-website; +} diff --git a/nix/docker.nix b/nix/docker.nix new file mode 100644 index 000000000..acda1c971 --- /dev/null +++ b/nix/docker.nix @@ -0,0 +1,10 @@ +{ dockerTools, writeShellScriptBin, runCommand, mcpp, bash, coreutils, ligo, name ? "ligo" }: +let + tmp = runCommand "tmp" {} "mkdir -p $out/tmp"; +in +dockerTools.buildLayeredImage { + inherit name; + tag = "latest"; + contents = [ ligo tmp bash ]; + config.Entrypoint = name; +} diff --git a/nix/ligo-editor.nix b/nix/ligo-editor.nix new file mode 100644 index 000000000..9a1a88920 --- /dev/null +++ b/nix/ligo-editor.nix @@ -0,0 +1,71 @@ +{ stdenv, lib, mkYarnPackage, nodejs, python2, ligo-bin, coreutils +, writeShellScriptBin, makeFontsConf, buildEnv, rsync, sources +, chromium ? null }: +let + yarnLock = ../tools/webide/yarn.lock; + + installPhase = "mkdir $out; cp -Lr node_modules $out/node_modules"; + + server = mkYarnPackage { + name = "webide-server"; + src = ../tools/webide/packages/server; + buildPhase = '' + cp ${../tools/webide/tsconfig.json} tsconfig.json + yarn --offline run build + rm node_modules/server/server + ''; + doCheck = true; + checkPhase = "DATA_DIR=/tmp LIGO_CMD=${ligo-bin}/bin/ligo yarn --offline jest"; + distPhase = "true"; + inherit yarnLock installPhase; + }; + client = mkYarnPackage rec { + name = "webide-client"; + src = ../tools/webide/packages/client; + buildPhase = '' + export EXAMPLES_DIR=${../src/test/examples} + yarn --offline run build + rm node_modules/client/client + find deps/client/build -type f -exec sed -r "s,/nix/store/[a-z0-9]{32}-[^/]*,$out,g" -i '{}' \; + ''; + distPhase = "true"; + installPhase = "mkdir $out; cp -Lr deps/client/build $out"; + inherit yarnLock; + # Downloads node-sass from the official github repo + # Uncomment the commented lines if you wish to build it from source + yarnPreBuild = "export SASS_BINARY_PATH=${sources.node-sass-bin}"; + /* '' + mkdir -p "$HOME/.node-gyp/${nodejs.version}" + echo 9 > "$HOME/.node-gyp/${nodejs.version}/installVersion" + ln -sfv "${nodejs}/include" "$HOME/.node-gyp/${nodejs.version}" + ''; + */ + }; + + e2e = mkYarnPackage rec { + name = "webide-e2e"; + src = ../tools/webide/packages/e2e; + # Provide puppeteer with chromium, since it can't download it inside the nix sandbox. + # Also, since we override nodejs in our overlays, import chromium from pure nixpkgs to avoid a rebuild + buildPhase = '' + export HOME="$(pwd)" + export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 + export PUPPETEER_EXECUTABLE_PATH=${(import (import ./sources.nix).nixpkgs {}).chromium.outPath}/bin/chromium + ${ligo-editor}/bin/ligo-editor & + export API_HOST=http://localhost:8080 + export FONTCONFIG_FILE=${makeFontsConf { fontDirectories = [ ]; }} + yarn --offline jest + ''; + distPhase = "true"; + installPhase = "touch $out"; + inherit yarnLock; + }; + + ligo-editor = writeShellScriptBin "ligo-editor" '' + set -e + LIGO_CMD=${ligo-bin}/bin/ligo \ + STATIC_ASSETS=${client} \ + DATA_DIR=/tmp \ + ${nodejs}/bin/node ${server}/node_modules/server/dist/src/index.js + ''; +in ligo-editor // { inherit e2e; } diff --git a/nix/ligo-website.nix b/nix/ligo-website.nix new file mode 100644 index 000000000..5e95e7b8c --- /dev/null +++ b/nix/ligo-website.nix @@ -0,0 +1,18 @@ +{ buildNpmPackage, writeShellScriptBin, yarn, linkFarm, nodejs-slim, python2 +, ligo-doc, ligo-deb, ligo-static }: +buildNpmPackage { + src = ../gitlab-pages/website; + npmBuild = "npm run build"; + preBuild = '' + cp -r ${../gitlab-pages/docs} $NIX_BUILD_TOP/docs + chmod 700 -R $NIX_BUILD_TOP/docs + ''; + installPhase = '' + cp -Lr build $out + cp -r ${ligo-deb}/* $out/deb + mkdir -p $out/bin/linux + cp -r ${ligo-static}/bin/ligo $out/bin/linux/ligo + cp -r ${ligo-doc}/share/doc $out/odoc + ''; + extraEnvVars.nativeBuildInputs = [ python2 ]; +} diff --git a/nix/mac-overlay.nix b/nix/mac-overlay.nix new file mode 100644 index 000000000..8cbedb0b9 --- /dev/null +++ b/nix/mac-overlay.nix @@ -0,0 +1,9 @@ +oself: osuper: +let + fixHardeningWarning = pkg: pkg.overrideAttrs (_: { + hardeningDisable = [ "strictoverflow" ]; + }); +in +{ + hacl = fixHardeningWarning osuper.hacl; +} diff --git a/nix/nodejs-overlay.nix b/nix/nodejs-overlay.nix new file mode 100644 index 000000000..7a5badf30 --- /dev/null +++ b/nix/nodejs-overlay.nix @@ -0,0 +1,5 @@ +self: super: { + nodejs = super.nodejs-12_x; + nodePackages = super.nodePackages_12_x; + nodejs-slim = super.nodejs-slim-12_x; +} diff --git a/nix/ocaml-overlay.nix b/nix/ocaml-overlay.nix new file mode 100644 index 000000000..d811becad --- /dev/null +++ b/nix/ocaml-overlay.nix @@ -0,0 +1,140 @@ +{ sources ? import ./sources.nix +, CI_COMMIT_SHA ? builtins.getEnv "CI_COMMIT_SHA" +, COMMIT_DATE ? builtins.getEnv "COMMIT_DATE" }: +self: super: +let + opam-nix = import sources.opam-nix (import sources.nixpkgs { }); + inherit (import sources."gitignore.nix" { inherit (self) lib; }) + gitignoreSource; + filterOut = xs: + self.lib.cleanSourceWith { + filter = p: type: !(builtins.elem (builtins.baseNameOf p) xs); + src = gitignoreSource ../.; + }; +in { + ocamlPackages = self.ocaml-ng.ocamlPackages_4_07.overrideScope' + (builtins.foldl' self.lib.composeExtensions (_: _: { }) [ + (opam-nix.traverseOPAMRepo' sources.opam-repository) + (opam-nix.traverseOPAMRepo sources.tezos-opam-repository) + (opam-nix.callOPAMPackage (filterOut [ + ".git" + ".gitlab-ci.yml" + ".gitignore" + "nix" + "docker" + "tools" + "gitlab-pages" + ])) + (oself: osuper: { + ocamlfind = oself.findlib; + lablgtk = null; + lwt = oself.lwt4; + + conf-gmp = self.gmp; + conf-libev = self.libev; + conf-hidapi = self.hidapi; + conf-pkg-config = self.pkg-config; + + bigstring = osuper.bigstring.overrideAttrs (_: { doCheck = false; }); + xmldiff = osuper.xmldiff.overrideAttrs (_: { src = sources.xmldiff; }); + getopt = osuper.getopt.overrideAttrs (_: { configurePhase = "true"; }); + + ipaddr = osuper.ipaddr.versions."4.0.0"; + conduit = osuper.conduit.versions."2.1.0"; + conduit-lwt-unix = osuper.conduit-lwt-unix.versions."2.0.2"; + cohttp-lwt-unix = osuper.cohttp-lwt-unix.versions."2.4.0"; + cohttp-lwt = osuper.cohttp-lwt.versions."2.4.0"; + macaddr = osuper.macaddr.versions."4.0.0"; + ocaml-migrate-parsetree = + osuper.ocaml-migrate-parsetree.versions."1.4.0"; + ppx_tools_versioned = osuper.ppx_tools_versioned.versions."5.2.3"; + bisect_ppx = osuper.bisect_ppx.versions."2.0.0".overrideAttrs (_: { + src = builtins.fetchTarball + "https://github.com/aantron/bisect_ppx/archive/02dfb10188033a26d07d23480c2bc44a3a670357.tar.gz"; + }); + + proto-alpha-utils = osuper.proto-alpha-utils.overrideAttrs (oa: rec { + buildInputs = oa.buildInputs + ++ [ oself.tezos-protocol-006-PsCARTHA-parameters ]; + propagatedBuildInputs = buildInputs; + }); + tezos-protocol-compiler = osuper.tezos-protocol-compiler.overrideAttrs + (oa: rec { + buildInputs = oa.buildInputs ++ [ oself.pprint ]; + propagatedBuildInputs = buildInputs; + }); + + ligo = self.buildEnv { + name = "ligo"; + paths = with oself; [ + ligo-out.out + ligo-tests + ligo-doc + ligo-coverage + ]; + }; + + ligo-out = osuper.ligo.overrideAttrs (oa: { + name = "ligo-out"; + inherit CI_COMMIT_SHA COMMIT_DATE; + buildInputs = oa.buildInputs + ++ [ oself.UnionFind oself.Preprocessor ]; + nativeBuildInputs = oa.nativeBuildInputs + ++ [ self.buildPackages.rakudo ]; + }); + ligo-tests = osuper.ligo.overrideAttrs (oa: { + name = "ligo-tests"; + src = filterOut [ + ".git" + ".gitlab-ci.yml" + ".gitignore" + "nix" + "docker" + "tools" + ]; + outputs = [ "out" ]; + buildPhase = "dune runtest"; + nativeBuildInputs = oa.nativeBuildInputs + ++ [ self.buildPackages.rakudo ]; + installPhase = "mkdir $out"; + }); + ligo-doc = osuper.ligo.overrideAttrs (oa: { + name = "ligo-doc"; + buildInputs = oa.buildInputs + ++ [ oself.odoc oself.tezos-protocol-updater ]; + outputs = [ "out" ]; + buildPhase = "dune build @doc"; + nativeBuildInputs = oa.nativeBuildInputs + ++ [ self.buildPackages.rakudo ]; + installPhase = + "mkdir $out; cp -r _build/default/_doc/_html/ $out/doc"; + }); + ligo-coverage = oself.ligo-tests.overrideAttrs (oa: { + name = "ligo-coverage"; + nativeBuildInputs = oa.nativeBuildInputs + ++ [ self.buildPackages.rakudo ]; + buildPhase = '' + # Needed for coverage and nothing else + mkdir -p $out/share/coverage + echo "Coverage:" + BISECT_ENABLE=yes dune runtest --force + bisect-ppx-report html -o $out/share/coverage/all --title="LIGO overall test coverage" + bisect-ppx-report summary --per-file + echo "Test coverage:" + BISECT_ENABLE=yes dune runtest src/test --force + bisect-ppx-report html -o $out/share/coverage/ligo --title="LIGO test coverage" + bisect-ppx-report summary --per-file + echo "Doc coverage:" + BISECT_ENABLE=yes dune build @doc-test --force + bisect-ppx-report html -o $out/share/coverage/docs --title="LIGO doc coverage" + bisect-ppx-report summary --per-file + echo "CLI test coverage:" + BISECT_ENABLE=yes dune runtest src/bin/expect_tests + bisect-ppx-report html -o $out/share/coverage/cli --title="CLI test coverage" + bisect-ppx-report summary --per-file + ''; + installPhase = "true"; + }); + }) + ]); +} diff --git a/nix/packageDeb.nix b/nix/packageDeb.nix new file mode 100644 index 000000000..bb5f0a57b --- /dev/null +++ b/nix/packageDeb.nix @@ -0,0 +1,46 @@ +{ stdenv, lib, writeTextFile, ligo-static, dpkg }: +let + project = "ligo"; + version = "0.0.0"; + revision = lib.commitIdFromGitRepo ../.git; + pkgArch = "amd64"; + bin = "${ligo-static}/bin/ligo"; + pkgName = "${project}_0ubuntu${version}-${revision}_${pkgArch}"; + depends = ""; + maintainer = "ligolang ligolang.org"; + description = "A friendly Smart Contract Language for Tezos"; + + writeControlFile = writeTextFile { + name = "control"; + text = '' + Package: ${project} + Version: ${version}-${revision} + Priority: optional + Architecture: ${pkgArch} + Depends: ${depends} + Maintainer: ${maintainer} + Description: ${project} + ${description} + ''; + }; + +in stdenv.mkDerivation rec { + name = "${pkgName}.deb"; + + nativeBuildInputs = [ dpkg ]; + + phases = "packagePhase"; + + packagePhase = '' + mkdir ${pkgName} + mkdir -p ${pkgName}/usr/local/bin + cp ${bin} ${pkgName}/usr/local/bin/${project} + + mkdir ${pkgName}/DEBIAN + cp ${writeControlFile} ${pkgName}/DEBIAN/control + + dpkg-deb --build ${pkgName} + mkdir -p $out + cp ${name} $out/ + ''; +} diff --git a/nix/pkgs.nix b/nix/pkgs.nix new file mode 100644 index 000000000..d832dde56 --- /dev/null +++ b/nix/pkgs.nix @@ -0,0 +1,33 @@ +{ sources ? import ./sources.nix }: +let + ocaml-overlay = import ./ocaml-overlay.nix { inherit sources; }; + static-overlay = import ./static-overlay.nix pkgs; + mac-overlay = import ./mac-overlay.nix; + nodejs-overlay = import ./nodejs-overlay.nix; + pkgs = import sources.nixpkgs { + overlays = [ ocaml-overlay nodejs-overlay ] + ++ (if builtins.currentSystem == "x86_64-darwin" + then [ mac-overlay ] + else [ ]); + }; + separateBinary = pkg: + pkgs.runCommandNoCC "${pkg.name}-bin" { } + "mkdir -p $out/bin; cp -Lr ${pkg}/ligo $out/bin"; + + nix-npm-buildpackage = pkgs.callPackage sources.nix-npm-buildpackage { }; +in pkgs.extend (self: super: { + inherit (self.ocamlPackages) ligo ligo-out ligo-tests ligo-doc ligo-coverage; + ligo-bin = separateBinary self.ligo-out.bin; + ligo-docker = self.callPackage ./docker.nix { ligo = self.ligo-bin; }; + ligo-deb = self.callPackage ./packageDeb.nix { }; + ligo-editor = self.callPackage ./ligo-editor.nix { inherit sources; }; + ligo-editor-docker = self.callPackage ./docker.nix { + ligo = self.ligo-editor; + name = "ligo-editor"; + }; + ligo-website = self.callPackage ./ligo-website.nix { + inherit (nix-npm-buildpackage) buildNpmPackage; + }; + ligo-static = self.pkgsMusl.ligo-bin; + pkgsMusl = super.pkgsMusl.extend static-overlay; +}) diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..8af15c341 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,87 @@ +{ + "gitignore.nix": { + "branch": "master", + "description": "Nix function for filtering local git sources", + "homepage": "", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "2ced4519f865341adcb143c5d668f955a2cb997f", + "sha256": "0fc5bgv9syfcblp23y05kkfnpgh3gssz6vn24frs8dzw39algk2z", + "type": "tarball", + "url": "https://github.com/hercules-ci/gitignore.nix/archive/2ced4519f865341adcb143c5d668f955a2cb997f.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nix-npm-buildpackage": { + "branch": "balsoft/local-deps", + "description": "Build nix packages that use npm/yarn", + "homepage": "", + "owner": "serokell", + "repo": "nix-npm-buildpackage", + "rev": "14d03b37cd421b281835ae245b6cbf5b84c26e80", + "sha256": "13lz138rcy2vfd13sa4l2r4y5nx7v5pslxfy1vdq2phpmnn9j9yb", + "type": "tarball", + "url": "https://github.com/serokell/nix-npm-buildpackage/archive/14d03b37cd421b281835ae245b6cbf5b84c26e80.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixos-unstable", + "description": "Pinned Nixpkgs tree (master follows nixos-unstable-small, only tags have stable history)", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs-channels", + "rev": "3320a06049fc259e87a2bd98f4cd42f15f746b96", + "sha256": "1g5l186d5xh187vdcpfsz1ff8s749949c1pclvzfkylpar09ldkl", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs-channels/archive/3320a06049fc259e87a2bd98f4cd42f15f746b96.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "node-sass-bin": { + "sha256": "0dl91l414na44h090cgghd06q0j2whlj9h98im2qb9823glq7xff", + "type": "file", + "url": "https://github.com/sass/node-sass/releases/download/v4.12.0/linux-x64-64_binding.node", + "url_template": "https://github.com/sass/node-sass/releases/download/v/linux-x64-64_binding.node", + "version": "4.12.0" + }, + "opam-nix": { + "branch": "master", + "description": "A handy nix library to package OCaml software from OPAM repositories", + "homepage": null, + "owner": "balsoft", + "repo": "opam-nix", + "rev": "75ad75c18b8d80d82c13dc30c3b8d70960b1b441", + "sha256": "0mcr32f5i71diysvpblbvgdnx2pdymwjb9nxz5gbcsib3vdg2p83", + "type": "tarball", + "url": "https://github.com/balsoft/opam-nix/archive/75ad75c18b8d80d82c13dc30c3b8d70960b1b441.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "opam-repository": { + "branch": "master", + "description": "Main public package repository for OPAM, the source package manager of OCaml.", + "homepage": "https://opam.ocaml.org", + "owner": "ocaml", + "repo": "opam-repository", + "rev": "d67e70b40203a6a1c77ccb2edbe136c1509a73a3", + "sha256": "1yphw9xcss284p51qnml5jvfs4mhjcjgdka3wk25q0437zdzqj4n", + "type": "tarball", + "url": "https://github.com/ocaml/opam-repository/archive/d67e70b40203a6a1c77ccb2edbe136c1509a73a3.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "tezos-opam-repository": { + "ref": "master", + "repo": "https://gitlab.com/ligolang/tezos-opam-repository", + "rev": "dfc46bd895b070bd89028a7ad98741d05ec684df", + "type": "git" + }, + "xmldiff": { + "branch": "master", + "description": "Diffs on XML trees", + "homepage": "http://zoggy.github.io/xmldiff", + "owner": "zoggy", + "repo": "xmldiff", + "rev": "5873df80c1ba8c79287dc42e5008fac14cefffed", + "sha256": "0b92j9j2xdd2gyi9bgp39w0w1lkp0vc2rxq6wx6xdfvvklpprgb4", + "type": "tarball", + "url": "https://github.com/zoggy/xmldiff/archive/5873df80c1ba8c79287dc42e5008fac14cefffed.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..8a725cb4e --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,134 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: spec: + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; } + else + pkgs.fetchurl { inherit (spec) url sha256; }; + + fetch_tarball = pkgs: spec: + if spec.builtin or true then + builtins_fetchTarball { inherit (spec) url sha256; } + else + pkgs.fetchzip { inherit (spec) url sha256; }; + + fetch_git = spec: + builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; + + fetch_builtin-tarball = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-tarball" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=tarball -a builtin=true + '' + builtins_fetchTarball { inherit (spec) url sha256; }; + + fetch_builtin-url = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-url" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=file -a builtin=true + '' + (builtins_fetchurl { inherit (spec) url sha256; }); + + # + # Various helpers + # + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {}; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs spec + else if spec.type == "tarball" then fetch_tarball pkgs spec + else if spec.type == "git" then fetch_git spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec + else if spec.type == "builtin-url" then fetch_builtin-url spec + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball { inherit url; } + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl { inherit url; } + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = fetch config.pkgs name spec; } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? ./sources.json + , sources ? builtins.fromJSON (builtins.readFile sourcesFile) + , pkgs ? mkPkgs sources + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/nix/static-overlay.nix b/nix/static-overlay.nix new file mode 100644 index 000000000..5add8718f --- /dev/null +++ b/nix/static-overlay.nix @@ -0,0 +1,24 @@ +native: self: super: +let dds = x: x.overrideAttrs (o: { dontDisableStatic = true; }); +in { + buildPackages = super.buildPackages // { inherit (native) rakudo; }; + ocaml = self.ocaml-ng.ocamlPackages_4_07.ocaml; + libev = dds super.libev; + libusb = self.libusb1; + systemd = self.eudev; + libusb1 = dds (super.libusb1.override { + enableSystemd = true; + }); + gdb = null; + hidapi = dds (super.hidapi.override { systemd = self.eudev; }); + glib = (super.glib.override { libselinux = null; }).overrideAttrs + (o: { mesonFlags = o.mesonFlags ++ [ "-Dselinux=disabled" ]; }); + eudev = dds (super.eudev.overrideAttrs + (o: { nativeBuildInputs = o.nativeBuildInputs ++ [ super.gperf ]; })); + gmp = dds (super.gmp); + ocamlPackages = super.ocamlPackages.overrideScope' (self: super: { + ligo-out = super.ligo-out.overrideAttrs (_: { + patches = [ ./static.patch ]; + }); + }); +} diff --git a/nix/static.patch b/nix/static.patch new file mode 100644 index 000000000..f4ce39a39 --- /dev/null +++ b/nix/static.patch @@ -0,0 +1,12 @@ +diff --git a/src/bin/dune b/src/bin/dune +index 162963b4b..29dfa5191 100644 +--- a/src/bin/dune ++++ b/src/bin/dune +@@ -34,5 +34,6 @@ + (preprocess + (pps ppx_let ppx_blob bisect_ppx --conditional) + ) +- (flags (:standard -open Simple_utils)) ++ (flags (:standard -open Simple_utils) ++ -ccopt -static -cclib "-lgmp") + ) diff --git a/scripts/distribution/generic/env_variables.sh b/scripts/distribution/generic/env_variables.sh index 7279e4cbe..536ba0ad9 100755 --- a/scripts/distribution/generic/env_variables.sh +++ b/scripts/distribution/generic/env_variables.sh @@ -9,4 +9,6 @@ export LIGO_REGISTRY_IMAGE_BASE_NAME="ligolang/ligo" # ligo_incrementing-id_commit-hash export CI_JOB_ID="0" export CI_COMMIT_SHORT_SHA="$(git rev-parse --short HEAD)" +export CI_COMMIT_SHA="$(git rev-parse HEAD)" +export COMMIT_DATE="$(git show --no-patch --format=%ci)" export LIGO_DIST_DIR="./dist" diff --git a/scripts/distribution/generic/parameters.sh b/scripts/distribution/generic/parameters.sh index 2241b9fb0..16ff1f16b 100644 --- a/scripts/distribution/generic/parameters.sh +++ b/scripts/distribution/generic/parameters.sh @@ -15,6 +15,8 @@ target_os_version="$3" dist="$LIGO_DIST_DIR" version="$(echo $CI_JOB_ID)-$(echo $CI_COMMIT_SHORT_SHA)" ci_job_id="$CI_JOB_ID" +ci_commit_sha="$CI_COMMIT_SHA" +commit_date="$COMMIT_DATE" # Image names for building & packaging target="$target_os-$target_os_version" diff --git a/src/bin/cli.ml b/src/bin/cli.ml index 540b07f12..e7a629c32 100644 --- a/src/bin/cli.ml +++ b/src/bin/cli.ml @@ -2,12 +2,7 @@ open Cmdliner open Trace open Cli_helpers -let version = - Format.asprintf - "Rolling release\nHash: %s\nDate: %s\nCI job id: %s" - Version.hash - Version.commit_date - Version.job_id +let version = Version.version let main = let man = diff --git a/src/bin/dune b/src/bin/dune index 162963b4b..5d52779fc 100644 --- a/src/bin/dune +++ b/src/bin/dune @@ -18,7 +18,7 @@ (action (with-stdout-to version.ml (run "sh" "-c" - "printf 'let hash = \"%s\"\nlet commit_date = \"%s\"\nlet job_id = \"%s\"\n' \"$(git rev-parse HEAD)\" \"$(git show --no-patch --format=%ci)\" \"${CI_JOB_ID}\"")))) + "printf 'let version = \"Rolling Release\nCommit SHA1: %s\nCommit Date: %s\"' \"${CI_COMMIT_SHA}\" \"${COMMIT_DATE}\"")))) (executable (name runligo) diff --git a/src/stages/adt_generator/dune b/src/stages/adt_generator/dune index 0e1a15f71..e9f2660b3 100644 --- a/src/stages/adt_generator/dune +++ b/src/stages/adt_generator/dune @@ -1,6 +1,5 @@ (library (name adt_generator) (public_name ligo.adt_generator) - (libraries - ) + (libraries) ) diff --git a/src/test/adt_generator/dune b/src/test/adt_generator/dune index 63fabe8ed..4236b1815 100644 --- a/src/test/adt_generator/dune +++ b/src/test/adt_generator/dune @@ -7,7 +7,6 @@ (executable (name test_adt_generator) - (public_name ligo.test_adt_generator) (libraries adt_generator simple-utils) (preprocess (pps ppx_let bisect_ppx --conditional) diff --git a/tools/webide/packages/client/public/index.html b/tools/webide/packages/client/public/index.html index 7d5c78cf0..a1f3acb8d 100644 --- a/tools/webide/packages/client/public/index.html +++ b/tools/webide/packages/client/public/index.html @@ -34,7 +34,10 @@ gtag("config", "UA-153751765-1"); - + LIGO Playground diff --git a/tools/webide/packages/e2e/test/common-utils.js b/tools/webide/packages/e2e/test/common-utils.js index 68cddbadc..61cd3bef0 100644 --- a/tools/webide/packages/e2e/test/common-utils.js +++ b/tools/webide/packages/e2e/test/common-utils.js @@ -52,6 +52,8 @@ exports.API_ROOT = `${exports.API_HOST}/api`; exports.fetchExamples = async () => (await fetch(`${exports.API_HOST}/static/examples/list`)).json(); exports.runCommandAndGetOutputFor = async (command, endpoint) => { + page.on('console', msg => console.log('PAGE LOG:', msg.text())); + await page.click('#configure-tab'); await exports.sleep(1000); diff --git a/vendors/Preprocessor/Preprocessor.opam b/vendors/Preprocessor/Preprocessor.opam index e7d47fcdc..0a45620c3 100644 --- a/vendors/Preprocessor/Preprocessor.opam +++ b/vendors/Preprocessor/Preprocessor.opam @@ -8,7 +8,7 @@ authors : "Christian Rinderknecht" license : "MIT" homepage : "https://gitlab.com/ligolang/Preprocessor" bug-reports : "https://gitlab.com/ligolang/ligo-utils/issues" -depends : ["dune" "base" "ocaml" "simple-utils"] +depends : ["dune" "base" "ocaml" "simple-utils" "bisect_ppx" "menhir" "getopt"] build : [ [ "sh" "-c" "printf 'let version = \"%s\"' \"$(git describe --always --dirty --abbrev=0)\" > Version.ml" ] [ "dune" "build" "-p" name "-j" jobs ] diff --git a/vendors/ligo-utils/simple-utils/messages.sh b/vendors/ligo-utils/simple-utils/messages.sh index 418d09546..af46c9179 100755 --- a/vendors/ligo-utils/simple-utils/messages.sh +++ b/vendors/ligo-utils/simple-utils/messages.sh @@ -7,7 +7,7 @@ # specifications are located, in accordance with the convention of the # LIGO compiler source code. -#set -x +# set -x # ==================================================================== # General Settings and wrappers