diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 000000000..83d5dc40b --- /dev/null +++ b/nix/README.md @@ -0,0 +1,28 @@ +# Nix expressions for building LIGO + +Nix is a declarative package manager. Get it here: https://nixos.org/nix + +These expressions are used on CI to reproducibly build the LIGO compiler, as well as WebIDE and https://ligolang.org . + +If you wish to build it yourself with `nix build -f. $thing`, where `$thing` is + +- `ligo`: executables, libraries, documentation, coverage reports +- `ligo-bin`: a dynamically linked binary (Linux, Mac) +- `ligo-static`: a statically linked binary (Linux only) +- `ligo-doc`: documentation generated by odoc +- `ligo-editor`: WebIDE, it can be started with `result/bin/ligo-editor` +- `ligo-website`: the website, website root is `result` +- `ligo-docker`: a docker image with LIGO binaries +- `ligo-editor-docker`: a docker image with webide +- `ligo-deb`: debian package with static binaries + +## Quick maintenance guide + +- `opam-repository` and `tezos-opam-repository` are pinned. To update them when required, run `niv update` (you can get niv with `nix shell nixpkgs#niv`) +- `ocaml` version is pinned in `ocaml-overlay.nix`. If you want to update it, go there and change the version. +- If something fails, `nix repl pkgs.nix` can be very useful to investigate it. + +## Known caveats + +- This is not a nix flake. This will never be a flake if we want to keep this low-maintenance, because of the way `opam` sources are defined. Sometimes, the checksum is omitted there, so we have to use `fetchTarball` without the checksum, which won't work in restricted mode (which is required for flakes). The only solution would be to generate nix expressions for opam-repository separately, but it means a manual step in the process (and it's also impossible to make this work as a flake). +- For the same reason as above, evaluation can take a while because we need to download all the sources every `tarball-ttl` seconds. This can be mitigated by setting `tarball-ttl` to a high value. diff --git a/nix/docker.nix b/nix/docker.nix index acda1c971..99f262282 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,5 +1,6 @@ { dockerTools, writeShellScriptBin, runCommand, mcpp, bash, coreutils, ligo, name ? "ligo" }: let + # LIGO requires /tmp for compilation, which is missing in the default image tmp = runCommand "tmp" {} "mkdir -p $out/tmp"; in dockerTools.buildLayeredImage { diff --git a/nix/ligo-editor.nix b/nix/ligo-editor.nix index 9a1a88920..809881904 100644 --- a/nix/ligo-editor.nix +++ b/nix/ligo-editor.nix @@ -2,10 +2,12 @@ , writeShellScriptBin, makeFontsConf, buildEnv, rsync, sources , chromium ? null }: let + # Use a common yarn.lock for everything yarnLock = ../tools/webide/yarn.lock; installPhase = "mkdir $out; cp -Lr node_modules $out/node_modules"; + # node_modules of the server server = mkYarnPackage { name = "webide-server"; src = ../tools/webide/packages/server; @@ -19,6 +21,8 @@ let distPhase = "true"; inherit yarnLock installPhase; }; + + # node_modules of the client client = mkYarnPackage rec { name = "webide-client"; src = ../tools/webide/packages/client; @@ -42,6 +46,7 @@ let */ }; + # Perform the e2e tests; output is empty on purpose e2e = mkYarnPackage rec { name = "webide-e2e"; src = ../tools/webide/packages/e2e; @@ -61,6 +66,7 @@ let inherit yarnLock; }; + # Run the WebIDE server with all the needed env variables ligo-editor = writeShellScriptBin "ligo-editor" '' set -e LIGO_CMD=${ligo-bin}/bin/ligo \ diff --git a/nix/nodejs-overlay.nix b/nix/nodejs-overlay.nix index 7a5badf30..ab625530e 100644 --- a/nix/nodejs-overlay.nix +++ b/nix/nodejs-overlay.nix @@ -1,4 +1,5 @@ self: super: { + # Note: this overlay doesn't apply to nix-npm-buildpackage nodejs = super.nodejs-12_x; nodePackages = super.nodePackages_12_x; nodejs-slim = super.nodejs-slim-12_x; diff --git a/nix/ocaml-overlay.nix b/nix/ocaml-overlay.nix index d811becad..b44cfdcef 100644 --- a/nix/ocaml-overlay.nix +++ b/nix/ocaml-overlay.nix @@ -1,3 +1,5 @@ +# An overlay that adds ligo to ocamlPackages + { sources ? import ./sources.nix , CI_COMMIT_SHA ? builtins.getEnv "CI_COMMIT_SHA" , COMMIT_DATE ? builtins.getEnv "COMMIT_DATE" }: @@ -6,6 +8,7 @@ let opam-nix = import sources.opam-nix (import sources.nixpkgs { }); inherit (import sources."gitignore.nix" { inherit (self) lib; }) gitignoreSource; + # Remove list of directories or files from source (to stop unneeded rebuilds) filterOut = xs: self.lib.cleanSourceWith { filter = p: type: !(builtins.elem (builtins.baseNameOf p) xs); @@ -14,6 +17,7 @@ let in { ocamlPackages = self.ocaml-ng.ocamlPackages_4_07.overrideScope' (builtins.foldl' self.lib.composeExtensions (_: _: { }) [ + # Both opam-repository and tezos-opam-repository are updated manually with `niv update` (opam-nix.traverseOPAMRepo' sources.opam-repository) (opam-nix.traverseOPAMRepo sources.tezos-opam-repository) (opam-nix.callOPAMPackage (filterOut [ @@ -26,19 +30,23 @@ in { "gitlab-pages" ])) (oself: osuper: { + # Strange naming in nixpkgs ocamlfind = oself.findlib; lablgtk = null; lwt = oself.lwt4; + # Native dependencies conf-gmp = self.gmp; conf-libev = self.libev; conf-hidapi = self.hidapi; conf-pkg-config = self.pkg-config; + # Strange problems bigstring = osuper.bigstring.overrideAttrs (_: { doCheck = false; }); xmldiff = osuper.xmldiff.overrideAttrs (_: { src = sources.xmldiff; }); getopt = osuper.getopt.overrideAttrs (_: { configurePhase = "true"; }); + # Force certain versions ipaddr = osuper.ipaddr.versions."4.0.0"; conduit = osuper.conduit.versions."2.1.0"; conduit-lwt-unix = osuper.conduit-lwt-unix.versions."2.0.2"; @@ -64,6 +72,7 @@ in { propagatedBuildInputs = buildInputs; }); + # A combination of executables, libraries, documentation and test coverage ligo = self.buildEnv { name = "ligo"; paths = with oself; [ @@ -74,6 +83,7 @@ in { ]; }; + # LIGO executable and public libraries ligo-out = osuper.ligo.overrideAttrs (oa: { name = "ligo-out"; inherit CI_COMMIT_SHA COMMIT_DATE; @@ -82,6 +92,8 @@ in { nativeBuildInputs = oa.nativeBuildInputs ++ [ self.buildPackages.rakudo ]; }); + + # LIGO test suite; output empty on purpose ligo-tests = osuper.ligo.overrideAttrs (oa: { name = "ligo-tests"; src = filterOut [ @@ -98,6 +110,7 @@ in { ++ [ self.buildPackages.rakudo ]; installPhase = "mkdir $out"; }); + # LIGO odoc documentation ligo-doc = osuper.ligo.overrideAttrs (oa: { name = "ligo-doc"; buildInputs = oa.buildInputs @@ -109,6 +122,7 @@ in { installPhase = "mkdir $out; cp -r _build/default/_doc/_html/ $out/doc"; }); + # LIGO test coverage reports ligo-coverage = oself.ligo-tests.overrideAttrs (oa: { name = "ligo-coverage"; nativeBuildInputs = oa.nativeBuildInputs diff --git a/nix/packageDeb.nix b/nix/packageDeb.nix index bb5f0a57b..d7366e797 100644 --- a/nix/packageDeb.nix +++ b/nix/packageDeb.nix @@ -1,3 +1,4 @@ +# Create a debian package from static executable { stdenv, lib, writeTextFile, ligo-static, dpkg }: let project = "ligo"; diff --git a/nix/pkgs.nix b/nix/pkgs.nix index d832dde56..23c2abe5a 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -1,20 +1,25 @@ +# nixpkgs extended with all the overlays for LIGO { sources ? import ./sources.nix }: let ocaml-overlay = import ./ocaml-overlay.nix { inherit sources; }; static-overlay = import ./static-overlay.nix pkgs; mac-overlay = import ./mac-overlay.nix; nodejs-overlay = import ./nodejs-overlay.nix; + nix-npm-buildpackage = pkgs.callPackage sources.nix-npm-buildpackage { }; + pkgs = import sources.nixpkgs { overlays = [ ocaml-overlay nodejs-overlay ] + # This is done here to prevent the need for bootstrap nixpkgs ++ (if builtins.currentSystem == "x86_64-darwin" then [ mac-overlay ] else [ ]); }; + + # Takes $pkg/ligo and creates a new package with $pkg/bin/ligo separateBinary = pkg: pkgs.runCommandNoCC "${pkg.name}-bin" { } "mkdir -p $out/bin; cp -Lr ${pkg}/ligo $out/bin"; - nix-npm-buildpackage = pkgs.callPackage sources.nix-npm-buildpackage { }; in pkgs.extend (self: super: { inherit (self.ocamlPackages) ligo ligo-out ligo-tests ligo-doc ligo-coverage; ligo-bin = separateBinary self.ligo-out.bin; diff --git a/nix/static-overlay.nix b/nix/static-overlay.nix index 5add8718f..e16aa8646 100644 --- a/nix/static-overlay.nix +++ b/nix/static-overlay.nix @@ -1,3 +1,6 @@ +# An overlay that adds flags needed to build LIGO statically; +# Supposed to be applied to pkgsMusl +# Takes `native` as a package set that doesn't cause mass rebuilds (so that we don't have to build perl with musl) native: self: super: let dds = x: x.overrideAttrs (o: { dontDisableStatic = true; }); in { diff --git a/nix/static.patch b/nix/static.patch index f4ce39a39..efe5c43b8 100644 --- a/nix/static.patch +++ b/nix/static.patch @@ -1,5 +1,6 @@ diff --git a/src/bin/dune b/src/bin/dune index 162963b4b..29dfa5191 100644 +With this patch, a static executable is produced --- a/src/bin/dune +++ b/src/bin/dune @@ -34,5 +34,6 @@