Add documentation for nix expressions

nix/README.md

@@ -0,0 +1,28 @@
+# Nix expressions for building LIGO
+
+Nix is a declarative package manager. Get it here: https://nixos.org/nix
+
+These expressions are used on CI to reproducibly build the LIGO compiler, as well as WebIDE and https://ligolang.org .
+
+If you wish to build it yourself with `nix build -f. $thing`, where `$thing` is
+
+- `ligo`: executables, libraries, documentation, coverage reports
+- `ligo-bin`: a dynamically linked binary (Linux, Mac)
+- `ligo-static`: a statically linked binary (Linux only)
+- `ligo-doc`: documentation generated by odoc
+- `ligo-editor`: WebIDE, it can be started with `result/bin/ligo-editor`
+- `ligo-website`: the website, website root is `result`
+- `ligo-docker`: a docker image with LIGO binaries
+- `ligo-editor-docker`: a docker image with webide
+- `ligo-deb`: debian package with static binaries
+
+## Quick maintenance guide
+
+- `opam-repository` and `tezos-opam-repository` are pinned. To update them when required, run `niv update` (you can get niv with `nix shell nixpkgs#niv`)
+- `ocaml` version is pinned in `ocaml-overlay.nix`. If you want to update it, go there and change the version.
+- If something fails, `nix repl pkgs.nix` can be very useful to investigate it.
+
+## Known caveats
+
+- This is not a nix flake. This will never be a flake if we want to keep this low-maintenance, because of the way `opam` sources are defined. Sometimes, the checksum is omitted there, so we have to use `fetchTarball` without the checksum, which won't work in restricted mode (which is required for flakes). The only solution would be to generate nix expressions for opam-repository separately, but it means a manual step in the process (and it's also impossible to make this work as a flake).
+- For the same reason as above, evaluation can take a while because we need to download all the sources every `tarball-ttl` seconds. This can be mitigated by setting `tarball-ttl` to a high value.

nix/docker.nix

@@ -1,5 +1,6 @@
 { dockerTools, writeShellScriptBin, runCommand, mcpp, bash, coreutils, ligo, name ? "ligo" }:
 let
+  # LIGO requires /tmp for compilation, which is missing in the default image
   tmp = runCommand "tmp" {} "mkdir -p $out/tmp";
 in
 dockerTools.buildLayeredImage {

nix/ligo-editor.nix

@@ -2,10 +2,12 @@
 , writeShellScriptBin, makeFontsConf, buildEnv, rsync, sources
 , chromium ? null }:
 let
+  # Use a common yarn.lock for everything
   yarnLock = ../tools/webide/yarn.lock;
 
   installPhase = "mkdir $out; cp -Lr node_modules $out/node_modules";
 
+  # node_modules of the server
   server = mkYarnPackage {
     name = "webide-server";
     src = ../tools/webide/packages/server;
@@ -19,6 +21,8 @@ let
     distPhase = "true";
     inherit yarnLock installPhase;
   };
+
+  # node_modules of the client
   client = mkYarnPackage rec {
     name = "webide-client";
     src = ../tools/webide/packages/client;
@@ -42,6 +46,7 @@ let
     */
   };
 
+  # Perform the e2e tests; output is empty on purpose
   e2e = mkYarnPackage rec {
     name = "webide-e2e";
     src = ../tools/webide/packages/e2e;
@@ -61,6 +66,7 @@ let
     inherit yarnLock;
   };
 
+  # Run the WebIDE server with all the needed env variables
   ligo-editor = writeShellScriptBin "ligo-editor" ''
     set -e
     LIGO_CMD=${ligo-bin}/bin/ligo \

nix/nodejs-overlay.nix

@@ -1,4 +1,5 @@
 self: super: {
+  # Note: this overlay doesn't apply to nix-npm-buildpackage
   nodejs = super.nodejs-12_x;
   nodePackages = super.nodePackages_12_x;
   nodejs-slim = super.nodejs-slim-12_x;

nix/ocaml-overlay.nix

@@ -1,3 +1,5 @@
+# An overlay that adds ligo to ocamlPackages
+
 { sources ? import ./sources.nix
 , CI_COMMIT_SHA ? builtins.getEnv "CI_COMMIT_SHA"
 , COMMIT_DATE ? builtins.getEnv "COMMIT_DATE" }:
@@ -6,6 +8,7 @@ let
   opam-nix = import sources.opam-nix (import sources.nixpkgs { });
   inherit (import sources."gitignore.nix" { inherit (self) lib; })
     gitignoreSource;
+  # Remove list of directories or files from source (to stop unneeded rebuilds)
   filterOut = xs:
     self.lib.cleanSourceWith {
       filter = p: type: !(builtins.elem (builtins.baseNameOf p) xs);
@@ -14,6 +17,7 @@ let
 in {
   ocamlPackages = self.ocaml-ng.ocamlPackages_4_07.overrideScope'
     (builtins.foldl' self.lib.composeExtensions (_: _: { }) [
+      # Both opam-repository and tezos-opam-repository are updated manually with `niv update`
       (opam-nix.traverseOPAMRepo' sources.opam-repository)
       (opam-nix.traverseOPAMRepo sources.tezos-opam-repository)
       (opam-nix.callOPAMPackage (filterOut [
@@ -26,19 +30,23 @@ in {
         "gitlab-pages"
       ]))
       (oself: osuper: {
+        # Strange naming in nixpkgs
         ocamlfind = oself.findlib;
         lablgtk = null;
         lwt = oself.lwt4;
 
+        # Native dependencies
         conf-gmp = self.gmp;
         conf-libev = self.libev;
         conf-hidapi = self.hidapi;
         conf-pkg-config = self.pkg-config;
 
+        # Strange problems
         bigstring = osuper.bigstring.overrideAttrs (_: { doCheck = false; });
         xmldiff = osuper.xmldiff.overrideAttrs (_: { src = sources.xmldiff; });
         getopt = osuper.getopt.overrideAttrs (_: { configurePhase = "true"; });
 
+        # Force certain versions
         ipaddr = osuper.ipaddr.versions."4.0.0";
         conduit = osuper.conduit.versions."2.1.0";
         conduit-lwt-unix = osuper.conduit-lwt-unix.versions."2.0.2";
@@ -64,6 +72,7 @@ in {
             propagatedBuildInputs = buildInputs;
           });
 
+        # A combination of executables, libraries, documentation and test coverage
         ligo = self.buildEnv {
           name = "ligo";
           paths = with oself; [
@@ -74,6 +83,7 @@ in {
           ];
         };
 
+        # LIGO executable and public libraries
         ligo-out = osuper.ligo.overrideAttrs (oa: {
           name = "ligo-out";
           inherit CI_COMMIT_SHA COMMIT_DATE;
@@ -82,6 +92,8 @@ in {
           nativeBuildInputs = oa.nativeBuildInputs
             ++ [ self.buildPackages.rakudo ];
         });
+
+        # LIGO test suite; output empty on purpose
         ligo-tests = osuper.ligo.overrideAttrs (oa: {
           name = "ligo-tests";
           src = filterOut [
@@ -98,6 +110,7 @@ in {
             ++ [ self.buildPackages.rakudo ];
           installPhase = "mkdir $out";
         });
+        # LIGO odoc documentation
         ligo-doc = osuper.ligo.overrideAttrs (oa: {
           name = "ligo-doc";
           buildInputs = oa.buildInputs
@@ -109,6 +122,7 @@ in {
           installPhase =
             "mkdir $out; cp -r _build/default/_doc/_html/ $out/doc";
         });
+        # LIGO test coverage reports
         ligo-coverage = oself.ligo-tests.overrideAttrs (oa: {
           name = "ligo-coverage";
           nativeBuildInputs = oa.nativeBuildInputs

nix/packageDeb.nix

@@ -1,3 +1,4 @@
+# Create a debian package from static executable
 { stdenv, lib, writeTextFile, ligo-static, dpkg }:
 let
   project = "ligo";

nix/pkgs.nix

@@ -1,20 +1,25 @@
+# nixpkgs extended with all the overlays for LIGO
 { sources ? import ./sources.nix }:
 let
   ocaml-overlay = import ./ocaml-overlay.nix { inherit sources; };
   static-overlay = import ./static-overlay.nix pkgs;
   mac-overlay = import ./mac-overlay.nix;
   nodejs-overlay = import ./nodejs-overlay.nix;
+  nix-npm-buildpackage = pkgs.callPackage sources.nix-npm-buildpackage { };
+
   pkgs = import sources.nixpkgs {
     overlays = [ ocaml-overlay nodejs-overlay ]
+    # This is done here to prevent the need for bootstrap nixpkgs
       ++ (if builtins.currentSystem == "x86_64-darwin"
           then [ mac-overlay ]
           else [ ]);
   };
+
+  # Takes $pkg/ligo and creates a new package with $pkg/bin/ligo
   separateBinary = pkg:
     pkgs.runCommandNoCC "${pkg.name}-bin" { }
     "mkdir -p $out/bin; cp -Lr ${pkg}/ligo $out/bin";
 
-  nix-npm-buildpackage = pkgs.callPackage sources.nix-npm-buildpackage { };
 in pkgs.extend (self: super: {
   inherit (self.ocamlPackages) ligo ligo-out ligo-tests ligo-doc ligo-coverage;
   ligo-bin = separateBinary self.ligo-out.bin;

nix/static-overlay.nix

@@ -1,3 +1,6 @@
+# An overlay that adds flags needed to build LIGO statically;
+# Supposed to be applied to pkgsMusl
+# Takes `native` as a package set that doesn't cause mass rebuilds (so that we don't have to build perl with musl)
 native: self: super:
 let dds = x: x.overrideAttrs (o: { dontDisableStatic = true; });
 in {

nix/static.patch

@@ -1,5 +1,6 @@
 diff --git a/src/bin/dune b/src/bin/dune
 index 162963b4b..29dfa5191 100644
+With this patch, a static executable is produced
 --- a/src/bin/dune
 +++ b/src/bin/dune
 @@ -34,5 +34,6 @@