Merge branch 'feature/#3-add-odoc-to-website' into 'dev'

Feature/#3 add odoc to website

See merge request ligolang/ligo!23

.gitlab-ci.yml

@@ -12,32 +12,8 @@ stages:
   stage: build_and_deploy_website
   image: node:8
   before_script:
-    - cd gitlab-pages/website
+    - scripts/install_native_dependencies.sh
-    - npm install
+    # TODO: these things are moved to scripts in other branches.
-  script:
-    - npm run version next
-    - npm run build
-  after_script:
-    - cp -r gitlab-pages/website/build/ligo public 
-  artifacts:
-    paths:
-      - public
-
-.docker: &docker
-  image: docker:1.11
-  services:
-    - docker:dind
-
-.docker_build: &docker_build
-  script:
-    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
-
-.before_script: &before_script
-  before_script:
-    # Install dependencies
-    # rsync is needed by opam to sync a package installed from a local directory with the copy in ~/.opam
-    - apt-get update -qq
-    - apt-get -y -qq install rsync libhidapi-dev libcap-dev libev-dev bubblewrap
     - wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O opam-2.0.1-x86_64-linux
     - cp opam-2.0.1-x86_64-linux /usr/local/bin/opam
     - chmod +x /usr/local/bin/opam
@@ -57,12 +33,68 @@ stages:
     - printf '' | ocaml
     - opam switch
 
+    # install deps for internal documentation
+    - opam install -y odoc
+    - vendors/opam-repository-tools/rewrite-local-opam-repository.sh
+    - opam repository add localrepo "file://$PWD/vendors/ligo-opam-repository-local-generated/"
+    - opam install -y --build-test --deps-only ./src/
+    - dune build -p ligo
+    # TODO: also try instead from time to time:
+    #- (cd ./src/; dune build -p ligo)
+
+    # build with odoc
+    - dune build @doc
+
+    # npm
+    - cd gitlab-pages/website
+    - npm install
+  script:
+    - npm run version next
+    - npm run build
+    # move internal odoc documentation to the website folder
+    - mkdir -p gitlab-pages/website/build/ligo/
+    - mv ../../_build/default/_doc/_html/ gitlab-pages/website/build/ligo/odoc
+    - ls gitlab-pages/website/build/ligo/ # for debug
+  after_script:
+    - cp -r gitlab-pages/website/build/ligo public
+  artifacts:
+    paths:
+      - public
+
+.docker: &docker
+  image: docker:1.11
+  services:
+    - docker:dind
+
+.docker_build: &docker_build
+  script:
+    - docker build -t $LIGO_REGISTRY_IMAGE:next -f ./docker/Dockerfile .
+
+.before_script: &before_script
+  before_script:
+    # Install dependencies
+    # rsync is needed by opam to sync a package installed from a local directory with the copy in ~/.opam
+    - apt-get update -qq
+    - scripts/install_native_dependencies.sh
+    - scripts/install_opam.sh
+    - export PATH="/usr/local/bin${PATH:+:}${PATH:-}"
+
+    # Initialise opam, create switch, load opam environment variables
+    - printf '' | opam init --bare
+    - printf '' | opam switch create ligo-switch ocaml-base-compiler.4.06.1
+    - eval $(opam config env)
+
+    # Show versions and current switch
+    - echo "$PATH"
+    - opam --version
+    - printf '' | ocaml
+    - opam switch
+
 local-dune-job:
   <<: *before_script
   stage: test
   script:
-    - vendors/opam-repository-tools/rewrite-local-opam-repository.sh
+    - scripts/setup_ligo_opam_repository.sh
-    - opam repository add localrepo "file://$PWD/vendors/ligo-opam-repository-local-generated/"
     - opam install -y --build-test --deps-only ./src/
     - dune build -p ligo
     # TODO: also try instead from time to time:
@@ -126,4 +158,4 @@ pages:
   only:
     - master
     - dev
-
+    - feature/#3-add-odoc-to-website

Makefile

@@ -0,0 +1,3 @@
+build-deps:
+	scripts/install_native_dependencies.sh
+	scripts/install_opam.sh

docker/Dockerfile

@@ -16,12 +16,12 @@ ADD . /ligo
 # the upcoming scripts
 WORKDIR /ligo
 
-# Setup a custom opam repository where ligo is published
-RUN sh scripts/setup_ligo_opam_repository.sh
-
 # Install required native dependencies
 RUN sh scripts/install_native_dependencies.sh
 
+# Setup a custom opam repository where ligo is published
+RUN sh scripts/setup_ligo_opam_repository.sh
+
 RUN opam update
 
 # Install ligo

gitlab-pages/website/core/Footer.js

@@ -37,6 +37,9 @@ class Footer extends React.Component {
             <a href={this.docUrl('contributors/origin.html', this.props.language)}>
               Contribute
             </a>
+            <a href={this.docUrl('/odoc', this.props.language)}>
+              Api Documentation
+            </a>
           </div>
           <div>
             <h5>Community</h5>

gitlab-pages/website/siteConfig.js

@@ -100,6 +100,7 @@ const siteConfig = {
     {doc: 'setup/installation', label: 'Docs'},
     {doc: 'tutorials/get-started/tezos-taco-shop-smart-contract', label: 'Tutorials'},
     { blog: true, label: 'Blog' },
+    // TODO: { href: "/odoc", label: "Api" },
     {doc: 'contributors/origin', label: 'Contribute'},
     {href: 'https://discord.gg/9rhYaEt', label: ''},
     { search: true },

scripts/build_docker_image.sh

@@ -1 +1,4 @@
-docker build -t ligolang/ligo -f docker/Dockerfile .
+#!/bin/sh
+set -e
+
+docker build -t ligolang/ligo -f docker/Dockerfile .

scripts/install_ligo_with_dependencies.sh

@@ -1 +1,5 @@
-cd src && opam install . --yes
+#!/bin/sh
+set -e
+
+cd src
+opam install . --yes

scripts/install_native_dependencies.sh

@@ -1,7 +1,14 @@
-apt-get -y install \
+#!/bin/sh
+set -e
+
+apt-get update -qq
+apt-get -y -qq install \
     libev-dev \
     perl \
     pkg-config \
     libgmp-dev \
     libhidapi-dev \
-    m4
+    m4 \
+    libcap-dev \
+    bubblewrap \
+    rsync

scripts/install_opam.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+# TODO: this has many different modes of failure (file temp.opam-2.0.1-x86_64-linux.download-in-progress already exists, /usr/local/bin/opam already exists and is a directory or hard link, …)
+# Try to improve these aspects.
+
+wget https://github.com/ocaml/opam/releases/download/2.0.1/opam-2.0.1-x86_64-linux -O temp.opam-2.0.1-x86_64-linux.download-in-progress
+cp -i temp.opam-2.0.1-x86_64-linux.download-in-progress /usr/local/bin/opam
+chmod +x /usr/local/bin/opam
+rm temp.opam-2.0.1-x86_64-linux.download-in-progress

scripts/installer.sh

@@ -1,31 +1,96 @@
-#!/bin/bash
+#!/bin/sh
+set -e
+
 # You can run this installer like this:
 # curl https://gitlab.com/ligolang/ligo/blob/master/scripts/installer.sh | bash
-# Make sure the marigold/ligo image is published at docker hub first 
+# Make sure the marigold/ligo image is published at docker hub first
-set -euET -o pipefail
-version=$1
-printf "\nInstalling LIGO ($version)\n\n"
 
-if [ $version = "next" ]
+if test $# -ne 1; then
-    then
+  printf 'Usage: installer.sh VERSION'\\n
-        # Install the ligo.sh from master
+  printf \\n
-        wget https://gitlab.com/ligolang/ligo/raw/dev/scripts/ligo.sh
+  printf '  where VERSION can be "next" or a version number like 1.0.0'\\n
-    else
+  exit 1
-        # Install the ligo.sh from master
+else
-        wget https://gitlab.com/ligolang/ligo/raw/master/scripts/ligo.sh
+  version=$1
+  printf \\n'Installing LIGO (%s)'\\n\\n "$version"
+
+  if [ $version = "next" ]
+  then
+    # Install the ligo.sh from master
+    url=https://gitlab.com/ligolang/ligo/raw/dev/scripts/ligo.sh
+  else
+    # Install the ligo.sh from master
+    url=https://gitlab.com/ligolang/ligo/raw/master/scripts/ligo.sh
+  fi
+
+  # Pull the docker image used by ligo.sh
+  docker pull "ligolang/ligo:$version"
+
+  # Install ligo.sh
+  # Rationale behind this part of the script:
+  # * mv is one of the few commands which is atomic
+  #   * therefore we will create a file with the desired contents, and if that works, atomically mv it.
+  #     If something goes wrong it will attempt to remove the temporary file
+  #     (if removing the temporary file fails it's not a big deal due to the fairly explicit file name,
+  #      the fact that it is hidden, and its small size)
+  # * most utilities (e.g. touch) don't explicitly state that they support umask in their man page
+  #   * therefore we try to set the mode for the temporary file with an umask + do a chmod just to be sure
+  #   * this leaves open a race condition where:
+  #     0) umask isn't applied by touch (e.g. the file already exists)
+  #     1) for some reason touch creates an executable file (e.g. the file already exists)
+  #     2) a user grabs the file while it is executable, and triggers its execution (the process is created but execution of the script doesn't start yet)
+  #     3) chmod makes it non-executable
+  #     4) the file is partially written
+  #     5) the execution actually starts, and executes a prefix of the desired command, and that prefix is usable for adverse effects
+  #     To mitigate this, we wrap the command in the script with
+  #         if true; then the_command; fi
+  #     That way, the shell will raise an error due to a missing "fi" if the script executed while it is partially written
+  #   * This still leaves open the same race condition where a propper prefix of #!/bin/sh\nif can be used to adverse effect, but there's not much we can do about this.
+  # * after the file is completely written, we make it executable
+  # * we then check for the cases where `mv` misbehaves
+  # * we then atomically move it to (hopefully) its destination
+  # * the main risks here are if /usr/local/bin/ is writable by hostile users on the same machine (then there are bigger problems than what is our concern)
+  #   or if root itself tries to create a race condition (then there are bigger problems than what is our concern)
+
+  # It's hard to place comments inside a sequence of commands, so here are the comments for the following code:
+  # wget     download to stdout
+  # | sudo   become root (sudo) for the rest of the commands
+  #   (      subshell (to clean up temporary file if anything goes wrong)
+  #          remove temporary file in case it already exists
+  #   &&     create temporary file with (hopefully) the right permissions
+  #   &&     fix permisisons in case the creation didn't take umask into account
+  #   &&     redirect the output of the wget download to the temporary file
+  #   ) ||   clean up temporary file if any command in the previous block failed
+
+  wget "$url" -O - \
+  | sed -e "s/next/$version/g" \
+  | sudo sh -c ' \
+    ( \
+      rm -f /usr/local/bin/.temp.ligo.before-atomic-move \
+      && (umask 0600 > /dev/null 2>&1; UMASK=0600 touch /usr/local/bin/.temp.ligo.before-atomic-move) \
+      && chmod 0600 /usr/local/bin/.temp.ligo.before-atomic-move \
+      && cat > /usr/local/bin/.temp.ligo.before-atomic-move \
+    ) || (rm /usr/local/bin/.temp.ligo.before-atomic-move; exit 1)'
+
+  # sudo     become root (sudo) for the rest of the commands
+  #   (      subshell (to clean up temporary file if anything goes wrong)
+  #   &&     check that the download seems complete (one can't rely on sigpipe & failures to correctly stop the sudo session in case the download fails)
+  #   &&     overwite LIGO version in the executable
+  #   &&     now that the temporary file is complete, make it executable
+  #   && if  check for some corner cases: destination exists and is a directory
+  #   elif   check for some corner cases: destination exists and is symbolic link
+  #   else   atomically (hopefully) move temporary file to its destination
+  #   ) ||   clean up temporary file if any command in the previous block failed
+
+  sudo sh -c ' \
+    ( \
+         grep "END OF DOWNLOADED FILE" /usr/local/bin/.temp.ligo.before-atomic-move \
+      && chmod 0755 /usr/local/bin/.temp.ligo.before-atomic-move \
+      && if test -d /usr/local/bin/ligo; then printf "/usr/local/bin/ligo already exists and is a directory, cancelling installation"'\\\\'n; rm /usr/local/bin/.temp.ligo.before-atomic-move; \
+         elif test -L /usr/local/bin/ligo; then printf "/usr/local/bin/ligo already exists and is a symbolic link, cancelling installation"'\\\\'n; rm /usr/local/bin/.temp.ligo.before-atomic-move; \
+         else mv -i /usr/local/bin/.temp.ligo.before-atomic-move /usr/local/bin/ligo; fi \
+    ) || (rm /usr/local/bin/.temp.ligo.before-atomic-move; exit 1)'
+
+  # Installation finished, try running 'ligo' from your CLI
+  printf \\n'Installation successful, try to run '\''ligo --help'\'' now.'\\n
 fi
-
-
-# Overwrite LIGO version in the executable
-sed -i '' "s/latest/$version/g" ligo.sh
-
-# Copy the exucutable to the appropriate directory
-sudo cp ligo.sh /usr/local/bin/ligo
-sudo chmod +x /usr/local/bin/ligo
-rm ligo.sh
-
-# Pull the docker image used by ligo.sh
-docker pull "ligolang/ligo:$version"
-
-# Installation finished, try running 'ligo' from your CLI
-printf "\nInstallation successful, try to run 'ligo --help' now.\n"

scripts/ligo.sh

@@ -1,2 +1,10 @@
-#!/bin/bash
+#!/bin/sh
-docker run -it -v "$PWD":"$PWD" -w "$PWD" ligolang/ligo:latest "$@"
+set -e
+if [ test "x$PWD" = "x" ]; then
+  echo "Cannot detect the current directory, the environment variable PWD is empty."
+  exit 1
+else
+  docker run -it -v "$PWD":"$PWD" -w "$PWD" ligolang/ligo:next "$@"
+fi
+# Do not remove the next line. It is used as an approximate witness that the download of this file was complete. This string should not appear anywhere else in the file.
+# END OF DOWNLOADED FILE

scripts/setup_ligo_opam_repository.sh

@@ -1,3 +1,6 @@
+#!/bin/sh
+set -e
+
 vendors/opam-repository-tools/rewrite-local-opam-repository.sh
 opam repo add ligo-opam-repository ./vendors/ligo-opam-repository-local-generated
-opam update ligo-opam-repository
+opam update ligo-opam-repository

vendors/opam-repository-tools/rewrite-local-opam-repository.sh

@@ -1,13 +1,55 @@
-#!/bin/bash
+#!/bin/sh
-set -euET -o pipefail
+
-main(){
+# Stop on error.
-  root_dir="$(pwd | sed -e 's/\\/\\\\/' | sed -e 's/&/\\\&/' | sed -e 's/~/\\~/')"
+set -e
-  rm -fr vendors/ligo-opam-repository-local-generated
+
-  mkdir vendors/ligo-opam-repository-local-generated
+# Defensive checks. We're going to remove an entire folder so this script is somewhat dangerous. Better check in advance what can go wrong in the entire execution of the script.
-  cp -a index.tar.gz packages repo urls.txt vendors/ligo-opam-repository-local-generated
+if test -e index.tar.gz && test -e packages && test -e repo && test -e urls.txt; then
-  cd vendors/ligo-opam-repository-local-generated
+  if test -d vendors/; then
-  grep -r --null -l src: | grep -z 'opam$' | xargs -0 \
+   if test -d "$PWD"; then
-    sed -i -e 's~src:  *"https://gitlab.com/ligolang/ligo/-/archive/master/ligo\.tar\.gz"~src: "file://'"$root_dir"'"~'
+      if command -v sed   >/dev/null 2>&1 \
-  # TODO: run the update.sh script adequately to regenerate the index.tar.gz etc. in the local repo
+      && command -v rm    >/dev/null 2>&1 \
-}
+      && command -v mkdir >/dev/null 2>&1 \
-if main; then exit 0; else exit $?; fi
+      && command -v cp    >/dev/null 2>&1 \
+      && command -v find  >/dev/null 2>&1 \
+      && command -v xargs >/dev/null 2>&1 \
+      && command -v opam  >/dev/null 2>&1; then
+
+      # Escape the current directory, to be used as the replacement part of the sed regular expression
+      escaped_project_root="$(printf %s "$PWD" | sed -e 's/\\/\\\\/' | sed -e 's/&/\\\&/' | sed -e 's/~/\\~/')"
+
+      # Recreate vendors/ligo-opam-repository-local-generated which contains a copy of the files related to the opam repository
+      rm -fr vendors/ligo-opam-repository-local-generated
+      mkdir vendors/ligo-opam-repository-local-generated
+      cp -pR index.tar.gz packages repo urls.txt vendors/ligo-opam-repository-local-generated
+
+      # Rewrite the URLs in the opam repository to point to the project root
+      (
+        cd vendors/ligo-opam-repository-local-generated
+        find . -type f -name opam -print0 | xargs -0 sed -i -e 's~src:  *"https://gitlab.com/ligolang/ligo/-/archive/master/ligo\.tar\.gz"~src: "file://'"$escaped_project_root"'"~'
+      )
+
+      # Regenerate the index.tar.gz etc. in the local repo
+      (
+        cd vendors/ligo-opam-repository-local-generated
+        opam admin index
+        opam admin cache
+      )
+      else
+        echo "One of the following commands is unavailable: sed rm mkdir cp find xargs opam."
+        exit 1
+      fi
+    else
+      echo "Unable to access the current directory as indicated by PWD. Was the CWD of the current shell removed?"
+      exit 1
+    fi
+
+  else
+    echo "Cannot find the directory vendors/ in the current directory"
+    exit 1
+  fi
+else
+  echo "Cannot find some of the following files in the current directory"
+  echo "index.tar.gz packages repo urls.txt"
+  exit 1
+fi