From dbf9ccc16960a3e21742ab9455be92e392a25553 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 26 Mar 2020 02:48:32 +0300 Subject: [PATCH] Almost builds --- nix/lib.nix | 55 +++++++++------ nix/ocaml-overlay.nix | 78 +++++++++++++++++++-- nix/opam-parser.hs | 155 +++++++++++++++++++++++++++++------------- nix/sources.json | 30 ++++++++ 4 files changed, 246 insertions(+), 72 deletions(-) diff --git a/nix/lib.nix b/nix/lib.nix index 3ad473e7d..0fe10056b 100644 --- a/nix/lib.nix +++ b/nix/lib.nix @@ -22,7 +22,7 @@ in rec { opam2nix = { src, opamFile ? findOpamFile src, ... }@args: pkgs.runCommandNoCC (args.pname or opamFile + ".nix") args - "cat ${src}/${opamFile} | ${opam-parser} > $out"; + "cat ${src}/${opamFile} | ${pkgs.glibc.bin}/bin/iconv -c -f utf-8 -t ascii | ${opam-parser} > $out"; traverseOPAMRepo = repo: self: super: let @@ -33,25 +33,42 @@ in rec { (builtins.attrNames (builtins.readDir "${repo}/packages/${pkg}")); latestVersion = builtins.foldl' (x: acc: if builtins.compareVersions x acc == 1 then x else acc) ""; - opamPkgs = builtins.mapAttrs (n: _: + opamPkgs = builtins.mapAttrs (name: _: let - v = latestVersion (versions n); - file = opam2nix { - src = "${repo}/packages/${n}/${n}.${v}"; - opamFile = "opam"; - pname = n; - version = v; - }; - in self.callPackage file { }) pkgNames; + file = version: + opam2nix { + src = "${repo}/packages/${name}/${name}.${version}"; + opamFile = "opam"; + pname = name; + inherit version; + }; + package = version: self.callPackage (file version) { }; + latest = package (latestVersion (versions name)); + others = map package (versions name); + in latest // { + versions = builtins.listToAttrs (map (p: { + name = p.version; + value = p; + }) others); + }) pkgNames; in opamPkgs; - callOPAMPackage' = callPackage: src: newAttrs: overrides: - (callPackage (opam2nix (newAttrs // { inherit src; })) - overrides).overrideAttrs ({ buildInputs, src ? src, ... }: - { - inherit src; - buildInputs = buildInputs ++ newAttrs.extraBuildInputs or [ ]; - propagatedBuildInputs = buildInputs - ++ newAttrs.extraBuildInputs or [ ]; - } // newAttrs); + traverseOPAMRepo' = repo: self: super: + let traversed = traverseOPAMRepo repo self super; + in traversed // builtins.mapAttrs (name: v: + v // { + versions = (traversed.${name} or { versions = { }; }).versions; + }) super; + + callOPAMPackage = self: super: { + callOPAMPackage = src: newAttrs: overrides: + (self.callPackage (opam2nix (newAttrs // { inherit src; })) + overrides).overrideAttrs ({ buildInputs, ... }@args: + { + inherit src; + buildInputs = buildInputs ++ newAttrs.extraBuildInputs or [ ]; + propagatedBuildInputs = buildInputs + ++ newAttrs.extraBuildInputs or [ ]; + } // newAttrs); + }; } diff --git a/nix/ocaml-overlay.nix b/nix/ocaml-overlay.nix index 377ef67d7..c750a9f27 100644 --- a/nix/ocaml-overlay.nix +++ b/nix/ocaml-overlay.nix @@ -2,18 +2,82 @@ self: super: let lib = import ./lib.nix self; in { - ocamlPackages = super.ocaml-ng.ocamlPackages_4_09.overrideScope' + ocamlPackages = super.ocaml-ng.ocamlPackages_4_07.overrideScope' (builtins.foldl' self.lib.composeExtensions (_: _: { }) [ - (oself: osuper: - (lib.traverseOPAMRepo (builtins.fetchTarball - "https://github.com/ocaml/opam-repository/archive/master.tar.gz") oself osuper) - // osuper) - (lib.traverseOPAMRepo ../../tezos-opam-repository) + (lib.traverseOPAMRepo' sources.opam-repository) + (lib.traverseOPAMRepo sources.tezos-opam-repository) + (lib.callOPAMPackage) (oself: osuper: with oself; { ocamlfind = findlib; lwt = lwt4; - bigstring-unix = null; + bigstring = osuper.bigstring.overrideAttrs (_: { doCheck = false; }); + xmldiff = + osuper.xmldiff.overrideAttrs (_: { src = sources.xmldiff; }); + ipaddr = osuper.ipaddr.versions."4.0.0"; + conduit = osuper.conduit.versions."2.1.0"; + conduit-lwt-unix = osuper.conduit-lwt-unix.versions."2.0.2"; + cohttp-lwt-unix = osuper.cohttp-lwt-unix.versions."2.4.0"; + cohttp-lwt = osuper.cohttp-lwt.versions."2.4.0"; + macaddr = osuper.macaddr.versions."4.0.0"; + ocaml-migrate-parsetree = osuper.ocaml-migrate-parsetree.versions."1.4.0"; + ppx_tools_versioned = osuper.ppx_tools_versioned.versions."5.2.3"; + tezos-protocol-compiler = osuper.tezos-protocol-compiler.overrideAttrs + (oa: { + buildInputs = oa.buildInputs ++ [ oself.pprint ]; + postInstall = + "ln -s $out/lib/ocaml/*/site-lib/tezos-protocol-compiler/* $out"; + }); + conf-gmp = self.gmp; + conf-libev = self.libev; + conf-hidapi = self.hidapi; + conf-pkg-config = self.pkg-config; + + tezos-protocol-006-PsCARTHA = + oself.callOPAMPackage ../vendors/ligo-utils/tezos-protocol-alpha { + pname = "tezos-protocol-006-PsCARTHA"; + version = "0.0.0"; + opamFile = "tezos-protocol-006-PsCARTHA.opam"; + } { }; + + tezos-protocol-006-PsCARTHA-parameters = oself.callOPAMPackage + ../vendors/ligo-utils/tezos-protocol-alpha-parameters { + pname = "tezos-protocol-006-PsCARTHA-parameters"; + version = "0.0.0"; + } { }; + + tezos-utils = + oself.callOPAMPackage ../vendors/ligo-utils/tezos-utils { + pname = "tezos-utils"; + version = "0.1"; + } { }; + proto-alpha-utils = + oself.callOPAMPackage ../vendors/ligo-utils/proto-alpha-utils { + pname = "proto-alpha-utils"; + version = "0.1"; + extraBuildInputs = + [ oself.tezos-protocol-006-PsCARTHA-parameters ]; + } { }; + simple-utils = + oself.callOPAMPackage ../vendors/ligo-utils/simple-utils { + pname = "simple-utils"; + version = "0.1"; + } { }; + tezos-memory-proto-alpha = + oself.callOPAMPackage ../vendors/ligo-utils/memory-proto-alpha { + pname = "tezos-memory-proto-alpha"; + version = "0.0.0"; + } { }; + michelson-parser = oself.callOPAMPackage + ../vendors/ligo-utils/tezos-utils/michelson-parser { + pname = "michelson-parser"; + version = "0.0.0"; + } { ocamlfind = oself.findlib; }; + + ligo = oself.callOPAMPackage ../. { + version = "0.1"; + extraBuildInputs = with oself; [ ppx_tools_versioned getopt ]; + } { }; }) ]); } diff --git a/nix/opam-parser.hs b/nix/opam-parser.hs index a71315252..5c8cbd705 100644 --- a/nix/opam-parser.hs +++ b/nix/opam-parser.hs @@ -8,65 +8,121 @@ import Control.Monad (void) import Data.List (intersperse, nub, isPrefixOf) import qualified Control.Applicative as A (optional) -type Package = String - data OPAM = OPAM { name :: Maybe String , version :: Maybe String - , depends :: Maybe [String] - , build :: Maybe [[String]] - , source :: Maybe (String) + , nativeBuildInputs :: Maybe [String] + , buildInputs :: Maybe [String] + , buildPhase :: Maybe [[String]] + , checkInputs :: Maybe [String] + , checkPhase :: Maybe [[String]] + , source :: Maybe String } deriving Show - opam2nix :: OPAM -> String opam2nix OPAM {..} = - let depends' = nub ([ "findlib", "ocaml", "opaline", "dune" ] - ++ (map (\case 'b':'a':'s':'e':_ -> "base"; s -> s) - $ mconcat $ maybeToList depends)) + let + normalize = nub . map (\case 'b':'a':'s':'e':'-':_ -> "base"; s -> s) + buildInputs' = [ "findlib" ] ++ mconcat (maybeToList buildInputs); + checkInputs' = mconcat $ maybeToList checkInputs + nativeBuildInputs' = [ "dune", "opaline", "ocaml", "findlib" ] + ++ (if any (isPrefixOf "conf-") + (buildInputs' ++ checkInputs' ++ mconcat (maybeToList nativeBuildInputs)) + then ["conf-pkg-config"] + else []) + ++ mconcat (maybeToList nativeBuildInputs) + inputs = buildInputs' ++ checkInputs' ++ nativeBuildInputs' + deps = mconcat $ intersperse ", " $ normalize $ inputs + sepspace = mconcat . intersperse " " . normalize + preparephase = mconcat . intersperse " " . mconcat . intersperse ["\n"] in - "{ stdenv, fetchzip, " <> (mconcat $ intersperse ", " depends') <> " }:\n" + "{ stdenv, fetchzip, " <>deps<> " }:\n" <>"stdenv.mkDerivation rec {\n" - <>foldMap (\name -> " pname = \""<>name<>"\";\n") name - <>foldMap (\version -> " version = \""<>version<>"\";\n") version + <>foldMap (\name' -> " pname = \""<>name'<>"\";\n") name + <>foldMap (\version' -> " version = \""<>version'<>"\";\n") version <>foldMap (\url -> " src = builtins.fetchTarball { url = \""<>url<>"\"; };\n") source - <>" buildInputs = [ "<>(mconcat $ intersperse " " depends')<>" ];\n" + <>" buildInputs = [ "<>sepspace buildInputs'<>" ];\n" + <>" checkInputs = [ "<>sepspace checkInputs'<>" ];\n" + <>" nativeBuildInputs = [ "<>sepspace nativeBuildInputs'<>" ];\n" <>" propagatedBuildInputs = buildInputs;\n" - <>foldMap (\build -> " buildPhase = ''runHook preBuild\n"<>(mconcat $ intersperse " " $ mconcat $ intersperse ["\n"] $ build)<>"\nrunHook postBuild\n'';\n") build - <>(if "dune" `elem` depends' then " installPhase = ''\nrunHook preInstall\nopaline -prefix $out -libdir $OCAMLFIND_DESTDIR\nrunHook postInstall\n'';\n" else "") + <>" propagatedNativeBuildInputs = nativeBuildInputs;\n" + <>foldMap (\buildPhase' -> + " buildPhase = ''runHook preBuild\n" + <> preparephase buildPhase' + <>"\nrunHook postBuild\n'';\n") buildPhase + <>foldMap (\checkPhase' -> + " checkPhase = ''runHook preCheck\n" + <>preparephase checkPhase' + <>"\nrunHook postCheck\n'';\n") checkPhase + <>" installPhase = ''\nrunHook preInstall\nopaline -prefix $out -libdir $OCAMLFIND_DESTDIR\nrunHook postInstall\n'';\n" + <>"}\n" -getSha512 :: [String] -> String -getSha512 cs = (tail.tail.tail.tail.tail.tail.tail) $ head $ filter (isPrefixOf "sha512=") cs +update :: Maybe a -> a -> Maybe a +update old new = if isNothing old then Just new else old evaluateField :: OPAM -> Field -> OPAM -evaluateField o@(OPAM {..}) = \case - Name s -> o { name = if isNothing name then Just s else name } - Version s -> o { version = if isNothing version then Just s else version } - Depends s -> o { depends = if isNothing depends then Just s else depends } - Build e -> o { build = if isNothing build then Just (fmap (evaluateExp $ fromJust $ name) <$> e) else build } - URL url -> o { source = if isNothing source then Just url else source } +evaluateField o@OPAM {..} = \case + Name s -> o { name = update name s } + Version s -> o { version = update version s } + Depends s -> o { + buildInputs = update buildInputs $ + fmap identifier $ filter (\(Package _ info) -> not $ ("with-test" `elem` info || "build" `elem` info)) s, + nativeBuildInputs = update nativeBuildInputs $ + fmap identifier $ filter (\(Package _ info) -> "build" `elem` info) s, + checkInputs = update checkInputs $ + fmap identifier $ filter (\(Package _ info) -> "with-test" `elem` info) s + } + Build e -> o { + buildPhase = update buildPhase + $ fmap ((fmap evaluateExp) . command) $ filter (\(Command _ info) -> not $ "with-test" `elem` info) e, + checkPhase = update checkPhase + $ fmap ((fmap evaluateExp) . command) $ filter (\(Command _ info) -> "with-test" `elem` info) e + } + URL url -> o { source = update source url} Other _ -> o evaluateFields :: OPAM -> [Field] -> OPAM evaluateFields = foldl evaluateField +data Package + = Package + { identifier :: String + , additionalPackageInfo :: [String] + } deriving Show + +data Exp = Str String | Var String deriving Show + +data Command + = Command + { command :: [Exp] + , additionalCommandInfo :: [String] + } deriving Show + data Field = Name String | Version String | Depends [Package] - | Build [[Exp]] + | Build [Command] | URL String | Other String deriving Show -data Exp = Raw String | NameVar | JobsVar deriving Show - -evaluateExp :: String -> Exp -> String -evaluateExp name = \case - Raw s -> s - NameVar -> name - JobsVar -> "1" +evaluateExp :: Exp -> String +evaluateExp = + let + repl ('%':'{':xs) = '$':'{':repl xs + repl (':':_:_:_:'}':'%':xs) = '}':repl xs + repl (x:xs) = x:repl xs + repl "" = "" + in + \case + Str s -> repl s + Var "name" -> "${pname}" + Var "make" -> "make" + Var "prefix" -> "$out" + Var "jobs" -> "1" + Var s -> "${"<>s<>"}" opamFile :: ParsecT String u Identity [Field] opamFile = many field <* eof @@ -77,10 +133,10 @@ field = Name <$> fieldParser "name" stringParser <|> Depends <$> fieldParser "depends" (listParser packageParser) <|> Build <$> fieldParser "build" (pure <$> try commandParser <|> listParser commandParser) <|> sectionParser "url" (URL <$> (fieldParser "src" stringParser <* many (noneOf "}"))) - <|> Other <$> ((many (noneOf "\n")) <* char '\n') + <|> Other <$> (many (noneOf "\n") <* char '\n') fieldParser :: String -> ParsecT String u Identity t -> ParsecT String u Identity t -fieldParser name valueParser = try $ between (string (name<>":") >> (many $ oneOf " \n")) (many $ oneOf " \n") valueParser <* commentParser +fieldParser name valueParser = try $ between (string (name<>":") >> many (oneOf " \n")) (many $ oneOf " \n") valueParser <* commentParser sectionParser :: String -> ParsecT String u Identity t -> ParsecT String u Identity t sectionParser name valueParser = try $ between (string name >> many (oneOf " ") >> string "{" >> many (oneOf " \n")) (many (oneOf " \n") >> char '}' >> char '\n') valueParser @@ -89,15 +145,19 @@ stringParser :: ParsecT String u Identity String stringParser = between (char '"') (char '"') (many $ noneOf "\"") expParser :: ParsecT String u Identity Exp -expParser = try (string "name" >> return NameVar) - <|> try (string "jobs" >> return JobsVar) - <|> Raw <$> stringParser +expParser = try (Str <$> stringParser) + <|> Var <$> many1 (noneOf " \n\"{}[]") -commandParser :: ParsecT String u Identity [Exp] +additionalInfoParser :: ParsecT String u Identity [String] +additionalInfoParser = option [] $ try + $ between (many (char ' ') >> char '{') (char '}') + ((many $ noneOf " &}") `sepBy` (oneOf " &")) + +commandParser :: ParsecT String u Identity Command commandParser = do - command <- listParser expParser - optional $ try $ between (many (char ' ') >> char '{') (char '}') (many $ noneOf "}") - return command + command <- listParser $ try expParser + additionalInfo <- additionalInfoParser + return $ Command command additionalInfo commentParser :: ParsecT String u Identity () commentParser = optional $ do @@ -107,8 +167,8 @@ commentParser = optional $ do packageParser :: ParsecT String u Identity Package packageParser = do name <- stringParser - optional $ try $ between (many (char ' ') >> string "{") (char '}') (many $ noneOf "}") - return name + additionalInfo <- additionalInfoParser + return $ Package name additionalInfo listParser :: ParsecT String u Identity t -> ParsecT String u Identity [t] listParser valueParser = @@ -116,9 +176,9 @@ listParser valueParser = valueParser `sepBy` sep where startPadding = sep - endPadding = optional $ oneOf " \n" + endPadding = whiteSpace sep = (whiteSpace >> commentParser) <|> whiteSpace - whiteSpace = (optional $ many $ oneOf " \n") + whiteSpace = optional $ many $ oneOf " \n" main :: IO () main = do @@ -128,8 +188,11 @@ main = do <*> pure Nothing <*> pure Nothing <*> pure Nothing + <*> pure Nothing + <*> pure Nothing + <*> pure Nothing getContents >>= \s -> case parse opamFile "(unknown)" s of - Left e -> putStrLn $ show e + Left e -> print e Right fs -> putStrLn $ opam2nix $ evaluateFields initialOPAM fs - -- Right fs -> print fs + -- Right fs -> mapM_ print fs diff --git a/nix/sources.json b/nix/sources.json index 08be60b04..48ba70104 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,4 +1,16 @@ { + "UnionFind": { + "branch": "master", + "description": "Some variant implementations of the Union-Find algorithm", + "homepage": null, + "owner": "rinderknecht", + "repo": "UnionFind", + "rev": "97fdfa78a830e74fd3736e433152b662796840be", + "sha256": "0grsrv03x7r5azdcbcbsfdxq7myqnpr4wjgn14faxhnspkigcl9x", + "type": "tarball", + "url": "https://github.com/rinderknecht/UnionFind/archive/97fdfa78a830e74fd3736e433152b662796840be.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "bigstring": { "branch": "master", "description": "Overlay over bigarrays of chars", @@ -107,6 +119,18 @@ "url": "https://github.com/stedolan/ocaml-afl-persistent/archive/a37c4f581ba417d7e09a6efb34fdd8a90a7e9ede.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "opam-repository": { + "branch": "master", + "description": "Main public package repository for OPAM, the source package manager of OCaml.", + "homepage": "https://opam.ocaml.org", + "owner": "ocaml", + "repo": "opam-repository", + "rev": "d67e70b40203a6a1c77ccb2edbe136c1509a73a3", + "sha256": "1yphw9xcss284p51qnml5jvfs4mhjcjgdka3wk25q0437zdzqj4n", + "type": "tarball", + "url": "https://github.com/ocaml/opam-repository/archive/d67e70b40203a6a1c77ccb2edbe136c1509a73a3.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "resto": { "ref": "master", "repo": "https://gitlab.com/nomadic-labs/resto", @@ -137,6 +161,12 @@ "rev": "be706b7060cc9805d66fce1a032a586d09ee8718", "type": "git" }, + "tezos-opam-repository": { + "ref": "master", + "repo": "https://gitlab.com/ligolang/tezos-opam-repository", + "rev": "dfc46bd895b070bd89028a7ad98741d05ec684df", + "type": "git" + }, "uecc": { "ref": "master", "repo": "https://gitlab.com/nomadic-labs/ocaml-uecc",