View on GitHub
File Changes
+
#!/usr/bin/env bash
+

                      
+
set -euo pipefail
+

                      
+
stylish-haskell -i `git ls-files -- '*.hs'`
+

                      
+
git diff --exit-code
+
{ walletPackages ? import ./.. {}
+
, pkgs ? walletPackages.pkgs
+
}:
+

                      
+
with pkgs.lib;
+
with pkgs;
+

                      
+
let
+
  buildTools = [
+
    gnused coreutils git nix gnumake gnutar gzip lz4
+
    stack walletPackages.iohkLib.stack-hpc-coveralls
+
    haskellPackages.weeder
+
  ];
+
  libs = ps: with ps; [turtle safe transformers extra async];
+

                      
+
  ghc' = haskellPackages.ghcWithPackages libs;
+
  stackRebuild = runCommand "stack-rebuild" {
+
    buildInputs = [ ghc' makeWrapper ];
+
  } ''
+
    mkdir -p $out/bin
+
    ghc -Wall -threaded -o $out/bin/rebuild ${./rebuild.hs}
+
    wrapProgram $out/bin/rebuild --set PATH "${lib.makeBinPath buildTools}"
+
  '';
+

                      
+
in
+
  stackRebuild
    timeout_in_minutes: 60
    agents:
      system: x86_64-linux
+

                      
+
  - label: 'Clean up CI cache'
+
    env:
+
      STACK_ROOT: "/build/cardano-wallet.stack"
+
    command:
+
      - "nix-build .buildkite/default.nix -o sr"
+
      - "./sr/bin/rebuild cleanup-cache --build-dir=/build/cardano-wallet --cache-dir=/build/cardano-wallet.cache"
+
    agents:
+
      system: x86_64-linux
+

                      
+
  - block: 'Delete CI Caches'
+
  - label: 'Purge CI cache'
+
    env:
+
      STACK_ROOT: "/build/cardano-wallet.stack"
+
    command:
+
      - "nix-build .buildkite/default.nix -o sr"
+
      - "./sr/bin/rebuild purge-cache --build-dir=/build/cardano-wallet --cache-dir=/build/cardano-wallet.cache"
+
    agents:
+
      system: x86_64-linux
+
  - wait
+
  - label: "Rebuild master branch"
+
    trigger: "cardano-wallet"
steps:
+
  - label: 'Stack Rebuild'
+
    env:
+
      STACK_ROOT: "/build/cardano-wallet.stack"
+
    command:
+
      - "nix-build .buildkite/default.nix -o sr"
+
      - "./sr/bin/rebuild --build-dir /build/cardano-wallet --cache-dir /build/cardano-wallet.cache"
+
    timeout_in_minutes: 120
+
    agents:
+
      system: x86_64-linux
+

                      
  - label: Check Hydra evaluation of release.nix
    command: 'nix-build -A iohkLib.check-hydra -o check-hydra.sh && ./check-hydra.sh'
    agents:
      system: x86_64-linux

                      
  - label: Check auto-generated Nix
-
    command: 'nix-build -A checks.check-nix-tools -o check-nix-tools.sh && ./check-nix-tools.sh'
+
    command: 'nix-build -A iohkLib.check-nix-tools -o check-nix-tools.sh && ./check-nix-tools.sh'
+
    agents:
+
      system: x86_64-linux
+

                      
+
  - label: 'Check Stylish Haskell'
+
    command: 'nix-shell --run .buildkite/check-stylish.sh'
+
    agents:
+
      system: x86_64-linux
+

                      
+
  - label: 'HLint'
+
    command: 'nix-shell --run "hlint exe lib"'
+
    agents:
+
      system: x86_64-linux
+

                      
+
  - label: 'Validate OpenAPI Specification'
+
    command: 'nix-shell --run "openapi-spec-validator --schema 2.0 specifications/api/swagger.yaml"'
    agents:
      system: x86_64-linux
+
{-# LANGUAGE LambdaCase #-}
+
{-# LANGUAGE OverloadedStrings #-}
+
{-# LANGUAGE RecordWildCards #-}
+
{-# LANGUAGE ScopedTypeVariables #-}
+

                      
+
import Prelude hiding
+
    ( FilePath )
+

                      
+
import Options.Applicative
+

                      
+
import Control.Concurrent.Async
+
    ( race )
+
import Control.Exception
+
    ( IOException, catch )
+
import Control.Monad
+
    ( filterM, forM_ )
+
import Control.Monad.Extra
+
    ( whenM )
+
import Control.Monad.Trans.Class
+
    ( lift )
+
import Control.Monad.Trans.Maybe
+
    ( MaybeT (..) )
+
import Data.ByteString
+
    ( ByteString )
+
import Data.List.Extra
+
    ( dropSuffix )
+
import Data.Maybe
+
    ( mapMaybe, maybeToList )
+
import Safe
+
    ( headMay, readMay )
+
import System.Exit
+
    ( exitWith )
+
import Turtle hiding
+
    ( opt, option )
+

                      
+
import qualified Control.Foldl as Fold
+
import qualified Data.Text as T
+
import qualified Filesystem.Path.CurrentOS as FP
+
import qualified Turtle.Bytes as TB
+

                      
+
main :: IO ()
+
main = do
+
    (RebuildOpts{..}, cmd) <- parseOpts
+
    bk <- getBuildkiteEnv
+
    cacheConfig <- getCacheConfig bk optCacheDirectory
+
    case cmd of
+
        Build -> do
+
            doMaybe setupBuildDirectory optBuildDirectory
+
            cacheGetStep cacheConfig
+
            buildResult <- buildStep Nothing
+
            cachePutStep cacheConfig
+
            -- uploadCoverageStep
+
            weederStep
+
            exitWith buildResult
+
        CleanupCache ->
+
            cleanupCacheStep cacheConfig optBuildDirectory
+
        PurgeCache ->
+
            purgeCacheStep cacheConfig optBuildDirectory
+

                      
+
data BuildOpt = Opt | Fast deriving (Show, Eq)
+

                      
+
data RebuildOpts = RebuildOpts
+
    { optBuildDirectory :: Maybe FilePath
+
    , optCacheDirectory :: Maybe FilePath
+
    } deriving (Show)
+

                      
+
data Command = Build | CleanupCache | PurgeCache deriving (Show)
+

                      
+
rebuildOpts :: Parser RebuildOpts
+
rebuildOpts = RebuildOpts
+
    <$> optional buildDir
+
    <*> optional cacheName
+
  where
+
    buildDir = option
+
        (FP.decodeString <$> str)
+
        (  long "build-dir"
+
        <> metavar "DIR"
+
        <> help "Copy sources to directory before building"
+
        )
+
    cacheName = option
+
        (FP.decodeString <$> str)
+
        (  long "cache-dir"
+
        <> metavar "DIR"
+
        <> help "Location of project's cache"
+
        )
+

                      
+
parseOpts :: IO (RebuildOpts, Command)
+
parseOpts = execParser opts
+
  where
+
    opts = info
+
        (cmdOpts <**> helper)
+
        (fullDesc <> progDesc "Build cardano-wallet with stack in Buildkite")
+
    cmdOpts = (,)
+
        <$> rebuildOpts
+
        <*> (cmd <|> pure Build)
+
    cmd = subparser
+
        (  command "build" (info (pure Build) idm)
+
        <> command "cleanup-cache" (info (pure CleanupCache) idm)
+
        <> command "purge-cache" (info (pure PurgeCache) idm)
+
        )
+

                      
+
buildStep :: Maybe [Text] -> IO ExitCode
+
buildStep testArgs =
+
    echo "--- Build LTS Snapshot" *> buildSnapshot .&&.
+
    echo "--- Build dependencies" *> buildDeps .&&.
+
    echo "+++ Build" *> build .&&.
+
    echo "+++ Test" *> timeout 30 test
+
  where
+
    cfg = ["--dump-logs", "--color", "always"]
+
    buildArgs =
+
        [ "--bench"
+
        , "--no-run-benchmarks"
+
        , "--haddock"
+
        , "--haddock-internal"
+
        , "--no-haddock-deps"
+
        ]
+
    testArgs' = maybe [] ("--ta" :) testArgs
+

                      
+
    stackBuild opt args = run "stack" $ concat [cfg, ["build"], fastOpt opt, args]
+
    buildSnapshot = stackBuild Opt  $ buildArgs ++ ["--only-snapshot"]
+
    buildDeps     = stackBuild Opt  $ buildArgs ++ ["--only-dependencies"]
+
    build         = stackBuild Fast $ ["--test", "--no-run-tests"] ++ buildArgs
+
    test          = stackBuild Fast $ "--test" : testArgs'
+

                      
+
    fastOpt Opt = []
+
    fastOpt Fast = ["--fast"]
+

                      
+
-- Stack with caching needs a build directory that is the same across
+
-- all BuildKite agents. The build directory option can be used to
+
-- ensure this is the case.
+
setupBuildDirectory :: FilePath -> IO ()
+
setupBuildDirectory buildDir = do
+
    removeDirectory buildDir
+
    src <- pwd
+
    printf ("Copying source tree "%fp%" -> "%fp%"\n") src buildDir
+
    cptree src buildDir
+
    cd buildDir
+

                      
+
----------------------------------------------------------------------------
+
-- Buildkite
+
-- https://buildkite.com/docs/pipelines/environment-variables
+

                      
+
-- | A selection of relevant pipeline and build information from Buildkite.
+
data BuildkiteEnv = BuildkiteEnv
+
    { bkBuildNum      :: Int
+
    -- ^ The Buildkite build number.
+
    , bkPipeline      :: Text
+
    -- ^ The pipeline slug on Buildkite as used in URLs.
+
    , bkBranch        :: Text
+
    -- ^ The branch being built.
+
    , bkBaseBranch    :: Maybe Text
+
    -- ^ The base branch that the pull request is targeting, if this
+
    -- build is for a pull request.
+
    , bkDefaultBranch :: Text
+
    -- ^ The default branch for the pipeline (e.g. master).
+
    , bkTag           :: Maybe Text
+
    -- ^ The name of the tag being built, if this build was triggered from a tag.
+
    } deriving (Show)
+

                      
+
-- | Fetch build parameters from the environment.
+
getBuildkiteEnv :: IO (Maybe BuildkiteEnv)
+
getBuildkiteEnv = runMaybeT $ do
+
    bkBuildNum      <- MaybeT $ needRead "BUILDKITE_BUILD_NUMBER"
+
    bkPipeline      <- MaybeT $ need "BUILDKITE_PIPELINE_SLUG"
+
    bkBranch        <- MaybeT $ need "BUILDKITE_BRANCH"
+
    bkBaseBranch    <- lift   $ want "BUILDKITE_PULL_REQUEST_BASE_BRANCH"
+
    bkDefaultBranch <- MaybeT $ need "BUILDKITE_PIPELINE_DEFAULT_BRANCH"
+
    bkTag           <- lift   $ want "BUILDKITE_TAG"
+
    pure BuildkiteEnv {..}
+

                      
+
----------------------------------------------------------------------------
+
-- Weeder - uses contents of .stack-work to determine unused dependencies
+

                      
+
weederStep :: IO ()
+
weederStep = do
+
    echo "--- Weeder"
+
    procs "weeder" [] empty
+

                      
+
----------------------------------------------------------------------------
+
-- Stack Haskell Program Coverage and upload to Coveralls
+

                      
+
uploadCoverageStep :: IO ()
+
uploadCoverageStep = do
+
    echo "--- Upload Test Coverage"
+
    sh $ do
+
        -- Ignore modules that are full of Template Haskell auto-generated code
+
        pushd "lib/core"
+
        let localMixDir = ".stack-work/dist/x86_64-linux/Cabal-2.4.0.1/hpc/" -- fixme: find it
+
        void $ proc "sed" ["-i", "s/.*hpc\\/\\(.*\\).mix/module \"\\1\" {}/", "overlay.hpc"] empty
+
        let ignoredTix = "Cardano.Wallet.DB.Sqlite.TH.tix"
+
        inproc "hpc" ["overlay", "--hpcdir", localMixDir, "overlay.hpc"] empty &
+
            output ignoredTix
+
        void $ proc "sed" ["-i", "s/0,/1,/g", format fp ignoredTix] empty
+
    void $
+
        (proc "stack" ["hpc", "report", "**/*.tix"] empty) .&&.
+
        (proc "shc" ["combined", "custom"] empty)
+

                      
+
----------------------------------------------------------------------------
+
-- Stack root and .stack-work caching.
m
+3/-4
      text-class
    ];
    buildInputs =
-
      with pkgs.haskellPackages; [ hlint stylish-haskell weeder ghcid ]
+
      with pkgs.haskellPackages; [ stylish-haskell weeder ghcid ]
      ++ [ cardano-sl-node cardano-http-bridge jormungandr jormungandr-cli
-
           pkgs.pkgconfig pkgs.sqlite-interactive ];
+
           pkgs.pkgconfig pkgs.sqlite-interactive
+
           iohkLib.hlint iohkLib.openapi-spec-validator ];
  };
-

                      
-
  checks.check-nix-tools = pkgs.callPackage ./nix/check-nix-tools.nix {};
}
-
# This is a a CI check script that runs nix/regenerate.sh in the
-
# project repository. If anything changes, then it uploads the patch
-
# to Buildkite. If run on a PR branch, and there is a SSH key present,
-
# it will attempt to push the changes back to the PR.
-

                      
-
{ stdenv, writeScript, coreutils, nixStable, git, gawk }:
-

                      
-
with stdenv.lib;
-

                      
-
writeScript "check-nix-tools.sh" ''
-
  #!${stdenv.shell}
-

                      
-
  set -euo pipefail
-

                      
-
  export PATH="${makeBinPath [ stdenv.shellPackage coreutils nixStable git gawk ]}:$PATH"
-

                      
-
  cd $(git rev-parse --show-toplevel)
-

                      
-
  # The regenerate script is here by convention
-
  ./nix/regenerate.sh
-

                      
-
  # The generated files will appear somewhere under ./nix
-
  git add -A nix
-

                      
-
  # Check if there are changes staged for commit.
-
  if git diff-index --cached --quiet HEAD --; then
-
    echo "Generated Nix code is up-to-date."
-
    exit 0
-
  else
-
    echo "Committing changes..."
-
    commit_message="Regenerate nix"
-

                      
-
    # If on a PR branch, search for a previous regen commit to fix up.
-
    commit_fixup=""
-
    if [ -n "''${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}" ]; then
-
      git fetch -v origin $BUILDKITE_PULL_REQUEST_BASE_BRANCH
-
      commit_fixup=$(git log --pretty=oneline --no-decorate origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH..HEAD | awk "/$commit_message/ { print \$1; }")
-
    fi
-

                      
-
    # Create the commit
-
    export GIT_COMMITTER_NAME="IOHK"
-
    export GIT_COMMITTER_EMAIL="[email protected]"
-
    export GIT_AUTHOR_NAME="$GIT_COMMITTER_NAME"
-
    export GIT_AUTHOR_EMAIL="$GIT_COMMITTER_EMAIL"
-
    if [ -n "$commit_fixup" ]; then
-
      git commit --no-gpg-sign --fixup "$commit_fixup"
-
    else
-
      git commit --no-gpg-sign --message "$commit_message"
-
    fi
-

                      
-
    # If running in Buildkite...
-
    if [ -n "''${BUILDKITE_JOB_ID:-}" ]; then
-

                      
-
      # Upload the patch as a Buildkite artifact
-
      patch="$BUILDKITE_PIPELINE_SLUG-nix-$BUILDKITE_BUILD_NUMBER.patch"
-
      git format-patch --stdout -1 HEAD > "$patch"
-
      buildkite-agent artifact upload "$patch" --job "$BUILDKITE_JOB_ID"
-

                      
-
      # Push the changes back to the pull request
-
      if [ -n "''${BUILDKITE_PULL_REQUEST_REPO:-}" ]; then
-
        sshkey="/run/keys/buildkite-$BUILDKITE_PIPELINE_SLUG-ssh-private"
-
        if [ -e $sshkey ]; then
-
          echo "Authenticating using SSH with $sshkey"
-
          export GIT_SSH_COMMAND="ssh -i $sshkey -F /dev/null"
-
          remote=$(echo $BUILDKITE_PULL_REQUEST_REPO | sed -e 's=^[a-z]*://github.com/[email protected]:=')
-
          git push $remote HEAD:$BUILDKITE_BRANCH
-
          exit 0
-
        else
-
          echo "There is no SSH key at $sshkey"
-
          echo "The updates can't be pushed."
-
          echo "Apply the patch $patch from the build artifacts"
-
        fi
-
      fi
-

                      
-
    fi
-

                      
-
    exit 1
-
  fi
-
''
{
  "url": "https://github.com/input-output-hk/iohk-nix",
-
  "rev": "17b43fa1f9335b15065b2076512d8f24aa2c7442",
-
  "date": "2019-08-28T06:14:11+00:00",
-
  "sha256": "1z529msqjw2s8hbb4vmjii83d6s8aw4da09qdd58j08vzqkgwvp6",
+
  "rev": "a3343c32ade0a7f45ae54541e72f5cdf70f20650",
+
  "date": "2019-09-08T21:25:46+00:00",
+
  "sha256": "0y15jwg8wjnhv8y0h0nyk3s7c0lcn4b860qdz3lnhv3phbnld3by",
  "fetchSubmodules": false
}
  ghc = walletPackages.haskellPackages._config.ghc.package;

                      
  buildInputs =
-
    (with walletPackages; [ cardano-http-bridge jormungandr cardano-sl-node ]) ++
-
    [ zlib gmp ncurses lzma openssl git systemd.dev ] ++
+
    (with walletPackages; [ cardano-sl-node cardano-http-bridge jormungandr jormungandr-cli ]) ++
+
    [ zlib gmp ncurses lzma openssl git systemd.dev nodejs ] ++
    (lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ Cocoa CoreServices libcxx libiconv ]));

                      
  phases = ["nobuildPhase"];