Something went wrong. Try again.
A collection of +180 reusable WebAssembly modules for building composable CI/CD pipelines with FluentCI fluentci.io/modules
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328module CI.CacheHost where
import Preludeimport Control.Promise (Promise, toAffE)import Data.Array as Aimport Data.Either (Either(..), either)import Data.Foldable (for_)import Data.Int as Intimport Data.Maybe (Maybe(..), fromMaybe)import Data.Set (Set)import Data.Set as Setimport Data.String as Simport Data.String.Pattern (Pattern(..))import Data.Traversable (traverse)import Effect (Effect)import Effect.Aff (Aff, attempt, bracket, forkAff, joinFiber, throwError, catchError)import Effect.Class (liftEffect)import Effect.Exception (Error, error, throw)import Effect.Ref as Ref
foreign import data Env :: Typeforeign import data Hooks :: Typeforeign import data Controller :: Typeforeign import data Response :: Type
type Config = { cache :: String, publicKey :: String, token :: String, signingKey :: String, nixEnv :: Env }type UrlParts = { protocol :: String, username :: String, password :: String, pathname :: String, search :: String, hash :: String }type PathInfo = { path :: String, signatures :: Array String }type Timings = { lookup :: Number, export :: Number, upload :: Number }type State = { published :: Set String, reused :: Set String, known :: Set String, outcome :: Maybe (Either Error String), timings :: Timings }
foreign import envValue :: Env -> String -> Stringforeign import envForNix :: Env -> String -> Envforeign import matches :: String -> String -> Booleanforeign import urlParts :: String -> Effect UrlPartsforeign import decodePart :: String -> Effect Stringforeign import deriveKey :: String -> Effect { publicKey :: String, embeddedMatches :: Boolean }foreign import log :: String -> Effect Unitforeign import clock :: Effect Numberforeign import temporaryRoot :: Stringforeign import pathJoin :: Array String -> Stringforeign import fileUrl :: String -> Array { key :: String, value :: String } -> Stringforeign import tempDir :: String -> Effect (Promise String)foreign import makeDir :: String -> Effect (Promise Unit)foreign import removeDir :: String -> Effect (Promise Unit)foreign import writeText :: String -> String -> Boolean -> Effect (Promise Unit)foreign import readText :: String -> Effect (Promise String)foreign import listDir :: String -> Effect (Promise (Array String))foreign import fileSize :: String -> Effect (Promise Number)foreign import controller :: Effect Controllerforeign import noController :: Controllerforeign import abort :: Controller -> Effect Unitforeign import errorName :: Error -> Stringforeign import errorCode :: Error -> Stringforeign import wait :: Hooks -> Number -> Effect (Promise Unit)foreign import run :: Hooks -> String -> Array String -> Env -> Boolean -> Controller -> Effect (Promise String)foreign import hasLookup :: Hooks -> Booleanforeign import invokeLookup :: Hooks -> String -> Array String -> String -> Effect (Promise (Array String))foreign import invokeUpload :: Hooks -> String -> String -> String -> Array String -> Effect (Promise Unit)foreign import fetchGet :: Hooks -> String -> Number -> Effect (Promise Response)foreign import responseStatus :: Response -> Intforeign import responseHeader :: Response -> String -> Stringforeign import responseText :: Response -> Number -> Effect (Promise String)foreign import cancelBody :: Response -> Effect (Promise Unit)foreign import putStream :: Hooks -> String -> String -> String -> Number -> Effect (Promise { status :: Int, retryAfter :: String })foreign import headerDelay :: String -> Number -> Numberforeign import pathInfos :: String -> Effect (Array PathInfo)foreign import buildRoots :: String -> Effect (Array String)
ensure :: Boolean -> String -> Effect Unitensure valid reason = unless valid (throw reason)
ensureA :: Boolean -> String -> Aff UnitensureA valid reason = liftEffect (ensure valid reason)
trimSlash :: String -> StringtrimSlash value = fromMaybe value (S.stripSuffix (Pattern "/") value)
part :: Int -> Array String -> Stringpart index values = fromMaybe "" (A.index values index)
configuration :: Env -> Effect Configconfiguration env = do let base = envValue env "FLUENTCI_CACHE_URL" let publicKey = envValue env "FLUENTCI_NIX_PUBLIC_KEY" let token = envValue env "FLUENTCI_CACHE_TOKEN" let signingKey = envValue env "FLUENTCI_NIX_SIGNING_KEY" ensure ((base == "") == (publicKey == "")) "Set both FLUENTCI_CACHE_URL and FLUENTCI_NIX_PUBLIC_KEY" ensure ((token == "") == (signingKey == "")) "Signed uploads require both FLUENTCI_CACHE_TOKEN and FLUENTCI_NIX_SIGNING_KEY" ensure (token == "" || base /= "") "Upload credentials require cache URL and public key" when (base /= "") do url <- urlParts base let segments = A.drop 1 (S.split (Pattern "/") (trimSlash url.pathname)) actor <- decodePart (part 0 segments) let validAccount = (A.length segments == 2 && part 0 segments == "accounts" && matches "^[a-f0-9]{64}$" (part 1 segments)) || (A.length segments == 1 && (matches "^did:plc:[a-z2-7]{24}$" actor || matches "^did:web:[A-Za-z0-9._:%-]+$" actor || matches "^@?(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$" actor)) ensure (url.protocol == "https:" && url.username == "" && url.password == "" && url.search == "" && url.hash == "" && validAccount) "FLUENTCI_CACHE_URL must be an HTTPS account URL without /nix" ensure (matches "^[\\w.-]+:[A-Za-z0-9+/]{43}=$" publicKey) "FLUENTCI_NIX_PUBLIC_KEY must contain the full Nix public key" when (signingKey /= "") do ensure (matches "^[\\w.-]+:[A-Za-z0-9+/]{86}==$" signingKey) "FLUENTCI_NIX_SIGNING_KEY must contain a Nix private key, not a path" let pieces = S.split (Pattern ":") signingKey derived <- deriveKey (part 1 pieces) ensure (part 0 pieces <> ":" <> derived.publicKey == publicKey && derived.embeddedMatches) "Nix signing key does not match FLUENTCI_NIX_PUBLIC_KEY" ensure (not (matches "[\\r\\n]" token)) "Invalid upload token" let cache = if base == "" then "" else trimSlash base <> "/nix" let nixConfig = if cache == "" then "" else envValue env "NIX_CONFIG" <> "\nhttp-connections = 16\nsubstitute = true\nnarinfo-cache-negative-ttl = 0\nalways-allow-substitutes = true\nextra-substituters = " <> cache <> "\nextra-trusted-public-keys = " <> publicKey pure { cache, publicKey, token, signingKey, nixEnv: envForNix env nixConfig }
chunks :: forall a. Int -> Array a -> Array (Array a)chunks size values = if A.null values then [] else [ A.take size values ] <> chunks size (A.drop size values)
unique :: Array String -> Array Stringunique = Set.toUnfoldable <<< Set.fromFoldable
parallelAll :: forall a. Array (Aff a) -> Aff (Array a)parallelAll actions = do fibers <- traverse (forkAff <<< attempt) actions results <- traverse joinFiber fibers case A.findMap (either Just (const Nothing)) results of Just failure -> throwError failure Nothing -> pure (A.mapMaybe (either (const Nothing) Just) results)
basename :: String -> Stringbasename value = fromMaybe value (A.last (S.split (Pattern "/") value))
narinfoName :: String -> StringnarinfoName path = part 0 (S.split (Pattern "-") (basename path)) <> ".narinfo"
field :: String -> String -> Stringfield key text = S.trim $ fromMaybe "" $ A.findMap (S.stripPrefix (Pattern (key <> ": "))) (S.split (Pattern "\n") text)
success :: Int -> Booleansuccess status = status >= 200 && status < 300
retryableStatus :: Int -> BooleanretryableStatus status = A.elem status [ 408, 425, 429, 500, 502, 503, 504 ]
transportError :: Error -> BooleantransportError failure = A.elem (errorName failure) [ "TypeError", "TimeoutError", "AbortError" ] || A.elem (errorCode failure) [ "ECONNRESET", "ETIMEDOUT", "EPIPE" ]
uploadFile :: Hooks -> String -> String -> String -> Int -> Number -> Aff UnituploadFile hooks path url token maxAttempts timeoutMs = loop 1 1000.0 where loop index backoff = do result <- attempt (toAffE (putStream hooks path url token timeoutMs)) now <- liftEffect clock let outcome = case result of Right response -> { ok: success response.status, retry: retryableStatus response.status, after: headerDelay response.retryAfter now, reason: "HTTP " <> show response.status } Left failure -> { ok: false, retry: transportError failure, after: 0.0, reason: errorName failure } unless outcome.ok do let reason = "Cache upload failed for " <> basename path <> ": " <> outcome.reason if outcome.retry && index < maxAttempts then do let pause = min 30000.0 (max backoff outcome.after) liftEffect (log (reason <> "; retrying (" <> show index <> "/" <> show (maxAttempts - 1) <> ")")) toAffE (wait hooks pause) loop (index + 1) (backoff * 2.0) else throwError (error reason)
uploadFiles :: Hooks -> String -> String -> String -> Array String -> Int -> Number -> Aff UnituploadFiles hooks directory cache token metadata maxAttempts timeoutMs = do payloads <- map (map ("nar/" <> _)) (toAffE (listDir (pathJoin [ directory, "nar" ]))) allFiles <- toAffE (listDir directory) let infos = if A.null metadata then A.filter (matches "\\.narinfo$") allFiles else metadata for_ [ payloads, infos ] \files -> for_ (chunks 16 files) \group -> do void $ parallelAll $ map (\file -> uploadFile hooks (pathJoin [ directory, file ]) (cache <> "/" <> file) token maxAttempts timeoutMs) group
requireSignatures :: String -> String -> Array String -> Aff UnitrequireSignatures directory publicKey metadata = do allFiles <- if A.null metadata then toAffE (listDir directory) else pure metadata let files = A.filter (matches "\\.narinfo$") allFiles let key = part 0 (S.split (Pattern ":") publicKey) ensureA (not (A.null files)) "Nix export contains no signed metadata" for_ files \file -> do text <- toAffE (readText (pathJoin [ directory, file ])) let signatures = A.mapMaybe (S.stripPrefix (Pattern ("Sig: " <> key <> ":"))) (S.split (Pattern "\n") text) ensureA (A.any (matches "^[A-Za-z0-9+/]{86}==$" <<< S.trim) signatures) ("Refusing unsigned Nix metadata: " <> file)
lookupResponse :: Hooks -> String -> Aff ResponselookupResponse hooks url = loop 0 where loop index = do result <- attempt (toAffE (fetchGet hooks url 15000.0)) case result of Right response | retryableStatus (responseStatus response) && index < 2 -> do toAffE (cancelBody response) toAffE (wait hooks (250.0 * Int.toNumber (index + 1))) loop (index + 1) Right response -> pure response Left _ | index < 2 -> do toAffE (wait hooks 250.0) loop (index + 1) Left _ -> throwError (error "Cache availability lookup failed")
seedAvailablePaths :: Hooks -> String -> Array String -> String -> Aff (Array String)seedAvailablePaths hooks directory paths cache = do groups <- traverse (parallelAll <<< map lookupPath) (chunks 24 (unique paths)) pure (A.catMaybes (A.concat groups)) where lookupPath path = do let name = narinfoName path ensureA (matches "^[0-9a-z]{32}\\.narinfo$" name) "Invalid Nix store path" findRemote path name [ "https://cache.nixos.org", cache ] findRemote path name bases = case A.uncons bases of Nothing -> pure Nothing Just { head: base, tail: rest } -> do response <- lookupResponse hooks (base <> "/" <> name) if responseStatus response == 404 then do toAffE (cancelBody response) findRemote path name rest else do ensureA (success (responseStatus response)) ("Cache availability lookup failed: HTTP " <> show (responseStatus response)) text <- toAffE (responseText response 4194304.0) ensureA (S.length text <= 4194304 && field "StorePath" text == path) "Cache returned invalid store metadata" toAffE (writeText (pathJoin [ directory, name ]) text false) pure (Just path)
seconds :: Number -> Stringseconds value = show (Int.round (value / 1000.0))
storePaths :: String -> Array StringstorePaths text = A.filter (_ /= "") (S.split (Pattern "\n") (S.trim text))
build :: Hooks -> Env -> String -> Number -> Aff Unitbuild hooks env target pollIntervalMs = do ensureA (S.trim target /= "" && not (matches "[\\r\\n\\x00]" target)) "Set a valid Nix installable" ensureA (envValue env "FLUENTCI_CACHE_TOKEN" /= "") "Set FLUENTCI_CACHE_TOKEN" ensureA (envValue env "FLUENTCI_CACHE_URL" /= "") "Set FLUENTCI_CACHE_URL" config <- liftEffect (configuration env) liftEffect (log (if config.cache == "" then "No FluentCI cache configured; using default Nix substituters" else "Nix cache: " <> config.cache)) let execute program args capture = toAffE (run hooks program args config.nixEnv capture noController) let buildArgs = [ "build", "--no-write-lock-file", "--json", "--", target ] if config.token == "" then do void (execute "nix" buildArgs true) liftEffect (log "No upload credentials; cache is read-only") else do let root = envValue env "RUNNER_TEMP" bracket (toAffE (tempDir (if root == "" then temporaryRoot else root))) (toAffE <<< removeDir) \directory -> do let keyPath = pathJoin [ directory, "signing.key" ] let exportDir = pathJoin [ directory, "cache" ] toAffE (writeText keyPath (config.signingKey <> "\n") true) toAffE (makeDir exportDir) state <- liftEffect $ Ref.new { published: Set.empty, reused: Set.empty, known: Set.empty, outcome: Nothing, timings: { lookup: 0.0, export: 0.0, upload: 0.0 } } let verify trusted paths = for_ (chunks 100 paths) \batch -> void $ execute "nix" ([ "store", "verify", "--store", fileUrl exportDir [], "--no-contents", "--sigs-needed", "1" ] <> trusted <> batch) false let publish candidates = for_ (chunks 100 (unique (A.filter (not <<< matches "\\.drv$") candidates))) \batch -> do current <- liftEffect (Ref.read state) let fresh = A.filter (\path -> not (Set.member path current.published || Set.member path current.reused)) batch unless (A.null fresh) do infos <- execute "nix" ([ "path-info", "--json" ] <> fresh) true >>= liftEffect <<< pathInfos let paths = map _.path (A.filter (A.null <<< _.signatures) infos) unless (A.null paths) do closure <- map storePaths (execute "nix-store" ([ "--query", "--requisites" ] <> paths) true) lookupStart <- liftEffect clock present <- if hasLookup hooks then toAffE (invokeLookup hooks exportDir closure config.cache) else seedAvailablePaths hooks exportDir (A.filter (\path -> not (Set.member (narinfoName path) current.known) && not (matches "\\.drv$" path)) closure) config.cache unless (A.null present) do verify [] present liftEffect $ Ref.modify_ (\s -> s { reused = Set.union s.reused (Set.fromFoldable present), known = Set.union s.known (Set.fromFoldable (map narinfoName present)) }) state liftEffect (log ("Reused " <> show (A.length present) <> " paths already in an upstream or FluentCI cache")) lookupEnd <- liftEffect clock liftEffect $ Ref.modify_ (\s -> s { timings = s.timings { lookup = s.timings.lookup + lookupEnd - lookupStart } }) state updated <- liftEffect (Ref.read state) let pending = A.filter (\path -> not (Set.member path updated.reused)) paths unless (A.null pending) do exportStart <- liftEffect clock let destination = fileUrl exportDir [ { key: "secret-key", value: keyPath }, { key: "compression", value: "zstd" }, { key: "compression-level", value: "3" }, { key: "parallel-compression", value: "false" } ] void $ execute "nix" ([ "copy", "--to", destination ] <> pending) false files <- toAffE (listDir exportDir) let metadata = A.filter (\file -> matches "\\.narinfo$" file && not (Set.member file updated.known)) files unless (A.null metadata) do texts <- traverse (toAffE <<< readText <<< pathJoin <<< (\file -> [ exportDir, file ])) metadata let exported = map (field "StorePath") texts ensureA (A.all (_ /= "") exported) "Invalid exported Nix metadata" requireSignatures exportDir config.publicKey metadata verify [ "--trusted-public-keys", config.publicKey ] exported uploadStart <- liftEffect clock toAffE (invokeUpload hooks exportDir config.cache config.token metadata) uploadEnd <- liftEffect clock liftEffect $ Ref.modify_ (\s -> s { published = Set.union s.published (Set.fromFoldable exported), known = Set.union s.known (Set.fromFoldable metadata), timings = s.timings { export = s.timings.export + uploadStart - exportStart, upload = s.timings.upload + uploadEnd - uploadStart } }) state toAffE (removeDir (pathJoin [ exportDir, "nar" ])) toAffE (makeDir (pathJoin [ exportDir, "nar" ])) liftEffect (log ("Published " <> show (A.length exported) <> " signed paths")) seen <- execute "nix" [ "path-info", "--all" ] true >>= liftEffect <<< Ref.new <<< Set.fromFoldable <<< storePaths control <- liftEffect controller started <- liftEffect clock fiber <- forkAff do outcome <- attempt (toAffE (run hooks "nix" buildArgs config.nixEnv true control)) liftEffect $ Ref.modify_ (_ { outcome = Just outcome }) state now <- liftEffect clock liftEffect (log ("Nix build finished after " <> seconds (now - started) <> "s; draining cache uploads")) let scan = do current <- map storePaths (execute "nix" [ "path-info", "--all" ] true) before <- liftEffect (Ref.read seen) liftEffect (Ref.write (Set.union before (Set.fromFoldable current)) seen) publish (A.filter (\path -> not (Set.member path before)) current) let watch = do scan current <- liftEffect (Ref.read state) case current.outcome of Nothing -> toAffE (wait hooks pollIntervalMs) *> watch Just _ -> pure unit liftEffect (log "Watching Nix store; publishing new paths during the build") let finish = do watch joinFiber fiber scan current <- liftEffect (Ref.read state) case current.outcome of Just (Left failure) -> throwError failure Just (Right output) -> do roots <- liftEffect (buildRoots output) unless (A.null roots) do closure <- map storePaths (execute "nix-store" ([ "--query", "--requisites", "--include-outputs" ] <> roots) true) publish closure Nothing -> throwError (error "Missing Nix build result") summary <- liftEffect (Ref.read state) liftEffect $ log ("Nix cache uploads complete: " <> show (Set.size summary.published) <> " published, " <> show (Set.size summary.reused) <> " reused; lookup " <> seconds summary.timings.lookup <> "s, export/verify " <> seconds summary.timings.export <> "s, upload " <> seconds summary.timings.upload <> "s") catchError finish \failure -> do liftEffect (abort control) joinFiber fiber throwError failure