From a73e3189f9b6049b5d0296eda182dbe0c2ec965d Mon Sep 17 00:00:00 2001 From: Winter Date: Thu, 7 Aug 2025 17:41:13 -0400 Subject: [PATCH] gitignore: add .envrc for the direnv users --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 831abae..2e9ab2f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ patches .DS_Store .env *.rdb +.envrc -- 2.43.0 From 3c8332c6ac085a92f9372e930c7dfdbe514365ce Mon Sep 17 00:00:00 2001 From: Winter Date: Thu, 7 Aug 2025 16:25:18 -0400 Subject: [PATCH] nix: use filesets instead of gitignore.nix This allows us to easily do things like ignoring Nix source files within our source tree (also done in this commit), which prevents unnecessary rebuilds. Signed-off-by: Winter --- flake.lock | 21 --------------------- flake.nix | 13 +++++++------ nix/pkgs/appview.nix | 5 ++--- nix/pkgs/genjwks.nix | 5 ++--- nix/pkgs/knot-unwrapped.nix | 5 ++--- nix/pkgs/spindle.nix | 5 ++--- 6 files changed, 15 insertions(+), 39 deletions(-) diff --git a/flake.lock b/flake.lock index db4dc57..e7ed83e 100644 --- a/flake.lock +++ b/flake.lock @@ -18,26 +18,6 @@ "type": "github" } }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, "gomod2nix": { "inputs": { "flake-utils": "flake-utils", @@ -156,7 +136,6 @@ }, "root": { "inputs": { - "gitignore": "gitignore", "gomod2nix": "gomod2nix", "htmx-src": "htmx-src", "htmx-ws-src": "htmx-ws-src", diff --git a/flake.nix b/flake.nix index d797e37..41522b5 100644 --- a/flake.nix +++ b/flake.nix @@ -37,10 +37,6 @@ url = "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip"; flake = false; }; - gitignore = { - url = "github:hercules-ci/gitignore.nix"; - inputs.nixpkgs.follows = "nixpkgs"; - }; }; outputs = { @@ -51,7 +47,6 @@ htmx-src, htmx-ws-src, lucide-src, - gitignore, inter-fonts-src, sqlite-lib-src, ibm-plex-mono-src, @@ -62,7 +57,13 @@ mkPackageSet = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { - inherit (gitignore.lib) gitignoreSource; + src = let + fs = pkgs.lib.fileset; + in + fs.toSource { + root = ./.; + fileset = fs.difference (fs.intersection (fs.gitTracked ./.) (fs.fileFilter (file: !(file.hasExt "nix")) ./.)) (fs.maybeMissing ./.jj); + }; buildGoApplication = (self.callPackage "${gomod2nix}/builder" { gomod2nix = gomod2nix.legacyPackages.${pkgs.system}.gomod2nix; diff --git a/nix/pkgs/appview.nix b/nix/pkgs/appview.nix index 2b409e9..2d438c5 100644 --- a/nix/pkgs/appview.nix +++ b/nix/pkgs/appview.nix @@ -8,13 +8,12 @@ ibm-plex-mono-src, tailwindcss, sqlite-lib, - gitignoreSource, + src, }: buildGoApplication { pname = "appview"; version = "0.1.0"; - src = gitignoreSource ../..; - inherit modules; + inherit src modules; postUnpack = '' pushd source diff --git a/nix/pkgs/genjwks.nix b/nix/pkgs/genjwks.nix index af41ee3..4b84a18 100644 --- a/nix/pkgs/genjwks.nix +++ b/nix/pkgs/genjwks.nix @@ -1,13 +1,12 @@ { - gitignoreSource, + src, buildGoApplication, modules, }: buildGoApplication { pname = "genjwks"; version = "0.1.0"; - src = gitignoreSource ../..; - inherit modules; + inherit src modules; subPackages = ["cmd/genjwks"]; doCheck = false; CGO_ENABLED = 0; diff --git a/nix/pkgs/knot-unwrapped.nix b/nix/pkgs/knot-unwrapped.nix index 3df5df5..21b31f4 100644 --- a/nix/pkgs/knot-unwrapped.nix +++ b/nix/pkgs/knot-unwrapped.nix @@ -2,13 +2,12 @@ buildGoApplication, modules, sqlite-lib, - gitignoreSource, + src, }: buildGoApplication { pname = "knot"; version = "0.1.0"; - src = gitignoreSource ../..; - inherit modules; + inherit src modules; doCheck = false; diff --git a/nix/pkgs/spindle.nix b/nix/pkgs/spindle.nix index 7f2cd0b..6a9ffb1 100644 --- a/nix/pkgs/spindle.nix +++ b/nix/pkgs/spindle.nix @@ -2,13 +2,12 @@ buildGoApplication, modules, sqlite-lib, - gitignoreSource, + src, }: buildGoApplication { pname = "spindle"; version = "0.1.0"; - src = gitignoreSource ../..; - inherit modules; + inherit src modules; doCheck = false; -- 2.43.0 From 16255f924a3fdeccd2e2ef431c03d24eae2a2d6f Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Fri, 8 Aug 2025 09:39:50 +0100 Subject: [PATCH] nix: bump gomod2nix.toml Change-Id: vtzorzlsruztnuypmkkqtvskynypyono Signed-off-by: oppiliappan --- nix/gomod2nix.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index dae0315..179679d 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -66,6 +66,9 @@ schema = 3 [mod."github.com/cloudflare/circl"] version = "v1.6.2-0.20250618153321-aa837fd1539d" hash = "sha256-0s/i/XmMcuvPQ+qK9OIU5KxwYZyLVXRtdlYvIXRJT3Y=" + [mod."github.com/cloudflare/cloudflare-go"] + version = "v0.115.0" + hash = "sha256-jezmDs6IsHA4rag7DzcHDfDgde0vU4iKgCN9+0XDViw=" [mod."github.com/containerd/errdefs"] version = "v1.0.0" hash = "sha256-wMZGoeqvRhuovYCJx0Js4P3qFCNTZ/6Atea/kNYoPMI=" @@ -169,6 +172,9 @@ schema = 3 [mod."github.com/golang/mock"] version = "v1.6.0" hash = "sha256-fWdnMQisRbiRzGT3ISrUHovquzLRHWvcv1JEsJFZRno=" + [mod."github.com/google/go-querystring"] + version = "v1.1.0" + hash = "sha256-itsKgKghuX26czU79cK6C2n+lc27jm5Dw1XbIRgwZJY=" [mod."github.com/google/uuid"] version = "v1.6.0" hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=" -- 2.43.0 From 48467689090388d3c1b0648c9079dcaec3671e6e Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Fri, 8 Aug 2025 11:46:24 +0100 Subject: [PATCH] nix: add secrets config to spindle module Change-Id: vpsnurolyttsznvmrxnkyosqtkrtntkv Signed-off-by: oppiliappan --- nix/modules/spindle.nix | 22 ++++++++++++++++++++++ nix/vm.nix | 3 +++ 2 files changed, 25 insertions(+) diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix index 60081bd..dbde1eb 100644 --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -54,6 +54,25 @@ in example = "did:plc:qfpnj4og54vl56wngdriaxug"; description = "DID of owner (required)"; }; + + secrets = { + provider = mkOption { + type = types.str; + default = "sqlite"; + description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'."; + }; + + openbao = { + proxyAddr = mkOption { + type = types.str; + default = "http://127.0.0.1:8200"; + }; + mount = mkOption { + type = types.str; + default = "spindle"; + }; + }; + }; }; pipelines = { @@ -89,6 +108,9 @@ in "SPINDLE_SERVER_JETSTREAM=${cfg.server.jetstreamEndpoint}" "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" "SPINDLE_SERVER_OWNER=${cfg.server.owner}" + "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" + "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" + "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" "SPINDLE_PIPELINES_NIXERY=${cfg.pipelines.nixery}" "SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" ]; diff --git a/nix/vm.nix b/nix/vm.nix index 2f69fa9..41ff743 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -62,6 +62,9 @@ nixpkgs.lib.nixosSystem { hostname = "localhost:6555"; listenAddr = "0.0.0.0:6555"; dev = true; + secrets = { + provider = "sqlite"; + }; }; }; }) -- 2.43.0 From 4e03f2978a3185589848093270cb3b54fb6d53f0 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Fri, 8 Aug 2025 15:07:46 +0300 Subject: [PATCH] spindle: set worker count to 5 Change-Id: rvvxxuuqzppkwolurlmorxqmxmwqlowt Signed-off-by: Anirudh Oppiliappan --- spindle/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spindle/server.go b/spindle/server.go index ed81289..a04924c 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -98,7 +98,7 @@ func Run(ctx context.Context) error { return err } - jq := queue.NewQueue(100, 2) + jq := queue.NewQueue(100, 5) collections := []string{ tangled.SpindleMemberNSID, -- 2.43.0 From 23776d5eca3824f2410a9c9a4fcf64a07f0fa65d Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Fri, 8 Aug 2025 17:08:42 +0100 Subject: [PATCH] spindle: improve member ingestion Change-Id: vokloxluvoxxwoquqlmkvxpmzsswtvok Signed-off-by: oppiliappan --- appview/ingester.go | 4 +++ appview/spindles/spindles.go | 8 ++--- spindle/db/db.go | 15 +++++++++ spindle/db/member.go | 59 ++++++++++++++++++++++++++++++++++++ spindle/ingester.go | 43 +++++++++++++++++++++++--- 5 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 spindle/db/member.go diff --git a/appview/ingester.go b/appview/ingester.go index adc2179..b8585fd 100644 --- a/appview/ingester.go +++ b/appview/ingester.go @@ -387,6 +387,8 @@ func (i *Ingester) ingestSpindleMember(e *models.Event) error { if err != nil { return fmt.Errorf("failed to update ACLs: %w", err) } + + l.Info("added spindle member") case models.CommitOperationDelete: rkey := e.Commit.RKey @@ -433,6 +435,8 @@ func (i *Ingester) ingestSpindleMember(e *models.Event) error { if err = i.Enforcer.E.SavePolicy(); err != nil { return fmt.Errorf("failed to save ACLs: %w", err) } + + l.Info("removed spindle member") } return nil diff --git a/appview/spindles/spindles.go b/appview/spindles/spindles.go index f903fa2..f697865 100644 --- a/appview/spindles/spindles.go +++ b/appview/spindles/spindles.go @@ -619,14 +619,14 @@ func (s *Spindles) removeMember(w http.ResponseWriter, r *http.Request) { if string(spindles[0].Owner) != user.Did { l.Error("unauthorized", "user", user.Did, "owner", spindles[0].Owner) - s.Pages.Notice(w, noticeId, "Failed to add member, unauthorized attempt.") + s.Pages.Notice(w, noticeId, "Failed to remove member, unauthorized attempt.") return } member := r.FormValue("member") if member == "" { l.Error("empty member") - s.Pages.Notice(w, noticeId, "Failed to add member, empty form.") + s.Pages.Notice(w, noticeId, "Failed to remove member, empty form.") return } l = l.With("member", member) @@ -634,12 +634,12 @@ func (s *Spindles) removeMember(w http.ResponseWriter, r *http.Request) { memberId, err := s.IdResolver.ResolveIdent(r.Context(), member) if err != nil { l.Error("failed to resolve member identity to handle", "err", err) - s.Pages.Notice(w, noticeId, "Failed to add member, identity resolution failed.") + s.Pages.Notice(w, noticeId, "Failed to remove member, identity resolution failed.") return } if memberId.Handle.IsInvalidHandle() { l.Error("failed to resolve member identity to handle") - s.Pages.Notice(w, noticeId, "Failed to add member, identity resolution failed.") + s.Pages.Notice(w, noticeId, "Failed to remove member, identity resolution failed.") return } diff --git a/spindle/db/db.go b/spindle/db/db.go index a81edc8..89be341 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -45,6 +45,21 @@ func Make(dbPath string) (*DB, error) { unique(owner, name) ); + create table if not exists spindle_members ( + -- identifiers for the record + id integer primary key autoincrement, + did text not null, + rkey text not null, + + -- data + instance text not null, + subject text not null, + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + + -- constraints + unique (did, instance, subject) + ); + -- status event for a single workflow create table if not exists events ( rkey text not null, diff --git a/spindle/db/member.go b/spindle/db/member.go new file mode 100644 index 0000000..a93b7f9 --- /dev/null +++ b/spindle/db/member.go @@ -0,0 +1,59 @@ +package db + +import ( + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type SpindleMember struct { + Id int + Did syntax.DID // owner of the record + Rkey string // rkey of the record + Instance string + Subject syntax.DID // the member being added + Created time.Time +} + +func AddSpindleMember(db *DB, member SpindleMember) error { + _, err := db.Exec( + `insert or ignore into spindle_members (did, rkey, instance, subject) values (?, ?, ?, ?)`, + member.Did, + member.Rkey, + member.Instance, + member.Subject, + ) + return err +} + +func RemoveSpindleMember(db *DB, owner_did, rkey string) error { + _, err := db.Exec( + "delete from spindle_members where did = ? and rkey = ?", + owner_did, + rkey, + ) + return err +} + +func GetSpindleMember(db *DB, did, rkey string) (*SpindleMember, error) { + query := + `select id, did, rkey, instance, subject, created + from spindle_members + where did = ? and rkey = ?` + + var member SpindleMember + var createdAt string + err := db.QueryRow(query, did, rkey).Scan( + &member.Id, + &member.Did, + &member.Rkey, + &member.Instance, + &member.Subject, + &createdAt, + ) + if err != nil { + return nil, err + } + + return &member, nil +} diff --git a/spindle/ingester.go b/spindle/ingester.go index a47fa97..cf45fd6 100644 --- a/spindle/ingester.go +++ b/spindle/ingester.go @@ -5,11 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "time" "tangled.sh/tangled.sh/core/api/tangled" "tangled.sh/tangled.sh/core/eventconsumer" "tangled.sh/tangled.sh/core/idresolver" "tangled.sh/tangled.sh/core/rbac" + "tangled.sh/tangled.sh/core/spindle/db" comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/identity" @@ -50,8 +52,9 @@ func (s *Spindle) ingest() Ingester { } func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error { - did := e.Did var err error + did := e.Did + rkey := e.Commit.RKey l := s.l.With("component", "ingester", "record", tangled.SpindleMemberNSID) @@ -66,9 +69,6 @@ func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error { } domain := s.cfg.Server.Hostname - if s.cfg.Server.Dev { - domain = s.cfg.Server.ListenAddr - } recordInstance := record.Instance if recordInstance != domain { @@ -82,6 +82,17 @@ func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error { return fmt.Errorf("failed to enforce permissions: %w", err) } + if err := db.AddSpindleMember(s.db, db.SpindleMember{ + Did: syntax.DID(did), + Rkey: rkey, + Instance: recordInstance, + Subject: syntax.DID(record.Subject), + Created: time.Now(), + }); err != nil { + l.Error("failed to add member", "error", err) + return fmt.Errorf("failed to add member: %w", err) + } + if err := s.e.AddSpindleMember(rbacDomain, record.Subject); err != nil { l.Error("failed to add member", "error", err) return fmt.Errorf("failed to add member: %w", err) @@ -96,6 +107,30 @@ func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error { return nil + case models.CommitOperationDelete: + record, err := db.GetSpindleMember(s.db, did, rkey) + if err != nil { + l.Error("failed to find member", "error", err) + return fmt.Errorf("failed to find member: %w", err) + } + + if err := db.RemoveSpindleMember(s.db, did, rkey); err != nil { + l.Error("failed to remove member", "error", err) + return fmt.Errorf("failed to remove member: %w", err) + } + + if err := s.e.RemoveSpindleMember(rbacDomain, record.Subject.String()); err != nil { + l.Error("failed to add member", "error", err) + return fmt.Errorf("failed to add member: %w", err) + } + l.Info("added member from firehose", "member", record.Subject) + + if err := s.db.RemoveDid(record.Subject.String()); err != nil { + l.Error("failed to add did", "error", err) + return fmt.Errorf("failed to add did: %w", err) + } + s.jc.RemoveDid(record.Subject.String()) + } return nil } -- 2.43.0 From 15f63d03c58450cd1fa9b80e63194f828031aa0c Mon Sep 17 00:00:00 2001 From: Winter Date: Thu, 7 Aug 2025 20:11:28 -0400 Subject: [PATCH] nix: update lexgen package Now we can generate the same files that have been committed recently. ;) --- flake.lock | 6 +++--- nix/pkgs/lexgen.nix | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index e7ed83e..d855156 100644 --- a/flake.lock +++ b/flake.lock @@ -79,11 +79,11 @@ "indigo": { "flake": false, "locked": { - "lastModified": 1745333930, - "narHash": "sha256-83fIHqDE+dfnZ88HaNuwfKFO+R0RKAM1WxMfNh/Matk=", + "lastModified": 1753693716, + "narHash": "sha256-DMIKnCJRODQXEHUxA+7mLzRALmnZhkkbHlFT2rCQYrE=", "owner": "oppiliappan", "repo": "indigo", - "rev": "e4e59280737b8676611fc077a228d47b3e8e9491", + "rev": "5f170569da9360f57add450a278d73538092d8ca", "type": "github" }, "original": { diff --git a/nix/pkgs/lexgen.nix b/nix/pkgs/lexgen.nix index f0fef68..aca542e 100644 --- a/nix/pkgs/lexgen.nix +++ b/nix/pkgs/lexgen.nix @@ -7,6 +7,6 @@ buildGoModule { version = "0.1.0"; src = indigo; subPackages = ["cmd/lexgen"]; - vendorHash = "sha256-pGc29fgJFq8LP7n/pY1cv6ExZl88PAeFqIbFEhB3xXs="; + vendorHash = "sha256-VbDrcN4r5b7utRFQzVsKgDsVgdQLSXl7oZ5kdPA/huw="; doCheck = false; } -- 2.43.0 From 3fb08dd7c126113134133e793ac21df5bba18a77 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Fri, 8 Aug 2025 11:38:39 +0300 Subject: [PATCH] appview/oauth: add to default spindle Signed-off-by: Anirudh Oppiliappan --- appview/config/config.go | 3 + appview/oauth/handler/handler.go | 141 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/appview/config/config.go b/appview/config/config.go index 3d50ba8..213e343 100644 --- a/appview/config/config.go +++ b/appview/config/config.go @@ -16,6 +16,9 @@ type CoreConfig struct { AppviewHost string `env:"APPVIEW_HOST, default=https://tangled.sh"` Dev bool `env:"DEV, default=false"` DisallowedNicknamesFile string `env:"DISALLOWED_NICKNAMES_FILE"` + + // temporarily, to add users to default spindle + AppPassword string `env:"APP_PASSWORD"` } type OAuthConfig struct { diff --git a/appview/oauth/handler/handler.go b/appview/oauth/handler/handler.go index d9d23a7..c4a6c41 100644 --- a/appview/oauth/handler/handler.go +++ b/appview/oauth/handler/handler.go @@ -1,18 +1,22 @@ package oauth import ( + "bytes" + "context" "encoding/json" "fmt" "log" "net/http" "net/url" "strings" + "time" "github.com/go-chi/chi/v5" "github.com/gorilla/sessions" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/posthog/posthog-go" "tangled.sh/icyphox.sh/atproto-oauth/helpers" + tangled "tangled.sh/tangled.sh/core/api/tangled" sessioncache "tangled.sh/tangled.sh/core/appview/cache/session" "tangled.sh/tangled.sh/core/appview/config" "tangled.sh/tangled.sh/core/appview/db" @@ -23,6 +27,7 @@ import ( "tangled.sh/tangled.sh/core/idresolver" "tangled.sh/tangled.sh/core/knotclient" "tangled.sh/tangled.sh/core/rbac" + "tangled.sh/tangled.sh/core/tid" ) const ( @@ -294,6 +299,7 @@ func (o *OAuthHandler) callback(w http.ResponseWriter, r *http.Request) { log.Println("session saved successfully") go o.addToDefaultKnot(oauthRequest.Did) + go o.addToDefaultSpindle(oauthRequest.Did) if !o.config.Core.Dev { err = o.posthog.Enqueue(posthog.Capture{ @@ -332,6 +338,141 @@ func pubKeyFromJwk(jwks string) (jwk.Key, error) { return pubKey, nil } +func (o *OAuthHandler) addToDefaultSpindle(did string) { + // use the tangled.sh app password to get an accessJwt + // and create an sh.tangled.spindle.member record with that + + defaultSpindle := "spindle.tangled.sh" + appPassword := o.config.Core.AppPassword + + spindleMembers, err := db.GetSpindleMembers( + o.db, + db.FilterEq("instance", "spindle.tangled.sh"), + db.FilterEq("subject", did), + ) + if err != nil { + log.Printf("failed to get spindle members for did %s: %v", did, err) + return + } + + if len(spindleMembers) != 0 { + log.Printf("did %s is already a member of the default spindle", did) + return + } + + // TODO: hardcoded tangled handle and did for now + tangledHandle := "tangled.sh" + tangledDid := "did:plc:wshs7t2adsemcrrd4snkeqli" + + if appPassword == "" { + log.Println("no app password configured, skipping spindle member addition") + return + } + + log.Printf("adding %s to default spindle", did) + + resolved, err := o.idResolver.ResolveIdent(context.Background(), tangledDid) + if err != nil { + log.Printf("failed to resolve tangled.sh DID %s: %v", tangledDid, err) + return + } + + pdsEndpoint := resolved.PDSEndpoint() + if pdsEndpoint == "" { + log.Printf("no PDS endpoint found for tangled.sh DID %s", tangledDid) + return + } + + sessionPayload := map[string]string{ + "identifier": tangledHandle, + "password": appPassword, + } + sessionBytes, err := json.Marshal(sessionPayload) + if err != nil { + log.Printf("failed to marshal session payload: %v", err) + return + } + + sessionURL := pdsEndpoint + "/xrpc/com.atproto.server.createSession" + sessionReq, err := http.NewRequestWithContext(context.Background(), "POST", sessionURL, bytes.NewBuffer(sessionBytes)) + if err != nil { + log.Printf("failed to create session request: %v", err) + return + } + sessionReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: 30 * time.Second} + sessionResp, err := client.Do(sessionReq) + if err != nil { + log.Printf("failed to create session: %v", err) + return + } + defer sessionResp.Body.Close() + + if sessionResp.StatusCode != http.StatusOK { + log.Printf("failed to create session: HTTP %d", sessionResp.StatusCode) + return + } + + var session struct { + AccessJwt string `json:"accessJwt"` + } + if err := json.NewDecoder(sessionResp.Body).Decode(&session); err != nil { + log.Printf("failed to decode session response: %v", err) + return + } + + record := tangled.SpindleMember{ + LexiconTypeID: "sh.tangled.spindle.member", + Subject: did, + Instance: defaultSpindle, + CreatedAt: time.Now().Format(time.RFC3339), + } + + recordBytes, err := json.Marshal(record) + if err != nil { + log.Printf("failed to marshal spindle member record: %v", err) + return + } + + payload := map[string]interface{}{ + "repo": tangledDid, + "collection": tangled.SpindleMemberNSID, + "rkey": tid.TID(), + "record": json.RawMessage(recordBytes), + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + log.Printf("failed to marshal request payload: %v", err) + return + } + + url := pdsEndpoint + "/xrpc/com.atproto.repo.putRecord" + req, err := http.NewRequestWithContext(context.Background(), "POST", url, bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Printf("failed to create HTTP request: %v", err) + return + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := client.Do(req) + if err != nil { + log.Printf("failed to add user to default spindle: %v", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("failed to add user to default spindle: HTTP %d", resp.StatusCode) + return + } + + log.Printf("successfully added %s to default spindle", did) +} + func (o *OAuthHandler) addToDefaultKnot(did string) { defaultKnot := "knot1.tangled.sh" -- 2.43.0 From 2bc0d160218c1d4ce0a2c3dff4cd7c7f03320d70 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Fri, 8 Aug 2025 23:17:14 +0300 Subject: [PATCH] appview/signup: wrap the TXT record content in quotes Change-Id: wknmrsnonxrslrvkltmotvzvsntwtmvx Signed-off-by: Anirudh Oppiliappan --- appview/signup/signup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appview/signup/signup.go b/appview/signup/signup.go index fcadd7d..c720818 100644 --- a/appview/signup/signup.go +++ b/appview/signup/signup.go @@ -219,7 +219,7 @@ func (s *Signup) complete(w http.ResponseWriter, r *http.Request) { err = s.cf.CreateDNSRecord(r.Context(), dns.Record{ Type: "TXT", Name: "_atproto." + username, - Content: "did=" + did, + Content: fmt.Sprintf(`"did=%s"`, did), TTL: 6400, Proxied: false, }) -- 2.43.0 From f42b2ad0caf834357ae0cc1c7420f81188f4fb9b Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Sat, 9 Aug 2025 09:10:20 +0100 Subject: [PATCH] spindle: subscribe to all DIDs on boot Change-Id: xsxoltplnowupzqkwrkmmmmpsrorlqrx Signed-off-by: oppiliappan --- spindle/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spindle/server.go b/spindle/server.go index a04924c..59e52fe 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -111,6 +111,15 @@ func Run(ctx context.Context) error { } jc.AddDid(cfg.Server.Owner) + // Check if the spindle knows about any Dids; + dids, err := d.GetAllDids() + if err != nil { + return fmt.Errorf("failed to get all dids: %w", err) + } + for _, d := range dids { + jc.AddDid(d) + } + resolver := idresolver.DefaultResolver() spindle := Spindle{ -- 2.43.0 From 879790a5bbddd8691570b065c670ffa9d3fe70a3 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Sat, 9 Aug 2025 11:42:50 +0300 Subject: [PATCH] spindle/secrets: delete metadata so key doesn't persist Change-Id: mlypwrorkmkqruvlwzrmlswyzpmuxpvn Signed-off-by: Anirudh Oppiliappan --- spindle/secrets/openbao.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spindle/secrets/openbao.go b/spindle/secrets/openbao.go index 4c6cce2..659a246 100644 --- a/spindle/secrets/openbao.go +++ b/spindle/secrets/openbao.go @@ -132,7 +132,7 @@ func (v *OpenBaoManager) RemoveSecret(ctx context.Context, secret Secret[any]) e return ErrKeyNotFound } - err = v.client.KVv2(v.mountPath).Delete(ctx, secretPath) + err = v.client.KVv2(v.mountPath).DeleteMetadata(ctx, secretPath) if err != nil { return fmt.Errorf("failed to delete secret from openbao: %w", err) } -- 2.43.0 From b185e73d55cdf9f6e61547bec271dc0d22feb792 Mon Sep 17 00:00:00 2001 From: Anil Madhavapeddy Date: Sat, 9 Aug 2025 12:28:34 +0100 Subject: [PATCH] docs: fix typo --- docs/knot-hosting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/knot-hosting.md b/docs/knot-hosting.md index e488984..22aed65 100644 --- a/docs/knot-hosting.md +++ b/docs/knot-hosting.md @@ -89,7 +89,7 @@ systemctl enable knotserver systemctl start knotserver ``` -The last step is to configure a reverse proxy like Nginx or Caddy to front yourself +The last step is to configure a reverse proxy like Nginx or Caddy to front your knot. Here's an example configuration for Nginx: ``` -- 2.43.0 From 0e941b22ac08f695b9848a810ab579437def41dc Mon Sep 17 00:00:00 2001 From: dusk Date: Sat, 9 Aug 2025 14:24:50 +0300 Subject: [PATCH] docs: spindle: openbao: add force flag when generating secret id Signed-off-by: dusk --- docs/spindle/openbao.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spindle/openbao.md b/docs/spindle/openbao.md index c5b3e20..88c7aca 100644 --- a/docs/spindle/openbao.md +++ b/docs/spindle/openbao.md @@ -114,7 +114,7 @@ Get the credentials: ROLE_ID=$(bao read -field=role_id auth/approle/role/spindle/role-id) # Generate secret ID -SECRET_ID=$(bao write -field=secret_id auth/approle/role/spindle/secret-id) +SECRET_ID=$(bao write -f -field=secret_id auth/approle/role/spindle/secret-id) echo "Role ID: $ROLE_ID" echo "Secret ID: $SECRET_ID" -- 2.43.0 From 3d6e9ffc3a8d7aa7344380b629985f93783c5ccb Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 19:23:20 -0400 Subject: [PATCH] nix/vm: add aarch64 support Signed-off-by: Winter --- flake.nix | 18 +++++++++++++++--- nix/vm.nix | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 41522b5..92bdc6b 100644 --- a/flake.nix +++ b/flake.nix @@ -177,10 +177,15 @@ type = "app"; program = ''${tailwind-watcher}/bin/run''; }; - vm = { + vm = let + system = + if pkgs.stdenv.hostPlatform.isAarch64 + then "aarch64" + else "x86_64"; + in { type = "app"; program = toString (pkgs.writeShellScript "vm" '' - ${pkgs.nixos-shell}/bin/nixos-shell --flake .#vm + ${pkgs.nixos-shell}/bin/nixos-shell --flake .#vm-${system} ''); }; gomod2nix = { @@ -218,6 +223,13 @@ services.tangled-spindle.package = lib.mkDefault self.packages.${pkgs.system}.spindle; }; - nixosConfigurations.vm = import ./nix/vm.nix {inherit self nixpkgs;}; + nixosConfigurations.vm-x86_64 = import ./nix/vm.nix { + inherit self nixpkgs; + system = "x86_64-linux"; + }; + nixosConfigurations.vm-aarch64 = import ./nix/vm.nix { + inherit self nixpkgs; + system = "aarch64-linux"; + }; }; } diff --git a/nix/vm.nix b/nix/vm.nix index 41ff743..07498b0 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -1,9 +1,10 @@ { nixpkgs, + system, self, }: nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; + inherit system; modules = [ self.nixosModules.knot self.nixosModules.spindle -- 2.43.0 From cf07090e9f9976ff6fa14558f44486bfa06827e5 Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 19:28:32 -0400 Subject: [PATCH] nix/vm: don't hardcode knot secret and spindle owner Signed-off-by: Winter --- docs/hacking.md | 19 ++++--- nix/vm.nix | 137 +++++++++++++++++++++++++----------------------- 2 files changed, 81 insertions(+), 75 deletions(-) diff --git a/docs/hacking.md b/docs/hacking.md index e7d279b..a16c0ab 100644 --- a/docs/hacking.md +++ b/docs/hacking.md @@ -56,9 +56,9 @@ quite cumbersome. So the nix flake provides a `nixosConfiguration` to do so. To begin, head to `http://localhost:3000/knots` in the browser -and generate a knot secret. Replace the existing secret in -`nix/vm.nix` (`KNOT_SERVER_SECRET`) with the newly generated -secret. +and generate a knot secret. Set `$TANGLED_KNOT_SECRET` to it, +ideally in a `.envrc` with [direnv](https://direnv.net) so you +don't lose it. You can now start a lightweight NixOS VM using `nixos-shell` like so: @@ -91,13 +91,12 @@ git push local-dev main ## running a spindle -Be sure to change the `owner` field for the spindle in -`nix/vm.nix` to your own DID. The above VM should already -be running a spindle on `localhost:6555`. You can head to -the spindle dashboard on `http://localhost:3000/spindles`, -and register a spindle with hostname `localhost:6555`. It -should instantly be verified. You can then configure each -repository to use this spindle and run CI jobs. +Be sure to set `$TANGLED_SPINDLE_OWNER` to your own DID. +The above VM should already be running a spindle on `localhost:6555`. +You can head to the spindle dashboard on `http://localhost:3000/spindles`, +and register a spindle with hostname `localhost:6555`. It should instantly +be verified. You can then configure each repository to use this spindle +and run CI jobs. Of interest when debugging spindles: diff --git a/nix/vm.nix b/nix/vm.nix index 07498b0..e7c5d44 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -2,72 +2,79 @@ nixpkgs, system, self, -}: -nixpkgs.lib.nixosSystem { - inherit system; - modules = [ - self.nixosModules.knot - self.nixosModules.spindle - ({ - config, - pkgs, - ... - }: { - virtualisation = { - memorySize = 2048; - diskSize = 10 * 1024; - cores = 2; - forwardPorts = [ - # ssh - { - from = "host"; - host.port = 2222; - guest.port = 22; - } - # knot - { - from = "host"; - host.port = 6000; - guest.port = 6000; - } - # spindle - { - from = "host"; - host.port = 6555; - guest.port = 6555; - } +}: let + envVar = name: let + var = builtins.getEnv name; + in + if var == "" + then throw "\$${name} must be defined, see docs/hacking.md for more details" + else var; +in + nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.knot + self.nixosModules.spindle + ({ + config, + pkgs, + ... + }: { + virtualisation = { + memorySize = 2048; + diskSize = 10 * 1024; + cores = 2; + forwardPorts = [ + # ssh + { + from = "host"; + host.port = 2222; + guest.port = 22; + } + # knot + { + from = "host"; + host.port = 6000; + guest.port = 6000; + } + # spindle + { + from = "host"; + host.port = 6555; + guest.port = 6555; + } + ]; + }; + services.getty.autologinUser = "root"; + environment.systemPackages = with pkgs; [curl vim git]; + systemd.tmpfiles.rules = let + u = config.services.tangled-knot.gitUser; + g = config.services.tangled-knot.gitUser; + in [ + "d /var/lib/knot 0770 ${u} ${g} - -" # Create the directory first + "f+ /var/lib/knot/secret 0660 ${u} ${g} - KNOT_SERVER_SECRET=${envVar "TANGLED_VM_KNOT_SECRET"}" ]; - }; - services.getty.autologinUser = "root"; - environment.systemPackages = with pkgs; [curl vim git]; - systemd.tmpfiles.rules = let - u = config.services.tangled-knot.gitUser; - g = config.services.tangled-knot.gitUser; - in [ - "d /var/lib/knot 0770 ${u} ${g} - -" # Create the directory first - "f+ /var/lib/knot/secret 0660 ${u} ${g} - KNOT_SERVER_SECRET=168c426fa6d9829fcbe85c96bdf144e800fb9737d6ca87f21acc543b1aa3e440" - ]; - services.tangled-knot = { - enable = true; - motd = "Welcome to the development knot!\n"; - server = { - secretFile = "/var/lib/knot/secret"; - hostname = "localhost:6000"; - listenAddr = "0.0.0.0:6000"; + services.tangled-knot = { + enable = true; + motd = "Welcome to the development knot!\n"; + server = { + secretFile = "/var/lib/knot/secret"; + hostname = "localhost:6000"; + listenAddr = "0.0.0.0:6000"; + }; }; - }; - services.tangled-spindle = { - enable = true; - server = { - owner = "did:plc:qfpnj4og54vl56wngdriaxug"; - hostname = "localhost:6555"; - listenAddr = "0.0.0.0:6555"; - dev = true; - secrets = { - provider = "sqlite"; + services.tangled-spindle = { + enable = true; + server = { + owner = envVar "TANGLED_VM_SPINDLE_OWNER"; + hostname = "localhost:6555"; + listenAddr = "0.0.0.0:6555"; + dev = true; + secrets = { + provider = "sqlite"; + }; }; }; - }; - }) - ]; -} + }) + ]; + } -- 2.43.0 From 9e7cdea49d23fe89868f3fac72282c33b55f80ed Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 20:20:54 -0400 Subject: [PATCH] nix/vm: fix on non-Linux systems Signed-off-by: Winter --- flake.nix | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 92bdc6b..bb19983 100644 --- a/flake.nix +++ b/flake.nix @@ -182,10 +182,23 @@ if pkgs.stdenv.hostPlatform.isAarch64 then "aarch64" else "x86_64"; + + nixos-shell = pkgs.nixos-shell.overrideAttrs (old: { + patches = + (old.patches or []) + ++ [ + # https://github.com/Mic92/nixos-shell/pull/94 + (pkgs.fetchpatch { + name = "fix-foreign-vm.patch"; + url = "https://github.com/Mic92/nixos-shell/commit/113e4cc55ae236b5b0b1fbd8b321e9b67c77580e.patch"; + hash = "sha256-eauetBK0wXAOcd9PYbExokNCiwz2QyFnZ4FnwGi9VCo="; + }) + ]; + }); in { type = "app"; program = toString (pkgs.writeShellScript "vm" '' - ${pkgs.nixos-shell}/bin/nixos-shell --flake .#vm-${system} + ${nixos-shell}/bin/nixos-shell --flake .#vm-${system} --guest-system ${system}-linux ''); }; gomod2nix = { -- 2.43.0 From 64d963ad61e5b6f1afc7d7393638d449822e72ac Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 20:22:53 -0400 Subject: [PATCH] nix/vm: isolate it a bit more I personally don't like that nixos-shell inherits a ton of stuff from the host by default, even mounting my home directory as r/w! I imagine I'm not the only one with this opinion, so let's put a stop to it by default. Signed-off-by: Winter --- nix/vm.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/vm.nix b/nix/vm.nix index e7c5d44..cfc50b7 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -20,6 +20,13 @@ in pkgs, ... }: { + nixos-shell = { + inheritPath = false; + mounts = { + mountHome = false; + mountNixProfile = false; + }; + }; virtualisation = { memorySize = 2048; diskSize = 10 * 1024; -- 2.43.0 From 1b0033f68b0f77897d97c0b5d7a485b489559230 Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 20:32:12 -0400 Subject: [PATCH] nix/vm: add debugging tools mentioned in docs Needed now that we're isolating things for everyone. Signed-off-by: Winter --- nix/vm.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/vm.nix b/nix/vm.nix index cfc50b7..6297fcb 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -53,7 +53,7 @@ in ]; }; services.getty.autologinUser = "root"; - environment.systemPackages = with pkgs; [curl vim git]; + environment.systemPackages = with pkgs; [curl vim git sqlite litecli]; systemd.tmpfiles.rules = let u = config.services.tangled-knot.gitUser; g = config.services.tangled-knot.gitUser; -- 2.43.0 From 10cd7fbbec8b26fa8f75b170e24c54a904f58eb6 Mon Sep 17 00:00:00 2001 From: Winter Date: Sun, 10 Aug 2025 02:55:09 -0400 Subject: [PATCH] nix: add script to do the lexgen dance h/t https://github.com/bluesky-social/indigo/issues/931#issuecomment-2635675099 for the inspiration Signed-off-by: Winter --- flake.nix | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/flake.nix b/flake.nix index bb19983..174f19b 100644 --- a/flake.nix +++ b/flake.nix @@ -207,6 +207,31 @@ ${gomod2nix.legacyPackages.${system}.gomod2nix}/bin/gomod2nix generate --outdir ./nix ''); }; + lexgen = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "lexgen"; + text = '' + if ! command -v lexgen > /dev/null; then + echo "error: must be executed from devshell" + exit 1 + fi + + rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1) + cd "$rootDir" + + rm api/tangled/* + lexgen --build-file lexicon-build-config.json lexicons + sed -i.bak 's/\tutil/\/\/\tutil/' api/tangled/* + ${pkgs.gotools}/bin/goimports -w api/tangled/* + go run cmd/gen.go + lexgen --build-file lexicon-build-config.json lexicons + rm api/tangled/*.bak + ''; + }) + + /bin/lexgen; + }; }); nixosModules.appview = { -- 2.43.0 From 64bc12979b38ff24007055be7aac4fb3a22e6884 Mon Sep 17 00:00:00 2001 From: Winter Date: Sat, 9 Aug 2025 22:02:31 -0400 Subject: [PATCH] appview/db: fix comparing against []byte SQLite stores byte sequences as blobs, which we were not handling properly, causing (at least) actions relating to artifacts to break. (Not sure when they broke, sorry!) Signed-off-by: Winter --- appview/db/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appview/db/db.go b/appview/db/db.go index e18646d..e3b3d6e 100644 --- a/appview/db/db.go +++ b/appview/db/db.go @@ -728,7 +728,7 @@ func (f filter) Condition() string { kind := rv.Kind() // if we have `FilterIn(k, [1, 2, 3])`, compile it down to `k in (?, ?, ?)` - if kind == reflect.Slice || kind == reflect.Array { + if (kind == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8) || kind == reflect.Array { if rv.Len() == 0 { // always false return "1 = 0" @@ -748,7 +748,7 @@ func (f filter) Condition() string { func (f filter) Arg() []any { rv := reflect.ValueOf(f.arg) kind := rv.Kind() - if kind == reflect.Slice || kind == reflect.Array { + if (kind == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8) || kind == reflect.Array { if rv.Len() == 0 { return nil } -- 2.43.0 From 7a076be1af4677a7246a792fecb463e4e6f1afee Mon Sep 17 00:00:00 2001 From: Safwan Parkar Date: Sun, 10 Aug 2025 01:49:18 +0200 Subject: [PATCH] appview: pages/templates/repo: fix styling for file tree items Signed-off-by: Safwan Parkar --- appview/pages/templates/repo/tree.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appview/pages/templates/repo/tree.html b/appview/pages/templates/repo/tree.html index f596f88..25882ff 100644 --- a/appview/pages/templates/repo/tree.html +++ b/appview/pages/templates/repo/tree.html @@ -61,11 +61,12 @@ {{ if .IsFile }} {{ $icon = "file" }} - {{ $iconStyle = "size-4" }} + {{ $iconStyle = "flex-shrink-0 size-4" }} {{ end }}
- {{ i $icon $iconStyle }}{{ .Name }} + {{ i $icon $iconStyle }} + {{ .Name }}
-- 2.43.0 From bdf27f0db30a2fe9858c01e1fa24376d89296535 Mon Sep 17 00:00:00 2001 From: dusk Date: Sat, 9 Aug 2025 16:18:21 +0300 Subject: [PATCH] nix: dedup appview static files into a package, and use that everywhere, including copying before watching appview Signed-off-by: dusk --- flake.nix | 29 +++++++++++++---------------- nix/pkgs/appview-static-files.nix | 23 +++++++++++++++++++++++ nix/pkgs/appview.nix | 17 +++-------------- 3 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 nix/pkgs/appview-static-files.nix diff --git a/flake.nix b/flake.nix index 174f19b..3bd2d01 100644 --- a/flake.nix +++ b/flake.nix @@ -75,9 +75,10 @@ }; genjwks = self.callPackage ./nix/pkgs/genjwks.nix {}; lexgen = self.callPackage ./nix/pkgs/lexgen.nix {inherit indigo;}; - appview = self.callPackage ./nix/pkgs/appview.nix { + appview-static-files = self.callPackage ./nix/pkgs/appview-static-files.nix { inherit htmx-src htmx-ws-src lucide-src inter-fonts-src ibm-plex-mono-src; }; + appview = self.callPackage ./nix/pkgs/appview.nix {}; spindle = self.callPackage ./nix/pkgs/spindle.nix {}; knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {}; knot = self.callPackage ./nix/pkgs/knot.nix {}; @@ -93,13 +94,7 @@ staticPackages = mkPackageSet pkgs.pkgsStatic; crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic; in { - appview = packages.appview; - lexgen = packages.lexgen; - knot = packages.knot; - knot-unwrapped = packages.knot-unwrapped; - spindle = packages.spindle; - genjwks = packages.genjwks; - sqlite-lib = packages.sqlite-lib; + inherit (packages) appview appview-static-files lexgen genjwks spindle knot knot-unwrapped sqlite-lib; pkgsStatic-appview = staticPackages.appview; pkgsStatic-knot = staticPackages.knot; @@ -132,16 +127,13 @@ pkgs.tailwindcss pkgs.nixos-shell pkgs.redis + pkgs.coreutils # for those of us who are on systems that use busybox (alpine) packages'.lexgen ]; shellHook = '' - mkdir -p appview/pages/static/{fonts,icons} - cp -f ${htmx-src} appview/pages/static/htmx.min.js - cp -f ${htmx-ws-src} appview/pages/static/htmx-ext-ws.min.js - cp -rf ${lucide-src}/*.svg appview/pages/static/icons/ - cp -f ${inter-fonts-src}/web/InterVariable*.woff2 appview/pages/static/fonts/ - cp -f ${inter-fonts-src}/web/InterDisplay*.woff2 appview/pages/static/fonts/ - cp -f ${ibm-plex-mono-src}/fonts/complete/woff2/IBMPlexMono-Regular.woff2 appview/pages/static/fonts/ + mkdir -p appview/pages/static + # no preserve is needed because watch-tailwind will want to be able to overwrite + cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static export TANGLED_OAUTH_JWKS="$(${packages'.genjwks}/bin/genjwks)" ''; env.CGO_ENABLED = 1; @@ -149,6 +141,7 @@ }); apps = forAllSystems (system: let pkgs = nixpkgsFor."${system}"; + packages' = self.packages.${system}; air-watcher = name: arg: pkgs.writeShellScriptBin "run" '' @@ -167,7 +160,11 @@ in { watch-appview = { type = "app"; - program = ''${air-watcher "appview" ""}/bin/run''; + program = toString (pkgs.writeShellScript "watch-appview" '' + echo "copying static files to appview/pages/static..." + ${pkgs.coreutils}/bin/cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static + ${air-watcher "appview" ""}/bin/run + ''); }; watch-knot = { type = "app"; diff --git a/nix/pkgs/appview-static-files.nix b/nix/pkgs/appview-static-files.nix new file mode 100644 index 0000000..3b55bc1 --- /dev/null +++ b/nix/pkgs/appview-static-files.nix @@ -0,0 +1,23 @@ +{ + runCommandLocal, + htmx-src, + htmx-ws-src, + lucide-src, + inter-fonts-src, + ibm-plex-mono-src, + sqlite-lib, + tailwindcss, + src, +}: +runCommandLocal "appview-static-files" {} '' + mkdir -p $out/{fonts,icons} && cd $out + cp -f ${htmx-src} htmx.min.js + cp -f ${htmx-ws-src} htmx-ext-ws.min.js + cp -rf ${lucide-src}/*.svg icons/ + cp -f ${inter-fonts-src}/web/InterVariable*.woff2 fonts/ + cp -f ${inter-fonts-src}/web/InterDisplay*.woff2 fonts/ + cp -f ${ibm-plex-mono-src}/fonts/complete/woff2/IBMPlexMono-Regular.woff2 fonts/ + # tailwindcss -c $src/tailwind.config.js -i $src/input.css -o tw.css won't work + # for whatever reason (produces broken css), so we are doing this instead + cd ${src} && ${tailwindcss}/bin/tailwindcss -i input.css -o $out/tw.css +'' diff --git a/nix/pkgs/appview.nix b/nix/pkgs/appview.nix index 2d438c5..90e7fb1 100644 --- a/nix/pkgs/appview.nix +++ b/nix/pkgs/appview.nix @@ -1,12 +1,7 @@ { buildGoApplication, modules, - htmx-src, - htmx-ws-src, - lucide-src, - inter-fonts-src, - ibm-plex-mono-src, - tailwindcss, + appview-static-files, sqlite-lib, src, }: @@ -17,14 +12,8 @@ buildGoApplication { postUnpack = '' pushd source - mkdir -p appview/pages/static/{fonts,icons} - cp -f ${htmx-src} appview/pages/static/htmx.min.js - cp -f ${htmx-ws-src} appview/pages/static/htmx-ext-ws.min.js - cp -rf ${lucide-src}/*.svg appview/pages/static/icons/ - cp -f ${inter-fonts-src}/web/InterVariable*.woff2 appview/pages/static/fonts/ - cp -f ${inter-fonts-src}/web/InterDisplay*.woff2 appview/pages/static/fonts/ - cp -f ${ibm-plex-mono-src}/fonts/complete/woff2/IBMPlexMono-Regular.woff2 appview/pages/static/fonts/ - ${tailwindcss}/bin/tailwindcss -i input.css -o appview/pages/static/tw.css + mkdir -p appview/pages/static + cp -frv ${appview-static-files}/* appview/pages/static popd ''; -- 2.43.0 From 7dda83762cee7a7d33e3061e60bc7f7f120f389c Mon Sep 17 00:00:00 2001 From: dusk Date: Sat, 9 Aug 2025 14:55:25 +0300 Subject: [PATCH] appview: strings: return 404 page if string wasnt found Signed-off-by: dusk --- appview/strings/strings.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appview/strings/strings.go b/appview/strings/strings.go index 130329f..8855a04 100644 --- a/appview/strings/strings.go +++ b/appview/strings/strings.go @@ -99,6 +99,11 @@ func (s *Strings) contents(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) return } + if len(strings) < 1 { + l.Error("string not found") + s.Pages.Error404(w) + return + } if len(strings) != 1 { l.Error("incorrect number of records returned", "len(strings)", len(strings)) w.WriteHeader(http.StatusInternalServerError) -- 2.43.0 From 730a252f125bba7f6c73218a31829389ec166d0e Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Sun, 10 Aug 2025 19:04:57 +0100 Subject: [PATCH] nix: fmt Change-Id: qostszkwtptyvpqnyuqonzosouwtssyu Signed-off-by: oppiliappan --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 3bd2d01..c064b7b 100644 --- a/flake.nix +++ b/flake.nix @@ -161,9 +161,9 @@ watch-appview = { type = "app"; program = toString (pkgs.writeShellScript "watch-appview" '' - echo "copying static files to appview/pages/static..." - ${pkgs.coreutils}/bin/cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static - ${air-watcher "appview" ""}/bin/run + echo "copying static files to appview/pages/static..." + ${pkgs.coreutils}/bin/cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static + ${air-watcher "appview" ""}/bin/run ''); }; watch-knot = { -- 2.43.0 From 3941b800076fa12bd33e19e941b4583fa2bc9456 Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Mon, 11 Aug 2025 08:33:13 +0100 Subject: [PATCH] spindle: ensure that event source is identical to the repo's knot Change-Id: sosworwrzrsntlukykwkpqxwtuylprlw thanks @winter.bsky.social for the report! Signed-off-by: oppiliappan --- spindle/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spindle/server.go b/spindle/server.go index 59e52fe..f04b206 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -242,6 +242,10 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, return fmt.Errorf("no repo data found") } + if src.Key() != tpl.TriggerMetadata.Repo.Knot { + return fmt.Errorf("repo knot does not match event source: %s != %s", src.Key(), tpl.TriggerMetadata.Repo.Knot) + } + // filter by repos _, err = s.db.GetRepo( tpl.TriggerMetadata.Repo.Knot, -- 2.43.0 From 6f33492348a0e14af22fefc4c5aeac3fd2804da8 Mon Sep 17 00:00:00 2001 From: Winter Date: Sun, 10 Aug 2025 18:53:24 -0400 Subject: [PATCH] nix: don't use cp --verbose It's noisy, especially upon entering every devshell. Signed-off-by: Winter --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index c064b7b..db11dd2 100644 --- a/flake.nix +++ b/flake.nix @@ -133,7 +133,7 @@ shellHook = '' mkdir -p appview/pages/static # no preserve is needed because watch-tailwind will want to be able to overwrite - cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static + cp -fr --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static export TANGLED_OAUTH_JWKS="$(${packages'.genjwks}/bin/genjwks)" ''; env.CGO_ENABLED = 1; @@ -162,7 +162,7 @@ type = "app"; program = toString (pkgs.writeShellScript "watch-appview" '' echo "copying static files to appview/pages/static..." - ${pkgs.coreutils}/bin/cp -frv --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static + ${pkgs.coreutils}/bin/cp -fr --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static ${air-watcher "appview" ""}/bin/run ''); }; -- 2.43.0 From 886b5a754f7da2646b4210dd4f5e25e284aa5814 Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Sun, 10 Aug 2025 22:56:13 +0100 Subject: [PATCH] appview/pages: remove repo/settings.html this file is unused Signed-off-by: oppiliappan --- appview/pages/templates/repo/settings.html | 168 --------------------- 1 file changed, 168 deletions(-) delete mode 100644 appview/pages/templates/repo/settings.html diff --git a/appview/pages/templates/repo/settings.html b/appview/pages/templates/repo/settings.html deleted file mode 100644 index cb6b2b7..0000000 --- a/appview/pages/templates/repo/settings.html +++ /dev/null @@ -1,168 +0,0 @@ -{{ define "title" }}settings · {{ .RepoInfo.FullName }}{{ end }} - -{{ define "repoContent" }} - {{ template "collaboratorSettings" . }} - {{ template "branchSettings" . }} - {{ template "dangerZone" . }} - {{ template "spindleSelector" . }} - {{ template "spindleSecrets" . }} -{{ end }} - -{{ define "collaboratorSettings" }} -
- Collaborators -
- -
- {{ range .Collaborators }} -
- - {{ didOrHandle .Did .Handle }} - -
- - {{ .Role }} - -
-
- {{ end }} -
- - {{ if .RepoInfo.Roles.CollaboratorInviteAllowed }} -
- - - -
- {{ end }} -{{ end }} - -{{ define "dangerZone" }} - {{ if .RepoInfo.Roles.RepoDeleteAllowed }} -
- - - - Deleting a repository is irreversible and permanent. - -
- {{ end }} -{{ end }} - -{{ define "branchSettings" }} -
- -
- - -
-
-{{ end }} - -{{ define "spindleSelector" }} - {{ if .RepoInfo.Roles.IsOwner }} -
- -
- - -
-
- {{ end }} -{{ end }} - -{{ define "spindleSecrets" }} - {{ if $.CurrentSpindle }} -
- Secrets -
- -
- {{ range $idx, $secret := .Secrets }} - {{ with $secret }} -
- {{ .Key }} created on {{ .CreatedAt }} by {{ .CreatedBy }} -
- {{ end }} - {{ end }} -
-
- - - - - - -
- {{ end }} -{{ end }} -- 2.43.0 From 7e4a93c013b915d296ac964661eff32233fc155c Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Sun, 10 Aug 2025 23:01:20 +0100 Subject: [PATCH] appview/pages: unify tab styles Signed-off-by: oppiliappan --- appview/pages/templates/repo/pipelines/workflow.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/appview/pages/templates/repo/pipelines/workflow.html b/appview/pages/templates/repo/pipelines/workflow.html index 62dc416..0ec3798 100644 --- a/appview/pages/templates/repo/pipelines/workflow.html +++ b/appview/pages/templates/repo/pipelines/workflow.html @@ -19,13 +19,17 @@ {{ define "sidebar" }} {{ $active := .Workflow }} + + {{ $activeTab := "bg-white dark:bg-gray-700 drop-shadow-sm" }} + {{ $inactiveTab := "bg-gray-100 dark:bg-gray-800" }} + {{ with .Pipeline }} {{ $id := .Id }}
{{ range $name, $all := .Statuses }}
+ class="flex gap-2 items-center justify-between p-2 {{ if eq $name $active }} {{ $activeTab }} {{ else }} {{ $inactiveTab }} {{ end }}"> {{ $lastStatus := $all.Latest }} {{ $kind := $lastStatus.Status.String }} -- 2.43.0 From fb64beba9c0bb4b9bf3b22d22f74c751def3ec4c Mon Sep 17 00:00:00 2001 From: Tom Sherman Date: Mon, 11 Aug 2025 11:41:01 +0100 Subject: [PATCH] Initial implementation sketch --- api/tangled/cbor_gen.go | 244 ++++++++++++++++++++++- api/tangled/tangledpipeline.go | 12 +- cmd/gen.go | 1 + flake.nix | 2 +- lexicons/pipeline/pipeline.json | 17 ++ spindle/engine/engine.go | 18 +- spindle/models/pipeline.go | 14 ++ spindle/oidc/oidc.go | 329 ++++++++++++++++++++++++++++++++ spindle/server.go | 15 +- 9 files changed, 640 insertions(+), 12 deletions(-) create mode 100644 spindle/oidc/oidc.go diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go index eba0337..6320bf7 100644 --- a/api/tangled/cbor_gen.go +++ b/api/tangled/cbor_gen.go @@ -3923,12 +3923,16 @@ func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 3 + fieldCount := 4 if t.Environment == nil { fieldCount-- } + if t.Oidcs_tokens == nil { + fieldCount-- + } + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -4007,6 +4011,35 @@ func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { } } + + // t.Oidcs_tokens ([]*tangled.Pipeline_Step_Oidcs_tokens_Elem) (slice) + if t.Oidcs_tokens != nil { + + if len("oidcs_tokens") > 1000000 { + return xerrors.Errorf("Value in field \"oidcs_tokens\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("oidcs_tokens"))); err != nil { + return err + } + if _, err := cw.WriteString(string("oidcs_tokens")); err != nil { + return err + } + + if len(t.Oidcs_tokens) > 8192 { + return xerrors.Errorf("Slice value in field t.Oidcs_tokens was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Oidcs_tokens))); err != nil { + return err + } + for _, v := range t.Oidcs_tokens { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } return nil } @@ -4035,7 +4068,7 @@ func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 11) + nameBuf := make([]byte, 12) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -4122,6 +4155,213 @@ func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { } } + // t.Oidcs_tokens ([]*tangled.Pipeline_Step_Oidcs_tokens_Elem) (slice) + case "oidcs_tokens": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Oidcs_tokens: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Oidcs_tokens = make([]*Pipeline_Step_Oidcs_tokens_Elem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Oidcs_tokens[i] = new(Pipeline_Step_Oidcs_tokens_Elem) + if err := t.Oidcs_tokens[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Oidcs_tokens[i] pointer: %w", err) + } + } + + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *Pipeline_Step_Oidcs_tokens_Elem) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 2 + + if t.Aud == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Aud (string) (string) + if t.Aud != nil { + + if len("aud") > 1000000 { + return xerrors.Errorf("Value in field \"aud\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("aud"))); err != nil { + return err + } + if _, err := cw.WriteString(string("aud")); err != nil { + return err + } + + if t.Aud == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Aud) > 1000000 { + return xerrors.Errorf("Value in field t.Aud was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Aud))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Aud)); err != nil { + return err + } + } + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + return nil +} + +func (t *Pipeline_Step_Oidcs_tokens_Elem) UnmarshalCBOR(r io.Reader) (err error) { + *t = Pipeline_Step_Oidcs_tokens_Elem{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Pipeline_Step_Oidcs_tokens_Elem: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 4) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Aud (string) (string) + case "aud": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Aud = (*string)(&sval) + } + } + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } default: // Field doesn't exist on this type, so ignore it diff --git a/api/tangled/tangledpipeline.go b/api/tangled/tangledpipeline.go index ae625c4..8611534 100644 --- a/api/tangled/tangledpipeline.go +++ b/api/tangled/tangledpipeline.go @@ -63,9 +63,15 @@ type Pipeline_PushTriggerData struct { // Pipeline_Step is a "step" in the sh.tangled.pipeline schema. type Pipeline_Step struct { - Command string `json:"command" cborgen:"command"` - Environment []*Pipeline_Pair `json:"environment,omitempty" cborgen:"environment,omitempty"` - Name string `json:"name" cborgen:"name"` + Command string `json:"command" cborgen:"command"` + Environment []*Pipeline_Pair `json:"environment,omitempty" cborgen:"environment,omitempty"` + Name string `json:"name" cborgen:"name"` + Oidcs_tokens []*Pipeline_Step_Oidcs_tokens_Elem `json:"oidcs_tokens,omitempty" cborgen:"oidcs_tokens,omitempty"` +} + +type Pipeline_Step_Oidcs_tokens_Elem struct { + Aud *string `json:"aud,omitempty" cborgen:"aud,omitempty"` + Name string `json:"name" cborgen:"name"` } // Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema. diff --git a/cmd/gen.go b/cmd/gen.go index 2e07c0e..a33c6da 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -34,6 +34,7 @@ func main() { tangled.Pipeline_PushTriggerData{}, tangled.PipelineStatus{}, tangled.Pipeline_Step{}, + tangled.Pipeline_Step_Oidcs_tokens_Elem{}, tangled.Pipeline_TriggerMetadata{}, tangled.Pipeline_TriggerRepo{}, tangled.Pipeline_Workflow{}, diff --git a/flake.nix b/flake.nix index db11dd2..739ef64 100644 --- a/flake.nix +++ b/flake.nix @@ -116,7 +116,7 @@ stdenv = pkgs.pkgsStatic.stdenv; }; in { - default = staticShell { + default = pkgs.mkShell { nativeBuildInputs = [ pkgs.go pkgs.air diff --git a/lexicons/pipeline/pipeline.json b/lexicons/pipeline/pipeline.json index d9e9872..b0894b8 100644 --- a/lexicons/pipeline/pipeline.json +++ b/lexicons/pipeline/pipeline.json @@ -241,6 +241,23 @@ "type": "ref", "ref": "#pair" } + }, + "oidcs_tokens": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "aud": { + "type": "string" + } + } + } } } }, diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index 65e32eb..2308699 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -25,6 +25,7 @@ import ( "tangled.sh/tangled.sh/core/spindle/config" "tangled.sh/tangled.sh/core/spindle/db" "tangled.sh/tangled.sh/core/spindle/models" + "tangled.sh/tangled.sh/core/spindle/oidc" "tangled.sh/tangled.sh/core/spindle/secrets" ) @@ -41,12 +42,13 @@ type Engine struct { n *notifier.Notifier cfg *config.Config vault secrets.Manager + oidc oidc.OidcTokenGenerator cleanupMu sync.Mutex cleanup map[string][]cleanupFunc } -func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifier, vault secrets.Manager) (*Engine, error) { +func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifier, vault secrets.Manager, oidc *oidc.OidcTokenGenerator) (*Engine, error) { dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return nil, err @@ -61,6 +63,7 @@ func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifie n: n, cfg: cfg, vault: vault, + oidc: *oidc, } e.cleanup = make(map[string][]cleanupFunc) @@ -124,7 +127,7 @@ func (e *Engine) StartWorkflows(ctx context.Context, pipeline *models.Pipeline, ctx, cancel := context.WithTimeout(ctx, workflowTimeout) defer cancel() - err = e.StartSteps(ctx, wid, w, allSecrets) + err = e.StartSteps(ctx, wid, w, allSecrets, pipeline, pipelineId) if err != nil { if errors.Is(err, ErrTimedOut) { dbErr := e.db.StatusTimeout(wid, e.n) @@ -202,7 +205,7 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId) error // ONLY marks pipeline as failed if container's exit code is non-zero. // All other errors are bubbled up. // Fixed version of the step execution logic -func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models.Workflow, secrets []secrets.UnlockedSecret) error { +func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models.Workflow, secrets []secrets.UnlockedSecret, pipeline *models.Pipeline, pipelineId models.PipelineId) error { workflowEnvs := ConstructEnvs(w.Environment) for _, s := range secrets { workflowEnvs.AddEnv(s.Key, s.Value) @@ -222,6 +225,15 @@ func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models envs.AddEnv("HOME", workspaceDir) e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice()) + for _, t := range step.OidcTokens { + token, err := e.oidc.CreateToken(t, pipelineId, pipeline.RepoOwner, pipeline.RepoName) + if err != nil { + e.l.Error("failed to get OIDC token", "error", err, "token", t.Name) + return fmt.Errorf("getting OIDC token: %w", err) + } + envs.AddEnv(t.Name, token) + } + hostConfig := hostConfig(wid) resp, err := e.docker.ContainerCreate(ctx, &container.Config{ Image: w.Image, diff --git a/spindle/models/pipeline.go b/spindle/models/pipeline.go index 8561b21..85619e9 100644 --- a/spindle/models/pipeline.go +++ b/spindle/models/pipeline.go @@ -18,6 +18,7 @@ type Step struct { Name string Environment map[string]string Kind StepKind + OidcTokens []OidcToken } type StepKind int @@ -29,6 +30,11 @@ const ( StepKindUser ) +type OidcToken struct { + Name string + Aud *string +} + type Workflow struct { Steps []Step Environment map[string]string @@ -60,6 +66,14 @@ func ToPipeline(pl tangled.Pipeline, cfg config.Config) *Pipeline { sstep.Name = tstep.Name sstep.Kind = StepKindUser swf.Steps = append(swf.Steps, sstep) + + sstep.OidcTokens = make([]OidcToken, 0, len(tstep.Oidcs_tokens)) + for _, ttoken := range tstep.Oidcs_tokens { + sstep.OidcTokens = append(sstep.OidcTokens, OidcToken{ + Name: ttoken.Name, + Aud: ttoken.Aud, + }) + } } swf.Name = twf.Name swf.Environment = workflowEnvToMap(twf.Environment) diff --git a/spindle/oidc/oidc.go b/spindle/oidc/oidc.go new file mode 100644 index 0000000..3434660 --- /dev/null +++ b/spindle/oidc/oidc.go @@ -0,0 +1,329 @@ +package oidc + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jwt" + "tangled.sh/tangled.sh/core/spindle/models" +) + +const JWKSPath = "/.well-known/jwks.json" + +// OidcKeyPair represents an OIDC key pair with both private and public keys +type OidcKeyPair struct { + privateKey *ecdsa.PrivateKey + publicKey *ecdsa.PublicKey + keyID string + jwkKey jwk.Key +} + +// OidcTokenGenerator handles OIDC token generation and key management with rotation +type OidcTokenGenerator struct { + currentKeyPair *OidcKeyPair + nextKeyPair *OidcKeyPair + l *slog.Logger + issuer string +} + +// NewOidcTokenGenerator creates a new OIDC token generator with in-memory key management +func NewOidcTokenGenerator(issuer string) (*OidcTokenGenerator, error) { + // Create new keys + currentKeyPair, err := NewOidcKeyPair() + if err != nil { + return nil, fmt.Errorf("failed to generate initial current key pair: %w", err) + } + + return &OidcTokenGenerator{ + issuer: issuer, + currentKeyPair: currentKeyPair, + }, nil +} + +// NewOidcKeyPair generates a new ECDSA key pair for OIDC token signing +func NewOidcKeyPair() (*OidcKeyPair, error) { + privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate ECDSA key: %w", err) + } + + keyID := fmt.Sprintf("spindle-%d", time.Now().Unix()) + + // Create JWK from the private key + jwkKey, err := jwk.FromRaw(privKey) + if err != nil { + return nil, fmt.Errorf("failed to create JWK from private key: %w", err) + } + + // Set the key ID + if err := jwkKey.Set(jwk.KeyIDKey, keyID); err != nil { + return nil, fmt.Errorf("failed to set key ID: %w", err) + } + + // Set algorithm + if err := jwkKey.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { + return nil, fmt.Errorf("failed to set algorithm: %w", err) + } + + // Set usage + if err := jwkKey.Set(jwk.KeyUsageKey, "sig"); err != nil { + return nil, fmt.Errorf("failed to set key usage: %w", err) + } + + return &OidcKeyPair{ + privateKey: privKey, + publicKey: &privKey.PublicKey, + keyID: keyID, + jwkKey: jwkKey, + }, nil +} + +// LoadOidcKeyPair loads an existing key pair from JWK JSON +func LoadOidcKeyPair(jwkJSON []byte) (*OidcKeyPair, error) { + jwkKey, err := jwk.ParseKey(jwkJSON) + if err != nil { + return nil, fmt.Errorf("failed to parse JWK: %w", err) + } + + var privKey *ecdsa.PrivateKey + if err := jwkKey.Raw(&privKey); err != nil { + return nil, fmt.Errorf("failed to extract private key: %w", err) + } + + keyID, ok := jwkKey.Get(jwk.KeyIDKey) + if !ok { + return nil, fmt.Errorf("JWK missing key ID") + } + + keyIDStr, ok := keyID.(string) + if !ok { + return nil, fmt.Errorf("JWK key ID is not a string") + } + + return &OidcKeyPair{ + privateKey: privKey, + publicKey: &privKey.PublicKey, + keyID: keyIDStr, + jwkKey: jwkKey, + }, nil +} + +// GetKeyID returns the key ID +func (k *OidcKeyPair) GetKeyID() string { + return k.keyID +} + +// RotateKeys performs key rotation: generates new next key, moves next to current +func (g *OidcTokenGenerator) RotateKeys() error { + // Generate a new key pair for the next key + newNextKeyPair, err := NewOidcKeyPair() + if err != nil { + return fmt.Errorf("failed to generate new next key pair: %w", err) + } + + // Perform rotation: next becomes current, new key becomes next + g.currentKeyPair = g.nextKeyPair + g.nextKeyPair = newNextKeyPair + + // If we don't have a current key (first time setup), use the new key + if g.currentKeyPair == nil { + g.currentKeyPair = newNextKeyPair + // Generate another new key for next + g.nextKeyPair, err = NewOidcKeyPair() + if err != nil { + return fmt.Errorf("failed to generate next key pair for first setup: %w", err) + } + } + + return nil +} + +func (g *OidcTokenGenerator) GetCurrentKeyID() string { + if g.currentKeyPair == nil { + return "" + } + return g.currentKeyPair.GetKeyID() +} + +// GetNextKeyID returns the next key's ID +func (g *OidcTokenGenerator) GetNextKeyID() string { + if g.nextKeyPair == nil { + return "" + } + return g.nextKeyPair.GetKeyID() +} + +// HasKeys returns true if the generator has at least a current key +func (g *OidcTokenGenerator) HasKeys() bool { + return g.currentKeyPair != nil +} + +// OidcClaims represents the claims in an OIDC token +type OidcClaims struct { + // Standard JWT claims + Issuer string `json:"iss"` + Subject string `json:"sub"` + Audience string `json:"aud"` + ExpiresAt int64 `json:"exp"` + NotBefore int64 `json:"nbf"` + IssuedAt int64 `json:"iat"` + JWTID string `json:"jti"` +} + +// CreateToken creates a signed JWT token for the given OidcToken and pipeline context +func (g *OidcTokenGenerator) CreateToken( + oidcToken models.OidcToken, + pipelineId models.PipelineId, + repoOwner, repoName string, +) (string, error) { + now := time.Now() + exp := now.Add(5 * time.Minute) + + // Determine audience - use the provided audience or default to issuer + audience := fmt.Sprintf(g.issuer) + if oidcToken.Aud != nil && *oidcToken.Aud != "" { + audience = *oidcToken.Aud + } + + pipelineUri := pipelineId.AtUri() + + // Create claims + claims := OidcClaims{ + Issuer: g.issuer, + // Hardcode the did as did:web of the issuer. At some point knots will have their own DIDs which will be used here + Subject: pipelineUri.String(), + Audience: audience, + ExpiresAt: exp.Unix(), + NotBefore: now.Unix(), + IssuedAt: now.Unix(), + // Repo owner, name, and id should be global unique but we add timestamp to ensure uniqueness + JWTID: fmt.Sprintf("%s/%s-%s-%d", repoOwner, repoName, pipelineUri.RecordKey(), now.Unix()), + } + + // Create JWT token + token := jwt.New() + + // Set all claims + if err := token.Set(jwt.IssuerKey, claims.Issuer); err != nil { + return "", fmt.Errorf("failed to set issuer: %w", err) + } + if err := token.Set(jwt.SubjectKey, claims.Subject); err != nil { + return "", fmt.Errorf("failed to set subject: %w", err) + } + if err := token.Set(jwt.AudienceKey, claims.Audience); err != nil { + return "", fmt.Errorf("failed to set audience: %w", err) + } + if err := token.Set(jwt.ExpirationKey, claims.ExpiresAt); err != nil { + return "", fmt.Errorf("failed to set expiration: %w", err) + } + if err := token.Set(jwt.NotBeforeKey, claims.NotBefore); err != nil { + return "", fmt.Errorf("failed to set not before: %w", err) + } + if err := token.Set(jwt.IssuedAtKey, claims.IssuedAt); err != nil { + return "", fmt.Errorf("failed to set issued at: %w", err) + } + if err := token.Set(jwt.JwtIDKey, claims.JWTID); err != nil { + return "", fmt.Errorf("failed to set JWT ID: %w", err) + } + + // Sign the token with the current key + if g.currentKeyPair == nil { + return "", fmt.Errorf("no current key pair available for signing") + } + signedToken, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, g.currentKeyPair.jwkKey)) + if err != nil { + return "", fmt.Errorf("failed to sign token: %w", err) + } + + return string(signedToken), nil +} + +// JWKSHandler serves the JWKS endpoint as an HTTP handler +func (g *OidcTokenGenerator) JWKSHandler(w http.ResponseWriter, r *http.Request) { + var keys []jwk.Key + + // Add current key if available + if g.currentKeyPair != nil { + pubJWK, err := jwk.PublicKeyOf(g.currentKeyPair.jwkKey) + if err != nil { + http.Error(w, fmt.Sprintf("failed to extract current public key from JWK: %v", err), http.StatusInternalServerError) + return + } + keys = append(keys, pubJWK) + } + + // Add next key if available + if g.nextKeyPair != nil { + pubJWK, err := jwk.PublicKeyOf(g.nextKeyPair.jwkKey) + if err != nil { + http.Error(w, fmt.Sprintf("failed to extract next public key from JWK: %v", err), http.StatusInternalServerError) + return + } + keys = append(keys, pubJWK) + } + + if len(keys) == 0 { + http.Error(w, "no keys available for JWKS", http.StatusInternalServerError) + return + } + + jwks := map[string]interface{}{ + "keys": keys, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(jwks); err != nil { + http.Error(w, fmt.Sprintf("failed to encode JWKS: %v", err), http.StatusInternalServerError) + } +} + +// DiscoveryHandler serves the OIDC discovery endpoint for JWKS +func (g *OidcTokenGenerator) DiscoveryHandler(w http.ResponseWriter, r *http.Request) { + claimsSupported := []string{ + "iss", + "sub", + "aud", + "exp", + "nbf", + "iat", + "jti", + } + + responseTypesSupported := []string{ + "id_token", + } + + subjectTypesSupported := []string{ + "public", + } + + idTokenSigningAlgValuesSupported := []string{ + jwa.RS256.String(), + } + + scopesSupported := []string{ + "openid", + } + + discovery := map[string]interface{}{ + "issuer": g.issuer, + "jwks_uri": fmt.Sprintf("%s%s", g.issuer, JWKSPath), + "claims_supported": claimsSupported, + "response_types_supported": responseTypesSupported, + "subject_types_supported": subjectTypesSupported, + "id_token_signing_alg_values_supported": idTokenSigningAlgValuesSupported, + "scopes_supported": scopesSupported, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(discovery); err != nil { + http.Error(w, fmt.Sprintf("failed to encode discovery document: %v", err), http.StatusInternalServerError) + } +} diff --git a/spindle/server.go b/spindle/server.go index f04b206..e68741c 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -21,6 +21,7 @@ import ( "tangled.sh/tangled.sh/core/spindle/db" "tangled.sh/tangled.sh/core/spindle/engine" "tangled.sh/tangled.sh/core/spindle/models" + "tangled.sh/tangled.sh/core/spindle/oidc" "tangled.sh/tangled.sh/core/spindle/queue" "tangled.sh/tangled.sh/core/spindle/secrets" "tangled.sh/tangled.sh/core/spindle/xrpc" @@ -93,7 +94,12 @@ func Run(ctx context.Context) error { return fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) } - eng, err := engine.New(ctx, cfg, d, &n, vault) + oidc, err := oidc.NewOidcTokenGenerator(cfg.Server.Hostname) + if err != nil { + return fmt.Errorf("failed to create OIDC token generator: %w", err) + } + + eng, err := engine.New(ctx, cfg, d, &n, vault, oidc) if err != nil { return err } @@ -188,12 +194,12 @@ func Run(ctx context.Context) error { }() logger.Info("starting spindle server", "address", cfg.Server.ListenAddr) - logger.Error("server error", "error", http.ListenAndServe(cfg.Server.ListenAddr, spindle.Router())) + logger.Error("server error", "error", http.ListenAndServe(cfg.Server.ListenAddr, spindle.Router(oidc))) return nil } -func (s *Spindle) Router() http.Handler { +func (s *Spindle) Router(oidcg *oidc.OidcTokenGenerator) http.Handler { mux := chi.NewRouter() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -204,6 +210,9 @@ func (s *Spindle) Router() http.Handler { w.Write([]byte(s.cfg.Server.Owner)) }) mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) + mux.HandleFunc(oidc.JWKSPath, oidcg.JWKSHandler) + mux.HandleFunc("/.well-known/oidc-configuration", oidcg.DiscoveryHandler) + // TODO: Do we need webfinger issuer discovery? mux.Mount("/xrpc", s.XrpcRouter()) return mux -- 2.43.0