From 438af2b510d85adbb72b279fa2772744f94ac423 Mon Sep 17 00:00:00 2001 From: Tim Culverhouse Date: Sat, 17 May 2025 06:27:29 -0800 Subject: [PATCH 1/2] appview: repo: move repo validation to db package --- appview/db/repos.go | 55 ++++++++++++++++++++++++++++++ appview/state/state.go | 76 ++++++++---------------------------------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/appview/db/repos.go b/appview/db/repos.go index 4226beea7072..202580b0bb7b 100644 --- a/appview/db/repos.go +++ b/appview/db/repos.go @@ -3,6 +3,7 @@ package db import ( "database/sql" "fmt" + "strings" "time" "github.com/bluesky-social/indigo/atproto/syntax" @@ -397,3 +398,57 @@ func scanRepo(rows *sql.Rows, did, name, knot, rkey, description *string, create return nil } + +// ValidateRepo ensures the repo has a knot specified, a valid name, and does +// not exist on the knot +func ValidateRepo(e Execer, repo *Repo) error { + if repo.Knot == "" { + return fmt.Errorf("missing knot domain") + } + + if err := validateRepoName(repo.Name); err != nil { + return err + } + + existingRepo, _ := GetRepo(e, repo.Did, repo.Name) + if existingRepo != nil { + return fmt.Errorf("A repo by this name already exists on %s", existingRepo.Knot) + } + + return nil +} + +func validateRepoName(name string) error { + if name == "" { + return fmt.Errorf("Repository name cannot be empty") + } + // check for path traversal attempts + if name == "." || name == ".." || + strings.Contains(name, "/") || strings.Contains(name, "\\") { + return fmt.Errorf("Repository name contains invalid path characters") + } + + // check for sequences that could be used for traversal when normalized + if strings.Contains(name, "./") || strings.Contains(name, "../") || + strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") { + return fmt.Errorf("Repository name contains invalid path sequence") + } + + // then continue with character validation + for _, char := range name { + if !((char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '-' || char == '_' || char == '.') { + return fmt.Errorf("Repository name can only contain alphanumeric characters, periods, hyphens, and underscores") + } + } + + // additional check to prevent multiple sequential dots + if strings.Contains(name, "..") { + return fmt.Errorf("Repository name cannot contain sequential dots") + } + + // if all checks pass + return nil +} diff --git a/appview/state/state.go b/appview/state/state.go index 89b98dca5306..8e5b17df2c22 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -250,7 +250,6 @@ func (s *State) RegistrationKey(w http.ResponseWriter, r *http.Request) { } key, err := db.GenerateRegistrationKey(s.db, domain, did) - if err != nil { log.Println(err) http.Error(w, "unable to register this domain", http.StatusNotAcceptable) @@ -536,9 +535,9 @@ func (s *State) AddMember(w http.ResponseWriter, r *http.Request) { Subject: subjectIdentity.DID.String(), Domain: domain, CreatedAt: createdAt, - }}, + }, + }, }) - // invalid record if err != nil { log.Printf("failed to create record: %s", err) @@ -581,38 +580,6 @@ func (s *State) AddMember(w http.ResponseWriter, r *http.Request) { func (s *State) RemoveMember(w http.ResponseWriter, r *http.Request) { } -func validateRepoName(name string) error { - // check for path traversal attempts - if name == "." || name == ".." || - strings.Contains(name, "/") || strings.Contains(name, "\\") { - return fmt.Errorf("Repository name contains invalid path characters") - } - - // check for sequences that could be used for traversal when normalized - if strings.Contains(name, "./") || strings.Contains(name, "../") || - strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") { - return fmt.Errorf("Repository name contains invalid path sequence") - } - - // then continue with character validation - for _, char := range name { - if !((char >= 'a' && char <= 'z') || - (char >= 'A' && char <= 'Z') || - (char >= '0' && char <= '9') || - char == '-' || char == '_' || char == '.') { - return fmt.Errorf("Repository name can only contain alphanumeric characters, periods, hyphens, and underscores") - } - } - - // additional check to prevent multiple sequential dots - if strings.Contains(name, "..") { - return fmt.Errorf("Repository name cannot contain sequential dots") - } - - // if all checks pass - return nil -} - func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: @@ -632,18 +599,19 @@ func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { user := s.oauth.GetUser(r) domain := r.FormValue("domain") - if domain == "" { - s.pages.Notice(w, "repo", "Invalid form submission—missing knot domain.") - return - } - repoName := r.FormValue("name") - if repoName == "" { - s.pages.Notice(w, "repo", "Repository name cannot be empty.") - return + description := r.FormValue("description") + + rkey := appview.TID() + repo := &db.Repo{ + Did: user.Did, + Name: repoName, + Knot: domain, + Rkey: rkey, + Description: description, } - if err := validateRepoName(repoName); err != nil { + if err := db.ValidateRepo(s.db, repo); err != nil { s.pages.Notice(w, "repo", err.Error()) return } @@ -653,20 +621,12 @@ func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { defaultBranch = "main" } - description := r.FormValue("description") - ok, err := s.enforcer.E.Enforce(user.Did, domain, domain, "repo:create") if err != nil || !ok { s.pages.Notice(w, "repo", "You do not have permission to create a repo in this knot.") return } - existingRepo, err := db.GetRepo(s.db, user.Did, repoName) - if err == nil && existingRepo != nil { - s.pages.Notice(w, "repo", fmt.Sprintf("A repo by this name already exists on %s", existingRepo.Knot)) - return - } - secret, err := db.GetRegistrationKey(s.db, domain) if err != nil { s.pages.Notice(w, "repo", fmt.Sprintf("No registration key found for knot %s.", domain)) @@ -679,15 +639,6 @@ func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { return } - rkey := appview.TID() - repo := &db.Repo{ - Did: user.Did, - Name: repoName, - Knot: domain, - Rkey: rkey, - Description: description, - } - xrpcClient, err := s.oauth.AuthorizedClient(r) if err != nil { s.pages.Notice(w, "repo", "Failed to write record to PDS.") @@ -705,7 +656,8 @@ func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { Name: repoName, CreatedAt: createdAt, Owner: user.Did, - }}, + }, + }, }) if err != nil { log.Printf("failed to create record: %s", err) -- 2.49.0 From 4fd6be3c9217583474739c366185dbaf5527b437 Mon Sep 17 00:00:00 2001 From: Tim Culverhouse Date: Sat, 17 May 2025 07:16:05 -0800 Subject: [PATCH 2/2] ingester: ingest sh.tangled.repo records from jetstream Ingest sh.tangled.repo records off the jetstream. This enables third-party clients to create tangled.sh git repositories on a user's PDS and tangled to pick up the record and create the repo on the knot, and in the appview db. --- appview/ingester.go | 179 ++++++++++++++++++++++++++++++++++++++++- appview/state/repo.go | 8 +- appview/state/state.go | 2 +- 3 files changed, 181 insertions(+), 8 deletions(-) diff --git a/appview/ingester.go b/appview/ingester.go index 6537d2d23c2c..bdffbf28b46a 100644 --- a/appview/ingester.go +++ b/appview/ingester.go @@ -5,20 +5,23 @@ import ( "encoding/json" "fmt" "log" + "net/http" "time" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/jetstream/pkg/models" + securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-git/go-git/v5/plumbing" "github.com/ipfs/go-cid" "tangled.sh/tangled.sh/core/api/tangled" "tangled.sh/tangled.sh/core/appview/db" + "tangled.sh/tangled.sh/core/appview/knotclient" "tangled.sh/tangled.sh/core/rbac" ) type Ingester func(ctx context.Context, e *models.Event) error -func Ingest(d db.DbWrapper, enforcer *rbac.Enforcer) Ingester { +func Ingest(d db.DbWrapper, enforcer *rbac.Enforcer, dev bool) Ingester { return func(ctx context.Context, e *models.Event) error { var err error defer func() { @@ -44,6 +47,8 @@ func Ingest(d db.DbWrapper, enforcer *rbac.Enforcer) Ingester { ingestArtifact(&d, e, enforcer) case tangled.ActorProfileNSID: ingestProfile(&d, e) + case tangled.RepoNSID: + ingestRepo(ctx, &d, e, enforcer, dev) } return err @@ -285,3 +290,175 @@ func ingestProfile(d *db.DbWrapper, e *models.Event) error { return nil } + +func ingestRepo( + ctx context.Context, + d *db.DbWrapper, + e *models.Event, + enforcer *rbac.Enforcer, + dev bool, +) error { + did := e.Did + + raw := json.RawMessage(e.Commit.Record) + record := tangled.Repo{} + if err := json.Unmarshal(raw, &record); err != nil { + return err + } + + switch e.Commit.Operation { + case models.CommitOperationCreate, models.CommitOperationUpdate: + var description string + if record.Description != nil { + description = *record.Description + } + + created, err := time.Parse(time.RFC3339, record.CreatedAt) + if err != nil { + return fmt.Errorf("invalid createdAt format: %q", record.CreatedAt) + } + + repo := &db.Repo{ + Did: did, + Name: record.Name, + Knot: record.Knot, + Rkey: e.Commit.RKey, + Created: created, + AtUri: fmt.Sprintf("at://%s/%s/%s", did, e.Kind, e.Commit.RKey), + Description: description, + } + + if err := db.ValidateRepo(d, repo); err != nil { + return err + } + + ok, err := enforcer.E.Enforce(did, record.Knot, record.Knot, "repo:create") + if err != nil || !ok { + return fmt.Errorf("insufficient permissions to create a repo on this knot") + } + + secret, err := db.GetRegistrationKey(d, record.Knot) + if err != nil { + return err + } + + client, err := knotclient.NewSignedClient(record.Knot, secret, dev) + if err != nil { + return err + } + + // NOTE: The sh.tangled.repo lexicon has no branch field. For + // repos we ingest via the jetstream we always default to "main" + defaultBranch := "main" + + resp, err := client.NewRepo(did, record.Name, defaultBranch) + if err != nil { + return err + } + + switch resp.StatusCode { + case http.StatusConflict: + return fmt.Errorf("repo with that name already exists") + case http.StatusInternalServerError: + return fmt.Errorf("failed to create repo") + case http.StatusNoContent: + // continue + } + + ddb, ok := d.Execer.(*db.DB) + if !ok { + return fmt.Errorf("failed to index profile record, invalid db cast") + } + + tx, err := ddb.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to start transaction") + } + + defer func() { + tx.Rollback() + err = enforcer.E.LoadPolicy() + if err != nil { + log.Println("failed to rollback policies") + } + }() + + if err := db.AddRepo(tx, repo); err != nil { + return err + } + + // acls + p, _ := securejoin.SecureJoin(did, record.Name) + if err := enforcer.AddRepo(did, record.Knot, p); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + if err := enforcer.E.SavePolicy(); err != nil { + return err + } + case models.CommitOperationDelete: + secret, err := db.GetRegistrationKey(d, record.Knot) + if err != nil { + return err + } + + client, err := knotclient.NewSignedClient(record.Knot, secret, dev) + if err != nil { + return err + } + + // We don't do anything with the response from the knot. This is + // a fire and forget + _, _ = client.RemoveRepo(did, record.Name) + + ddb, ok := d.Execer.(*db.DB) + if !ok { + return fmt.Errorf("failed to index profile record, invalid db cast") + } + + tx, err := ddb.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to start transaction") + } + defer func() { + tx.Rollback() + err = enforcer.E.LoadPolicy() + if err != nil { + log.Println("failed to rollback policies") + } + }() + + p, _ := securejoin.SecureJoin(did, record.Name) + collaborators, err := enforcer.E.GetImplicitUsersForResourceByDomain(p, record.Knot) + if err != nil { + return err + } + + for _, c := range collaborators { + cDid := c[0] + enforcer.RemoveCollaborator(cDid, record.Knot, record.Name) + } + + if err := enforcer.RemoveRepo(did, record.Knot, p); err != nil { + return err + } + + if err := db.RemoveRepo(d, did, record.Name); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + if err := enforcer.E.SavePolicy(); err != nil { + return err + } + } + + return nil +} diff --git a/appview/state/repo.go b/appview/state/repo.go index 1e7b3fe9ad24..0cb0ba9247b0 100644 --- a/appview/state/repo.go +++ b/appview/state/repo.go @@ -283,7 +283,6 @@ func (s *State) RepoDescription(w http.ResponseWriter, r *http.Request) { }, }, }) - if err != nil { log.Println("failed to perferom update-description query", err) // failed to get record @@ -716,7 +715,6 @@ func (s *State) AddCollaborator(w http.ResponseWriter, r *http.Request) { } w.Write([]byte(fmt.Sprint("added collaborator: ", collaboratorIdent.Handle.String()))) - } func (s *State) DeleteRepo(w http.ResponseWriter, r *http.Request) { @@ -1167,7 +1165,6 @@ func (s *State) RepoSingleIssue(w http.ResponseWriter, r *http.Request) { IssueOwnerHandle: issueOwnerIdent.Handle.String(), DidHandleMap: didHandleMap, }) - } func (s *State) CloseIssue(w http.ResponseWriter, r *http.Request) { @@ -1223,7 +1220,6 @@ func (s *State) CloseIssue(w http.ResponseWriter, r *http.Request) { }, }, }) - if err != nil { log.Println("failed to update issue state", err) s.pages.Notice(w, "issue-action", "Failed to close issue. Try again later.") @@ -1564,7 +1560,6 @@ func (s *State) EditIssueComment(w http.ResponseWriter, r *http.Request) { return } - } func (s *State) DeleteIssueComment(w http.ResponseWriter, r *http.Request) { @@ -1938,7 +1933,8 @@ func (s *State) ForkRepo(w http.ResponseWriter, r *http.Request) { CreatedAt: createdAt, Owner: user.Did, Source: &sourceAt, - }}, + }, + }, }) if err != nil { log.Printf("failed to create record: %s", err) diff --git a/appview/state/state.go b/appview/state/state.go index 8e5b17df2c22..8360c3602aae 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -76,7 +76,7 @@ func Make(config *appview.Config) (*State, error) { if err != nil { return nil, fmt.Errorf("failed to create jetstream client: %w", err) } - err = jc.StartJetstream(context.Background(), appview.Ingest(wrapper, enforcer)) + err = jc.StartJetstream(context.Background(), appview.Ingest(wrapper, enforcer, config.Core.Dev)) if err != nil { return nil, fmt.Errorf("failed to start jetstream watcher: %w", err) } -- 2.49.0