From 7a04654cd936cf7e6312879c98888bf077fcee70 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 13:55:56 +0900 Subject: [PATCH] appview,knotserver: support immutable nix flakeref link header Change-Id: uxntvwzvsmqpkmonkspxmzqpxktqrusw This will allow users to directly request archive from the knot without using xrpc. Xrpc doesn't fit here because it strips out the http headers which might include valuable metadata like download filename or immutable link. - implement archive on knot as `/{owner}/{repo}/archive/{ref}` endpoint - appview proxies the request to knot on `/archive` like it is doing for git http endpoints. if knot version isn't compatible, it will fallback to legacy xrpc endpoint. - rename the `git_http.go` file to generalized `proxy_knot.go` filaname xrpc method `sh.tangled.repo.archive` will be deprecated in future added `go-version` depenedency to make version constraints Close: Signed-off-by: Seongmin Lee --- appview/repo/archive.go | 49 -------------- appview/repo/router.go | 4 -- appview/state/{git_http.go => proxy_knot.go} | 71 ++++++++++++++++++++ appview/state/router.go | 4 +- go.mod | 1 + go.sum | 2 + knotserver/archive.go | 69 +++++++++++++++++++ knotserver/git/git.go | 4 ++ knotserver/router.go | 2 + nix/gomod2nix.toml | 3 + 10 files changed, 155 insertions(+), 54 deletions(-) delete mode 100644 appview/repo/archive.go rename appview/state/{git_http.go => proxy_knot.go} (51%) create mode 100644 knotserver/archive.go diff --git a/appview/repo/archive.go b/appview/repo/archive.go deleted file mode 100644 index eac24201..00000000 --- a/appview/repo/archive.go +++ /dev/null @@ -1,49 +0,0 @@ -package repo - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - "tangled.org/core/api/tangled" - xrpcclient "tangled.org/core/appview/xrpcclient" - - indigoxrpc "github.com/bluesky-social/indigo/xrpc" - "github.com/go-chi/chi/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { - l := rp.logger.With("handler", "DownloadArchive") - ref := chi.URLParam(r, "ref") - ref, _ = url.PathUnescape(ref) - f, err := rp.repoResolver.Resolve(r) - if err != nil { - l.Error("failed to get repo and knot", "err", err) - return - } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - didSlashRepo := f.DidSlashRepo() - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) - rp.pages.Error503(w) - return - } - // Set headers for file download, just pass along whatever the knot specifies - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) - // Write the archive data directly - w.Write(archiveBytes) -} diff --git a/appview/repo/router.go b/appview/repo/router.go index 1667ae58..6e097e5d 100644 --- a/appview/repo/router.go +++ b/appview/repo/router.go @@ -40,10 +40,6 @@ func (rp *Repo) Router(mw *middleware.Middleware) http.Handler { r.Get("/blob/{ref}/*", rp.Blob) r.Get("/raw/{ref}/*", rp.RepoBlobRaw) - // intentionally doesn't use /* as this isn't - // a file path - r.Get("/archive/{ref}", rp.DownloadArchive) - r.Route("/fork", func(r chi.Router) { r.Use(middleware.AuthMiddleware(rp.oauth)) r.Get("/", rp.ForkRepo) diff --git a/appview/state/git_http.go b/appview/state/proxy_knot.go similarity index 51% rename from appview/state/git_http.go rename to appview/state/proxy_knot.go index 36ce3e8d..d13476f0 100644 --- a/appview/state/git_http.go +++ b/appview/state/proxy_knot.go @@ -5,10 +5,16 @@ import ( "io" "maps" "net/http" + "strings" "github.com/bluesky-social/indigo/atproto/identity" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/hashicorp/go-version" + "tangled.org/core/api/tangled" "tangled.org/core/appview/models" + xrpcclient "tangled.org/core/appview/xrpcclient" ) func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) { @@ -59,6 +65,71 @@ func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) { s.proxyRequest(w, r, targetURL) } +var knotVersionDownloadArchiveConstraint = version.MustConstraints(version.NewConstraint(">= 1.12.0-alpha")) + +func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) { + l := s.logger.With("handler", "DownloadArchive") + ref := chi.URLParam(r, "ref") + + user, ok := r.Context().Value("resolvedId").(identity.Identity) + if !ok { + l.Error("failed to resolve user") + http.Error(w, "failed to resolve user", http.StatusInternalServerError) + return + } + repo := r.Context().Value("repo").(*models.Repo) + + scheme := "https" + if s.config.Core.Dev { + scheme = "http" + } + + host := fmt.Sprintf("%s://%s", scheme, repo.Knot) + xrpcc := &indigoxrpc.Client{ + Host: host, + } + l = l.With("knot", repo.Knot) + + isCompatible := func() bool { + out, err := tangled.KnotVersion(r.Context(), xrpcc) + if err != nil { + l.Warn("failed to get knot version", "err", err) + return false + } + + v, err := version.NewVersion(out.Version) + if err != nil { + l.Warn("failed to parse knot version", "version", out.Version, "err", err) + return false + } + + if !knotVersionDownloadArchiveConstraint.Check(v) { + l.Warn("knot version incompatible.", "version", v) + return false + } + return true + }() + l.Debug("knot compatibility check", "isCompatible", isCompatible) + if isCompatible { + targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref) + s.proxyRequest(w, r, targetURL) + } else { + l.Debug("requesting xrpc/sh.tangled.repo.archive") + archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo.DidSlashRepo()) + if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { + l.Error("failed to call XRPC repo.archive", "err", xrpcerr) + s.pages.Error503(w) + return + } + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", repo.Name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) + w.Write(archiveBytes) + } +} + func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) { client := &http.Client{} diff --git a/appview/state/router.go b/appview/state/router.go index 968ed403..2f71a895 100644 --- a/appview/state/router.go +++ b/appview/state/router.go @@ -103,7 +103,9 @@ func (s *State) UserRouter(mw *middleware.Middleware) http.Handler { r.Get("/info/refs", s.InfoRefs) r.Post("/git-upload-pack", s.UploadPack) r.Post("/git-receive-pack", s.ReceivePack) - + // intentionally doesn't use /* as this isn't + // a file path + r.Get("/archive/{ref}", s.DownloadArchive) }) }) diff --git a/go.mod b/go.mod index 29ef27c5..66f1d594 100644 --- a/go.mod +++ b/go.mod @@ -131,6 +131,7 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-7 // indirect diff --git a/go.sum b/go.sum index fdb6ccc0..b2a7c34c 100644 --- a/go.sum +++ b/go.sum @@ -264,6 +264,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/knotserver/archive.go b/knotserver/archive.go new file mode 100644 index 00000000..9a5f48b1 --- /dev/null +++ b/knotserver/archive.go @@ -0,0 +1,69 @@ +package knotserver + +import ( + "compress/gzip" + "fmt" + "net/http" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/knotserver/git" +) + +func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) { + var ( + did = chi.URLParam(r, "did") + name = chi.URLParam(r, "name") + ref = chi.URLParam(r, "ref") + ) + repo, err := securejoin.SecureJoin(did, name) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + gr, err := git.Open(repoPath, ref) + + immutableLink := fmt.Sprintf( + "https://%s/%s/%s/archive/%s", + h.c.Server.Hostname, + did, + name, + gr.Hash(), + ) + + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) + + gw := gzip.NewWriter(w) + defer gw.Close() + + err = gr.WriteTar(gw, "") + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("writing tar file", "error", err) + return + } + + err = gw.Flush() + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("flushing", "error", err.Error()) + return + } +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go index 6fbb3329..5d888852 100644 --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -76,6 +76,10 @@ func PlainOpen(path string) (*GitRepo, error) { return &g, nil } +func (g *GitRepo) Hash() plumbing.Hash { + return g.h +} + // re-open a repository and update references func (g *GitRepo) Refresh() error { refreshed, err := PlainOpen(g.path) diff --git a/knotserver/router.go b/knotserver/router.go index 482f932d..2aef216d 100644 --- a/knotserver/router.go +++ b/knotserver/router.go @@ -84,6 +84,8 @@ func (h *Knot) Router() http.Handler { r.Get("/info/refs", h.InfoRefs) r.Post("/git-upload-pack", h.UploadPack) r.Post("/git-receive-pack", h.ReceivePack) + // convenience routes + r.Get("/archive/{ref}", h.Archive) }) }) diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index 1f3bde66..f46af247 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -304,6 +304,9 @@ schema = 3 [mod."github.com/hashicorp/go-sockaddr"] version = "v1.0.7" hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs=" + [mod."github.com/hashicorp/go-version"] + version = "v1.8.0" + hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8=" [mod."github.com/hashicorp/golang-lru"] version = "v1.0.2" hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48=" -- 2.43.0 From 4eea36c260e4e44d2a984905c7b49d9edf21f506 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 15:54:15 +0900 Subject: [PATCH] lexicons,knotserver: remove `/xrpc/sh.tangled.repo.archive` endpoint Change-Id: xuplvooxylpkzuvqzoryvlpxuzxpvmpn replaced by `/{did}/{reponame}/archive/{ref}`. The lexicon definitions will be removed later. Signed-off-by: Seongmin Lee --- knotserver/xrpc/repo_archive.go | 81 --------------------------------- knotserver/xrpc/xrpc.go | 1 - lexicons/repo/archive.json | 1 + 3 files changed, 1 insertion(+), 82 deletions(-) delete mode 100644 knotserver/xrpc/repo_archive.go diff --git a/knotserver/xrpc/repo_archive.go b/knotserver/xrpc/repo_archive.go deleted file mode 100644 index 42fe498e..00000000 --- a/knotserver/xrpc/repo_archive.go +++ /dev/null @@ -1,81 +0,0 @@ -package xrpc - -import ( - "compress/gzip" - "fmt" - "net/http" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - - "tangled.org/core/knotserver/git" - xrpcerr "tangled.org/core/xrpc/errors" -) - -func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { - repo := r.URL.Query().Get("repo") - repoPath, err := x.parseRepoParam(repo) - if err != nil { - writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest) - return - } - - ref := r.URL.Query().Get("ref") - // ref can be empty (git.Open handles this) - - format := r.URL.Query().Get("format") - if format == "" { - format = "tar.gz" // default - } - - prefix := r.URL.Query().Get("prefix") - - if format != "tar.gz" { - writeError(w, xrpcerr.NewXrpcError( - xrpcerr.WithTag("InvalidRequest"), - xrpcerr.WithMessage("only tar.gz format is supported"), - ), http.StatusBadRequest) - return - } - - gr, err := git.Open(repoPath, ref) - if err != nil { - writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound) - return - } - - repoParts := strings.Split(repo, "/") - repoName := repoParts[len(repoParts)-1] - - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - - var archivePrefix string - if prefix != "" { - archivePrefix = prefix - } else { - archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename) - } - - filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - - gw := gzip.NewWriter(w) - defer gw.Close() - - err = gr.WriteTar(gw, archivePrefix) - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("writing tar file", "error", err.Error()) - return - } - - err = gw.Flush() - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("flushing", "error", err.Error()) - return - } -} diff --git a/knotserver/xrpc/xrpc.go b/knotserver/xrpc/xrpc.go index c110356f..50615e80 100644 --- a/knotserver/xrpc/xrpc.go +++ b/knotserver/xrpc/xrpc.go @@ -64,7 +64,6 @@ func (x *Xrpc) Router() http.Handler { r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare) r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch) r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch) - r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive) r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages) // knot query endpoints (no auth required) diff --git a/lexicons/repo/archive.json b/lexicons/repo/archive.json index 05e313d1..5e9e038c 100644 --- a/lexicons/repo/archive.json +++ b/lexicons/repo/archive.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "deprecated. use `/{did}/{reponame}/archive/{ref} endpoint instead", "parameters": { "type": "params", "required": ["repo", "ref"], -- 2.43.0