From d8b6a8f553e5c0640a5214b58bcd3b7011a5e50b Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 13:55:56 +0900 Subject: [PATCH] appview,knotserver: support immutable nix flakeref link header Change-Id: uxntvwzvsmqpkmonkspxmzqpxktqrusw This will allow users to directly request archive from the knot without using xrpc. Xrpc doesn't fit here because it strips out the http headers which might include valuable metadata like download filename or immutable link. - implement archive on knot as `/{owner}/{repo}/archive/{ref}` endpoint - appview proxies the request to knot on `/archive` like it is doing for git http endpoints. if knot version isn't compatible, it will fallback to legacy xrpc endpoint. - rename the `git_http.go` file to generalized `proxy_knot.go` filaname xrpc method `sh.tangled.repo.archive` will be deprecated in future added `go-version` depenedency to make version constraints Close: Signed-off-by: Seongmin Lee --- appview/repo/archive.go | 49 -------------- appview/repo/router.go | 4 -- appview/state/{git_http.go => proxy_knot.go} | 71 ++++++++++++++++++++ appview/state/router.go | 4 +- go.mod | 1 + go.sum | 2 + knotserver/archive.go | 69 +++++++++++++++++++ knotserver/git/git.go | 4 ++ knotserver/router.go | 2 + nix/gomod2nix.toml | 3 + 10 files changed, 155 insertions(+), 54 deletions(-) delete mode 100644 appview/repo/archive.go rename appview/state/{git_http.go => proxy_knot.go} (55%) create mode 100644 knotserver/archive.go diff --git a/appview/repo/archive.go b/appview/repo/archive.go deleted file mode 100644 index eac24201..00000000 --- a/appview/repo/archive.go +++ /dev/null @@ -1,49 +0,0 @@ -package repo - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - "tangled.org/core/api/tangled" - xrpcclient "tangled.org/core/appview/xrpcclient" - - indigoxrpc "github.com/bluesky-social/indigo/xrpc" - "github.com/go-chi/chi/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { - l := rp.logger.With("handler", "DownloadArchive") - ref := chi.URLParam(r, "ref") - ref, _ = url.PathUnescape(ref) - f, err := rp.repoResolver.Resolve(r) - if err != nil { - l.Error("failed to get repo and knot", "err", err) - return - } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - didSlashRepo := f.DidSlashRepo() - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) - rp.pages.Error503(w) - return - } - // Set headers for file download, just pass along whatever the knot specifies - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) - // Write the archive data directly - w.Write(archiveBytes) -} diff --git a/appview/repo/router.go b/appview/repo/router.go index 1667ae58..6e097e5d 100644 --- a/appview/repo/router.go +++ b/appview/repo/router.go @@ -40,10 +40,6 @@ func (rp *Repo) Router(mw *middleware.Middleware) http.Handler { r.Get("/blob/{ref}/*", rp.Blob) r.Get("/raw/{ref}/*", rp.RepoBlobRaw) - // intentionally doesn't use /* as this isn't - // a file path - r.Get("/archive/{ref}", rp.DownloadArchive) - r.Route("/fork", func(r chi.Router) { r.Use(middleware.AuthMiddleware(rp.oauth)) r.Get("/", rp.ForkRepo) diff --git a/appview/state/git_http.go b/appview/state/proxy_knot.go similarity index 55% rename from appview/state/git_http.go rename to appview/state/proxy_knot.go index fd3da36e..3ebd6bc4 100644 --- a/appview/state/git_http.go +++ b/appview/state/proxy_knot.go @@ -5,10 +5,16 @@ import ( "io" "maps" "net/http" + "strings" "github.com/bluesky-social/indigo/atproto/identity" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/hashicorp/go-version" + "tangled.org/core/api/tangled" "tangled.org/core/appview/models" + xrpcclient "tangled.org/core/appview/xrpcclient" ) func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) { @@ -76,6 +82,71 @@ func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) { s.proxyRequest(w, r, targetURL) } +var knotVersionDownloadArchiveConstraint = version.MustConstraints(version.NewConstraint(">= 1.12.0-alpha")) + +func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) { + l := s.logger.With("handler", "DownloadArchive") + ref := chi.URLParam(r, "ref") + + user, ok := r.Context().Value("resolvedId").(identity.Identity) + if !ok { + l.Error("failed to resolve user") + http.Error(w, "failed to resolve user", http.StatusInternalServerError) + return + } + repo := r.Context().Value("repo").(*models.Repo) + + scheme := "https" + if s.config.Core.Dev { + scheme = "http" + } + + host := fmt.Sprintf("%s://%s", scheme, repo.Knot) + xrpcc := &indigoxrpc.Client{ + Host: host, + } + l = l.With("knot", repo.Knot) + + isCompatible := func() bool { + out, err := tangled.KnotVersion(r.Context(), xrpcc) + if err != nil { + l.Warn("failed to get knot version", "err", err) + return false + } + + v, err := version.NewVersion(out.Version) + if err != nil { + l.Warn("failed to parse knot version", "version", out.Version, "err", err) + return false + } + + if !knotVersionDownloadArchiveConstraint.Check(v) { + l.Warn("knot version incompatible.", "version", v) + return false + } + return true + }() + l.Debug("knot compatibility check", "isCompatible", isCompatible) + if isCompatible { + targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref) + s.proxyRequest(w, r, targetURL) + } else { + l.Debug("requesting xrpc/sh.tangled.repo.archive") + archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo.DidSlashRepo()) + if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { + l.Error("failed to call XRPC repo.archive", "err", xrpcerr) + s.pages.Error503(w) + return + } + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", repo.Name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) + w.Write(archiveBytes) + } +} + func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) { client := &http.Client{} diff --git a/appview/state/router.go b/appview/state/router.go index b11f12c2..45303f2b 100644 --- a/appview/state/router.go +++ b/appview/state/router.go @@ -104,7 +104,9 @@ func (s *State) UserRouter(mw *middleware.Middleware) http.Handler { r.Post("/git-upload-archive", s.UploadArchive) r.Post("/git-upload-pack", s.UploadPack) r.Post("/git-receive-pack", s.ReceivePack) - + // intentionally doesn't use /* as this isn't + // a file path + r.Get("/archive/{ref}", s.DownloadArchive) }) }) diff --git a/go.mod b/go.mod index d6dba6ce..c17d74e2 100644 --- a/go.mod +++ b/go.mod @@ -131,6 +131,7 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-7 // indirect diff --git a/go.sum b/go.sum index 993532bb..865e87c7 100644 --- a/go.sum +++ b/go.sum @@ -264,6 +264,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/knotserver/archive.go b/knotserver/archive.go new file mode 100644 index 00000000..9a5f48b1 --- /dev/null +++ b/knotserver/archive.go @@ -0,0 +1,69 @@ +package knotserver + +import ( + "compress/gzip" + "fmt" + "net/http" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/knotserver/git" +) + +func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) { + var ( + did = chi.URLParam(r, "did") + name = chi.URLParam(r, "name") + ref = chi.URLParam(r, "ref") + ) + repo, err := securejoin.SecureJoin(did, name) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + gr, err := git.Open(repoPath, ref) + + immutableLink := fmt.Sprintf( + "https://%s/%s/%s/archive/%s", + h.c.Server.Hostname, + did, + name, + gr.Hash(), + ) + + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) + + gw := gzip.NewWriter(w) + defer gw.Close() + + err = gr.WriteTar(gw, "") + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("writing tar file", "error", err) + return + } + + err = gw.Flush() + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("flushing", "error", err.Error()) + return + } +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go index 6fbb3329..5d888852 100644 --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -76,6 +76,10 @@ func PlainOpen(path string) (*GitRepo, error) { return &g, nil } +func (g *GitRepo) Hash() plumbing.Hash { + return g.h +} + // re-open a repository and update references func (g *GitRepo) Refresh() error { refreshed, err := PlainOpen(g.path) diff --git a/knotserver/router.go b/knotserver/router.go index 3b27d832..fdfbd201 100644 --- a/knotserver/router.go +++ b/knotserver/router.go @@ -85,6 +85,8 @@ func (h *Knot) Router() http.Handler { r.Post("/git-upload-archive", h.UploadArchive) r.Post("/git-upload-pack", h.UploadPack) r.Post("/git-receive-pack", h.ReceivePack) + // convenience routes + r.Get("/archive/{ref}", h.Archive) }) }) diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index 4c9fc051..c8ae7e29 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -304,6 +304,9 @@ schema = 3 [mod."github.com/hashicorp/go-sockaddr"] version = "v1.0.7" hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs=" + [mod."github.com/hashicorp/go-version"] + version = "v1.8.0" + hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8=" [mod."github.com/hashicorp/golang-lru"] version = "v1.0.2" hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48=" -- 2.43.0 From 1b2756045fdfe45b4ee6682ab53edb423875fbbb Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 15:54:15 +0900 Subject: [PATCH] lexicons,knotserver: remove `/xrpc/sh.tangled.repo.archive` endpoint Change-Id: xuplvooxylpkzuvqzoryvlpxuzxpvmpn replaced by `/{did}/{reponame}/archive/{ref}`. The lexicon definitions will be removed later. Signed-off-by: Seongmin Lee --- knotserver/xrpc/repo_archive.go | 81 --------------------------------- knotserver/xrpc/xrpc.go | 1 - lexicons/repo/archive.json | 1 + 3 files changed, 1 insertion(+), 82 deletions(-) delete mode 100644 knotserver/xrpc/repo_archive.go diff --git a/knotserver/xrpc/repo_archive.go b/knotserver/xrpc/repo_archive.go deleted file mode 100644 index 42fe498e..00000000 --- a/knotserver/xrpc/repo_archive.go +++ /dev/null @@ -1,81 +0,0 @@ -package xrpc - -import ( - "compress/gzip" - "fmt" - "net/http" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - - "tangled.org/core/knotserver/git" - xrpcerr "tangled.org/core/xrpc/errors" -) - -func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { - repo := r.URL.Query().Get("repo") - repoPath, err := x.parseRepoParam(repo) - if err != nil { - writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest) - return - } - - ref := r.URL.Query().Get("ref") - // ref can be empty (git.Open handles this) - - format := r.URL.Query().Get("format") - if format == "" { - format = "tar.gz" // default - } - - prefix := r.URL.Query().Get("prefix") - - if format != "tar.gz" { - writeError(w, xrpcerr.NewXrpcError( - xrpcerr.WithTag("InvalidRequest"), - xrpcerr.WithMessage("only tar.gz format is supported"), - ), http.StatusBadRequest) - return - } - - gr, err := git.Open(repoPath, ref) - if err != nil { - writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound) - return - } - - repoParts := strings.Split(repo, "/") - repoName := repoParts[len(repoParts)-1] - - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - - var archivePrefix string - if prefix != "" { - archivePrefix = prefix - } else { - archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename) - } - - filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - - gw := gzip.NewWriter(w) - defer gw.Close() - - err = gr.WriteTar(gw, archivePrefix) - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("writing tar file", "error", err.Error()) - return - } - - err = gw.Flush() - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("flushing", "error", err.Error()) - return - } -} diff --git a/knotserver/xrpc/xrpc.go b/knotserver/xrpc/xrpc.go index c110356f..50615e80 100644 --- a/knotserver/xrpc/xrpc.go +++ b/knotserver/xrpc/xrpc.go @@ -64,7 +64,6 @@ func (x *Xrpc) Router() http.Handler { r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare) r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch) r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch) - r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive) r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages) // knot query endpoints (no auth required) diff --git a/lexicons/repo/archive.json b/lexicons/repo/archive.json index 05e313d1..5e9e038c 100644 --- a/lexicons/repo/archive.json +++ b/lexicons/repo/archive.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "deprecated. use `/{did}/{reponame}/archive/{ref} endpoint instead", "parameters": { "type": "params", "required": ["repo", "ref"], -- 2.43.0