appview,knotserver: support lockable http tarball protocol #1001

+103 -19

3 changed files

Diff round #1

expand all

unified split

appview

repo

archive.go

knotserver

git

git.go

xrpc

repo_archive.go

+64 -19

appview/repo/archive.go

··· 2 2 3 3 import ( 4 4 "fmt" 5 + "io" 5 6 "net/http" 6 7 "net/url" 7 8 "strings" 8 9 9 - "tangled.org/core/api/tangled" 10 - xrpcclient "tangled.org/core/appview/xrpcclient" 11 - 12 - indigoxrpc "github.com/bluesky-social/indigo/xrpc" 13 10 "github.com/go-chi/chi/v5" 14 - "github.com/go-git/go-git/v5/plumbing" 15 11 ) 16 12 17 13 func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { ··· 29 25 scheme = "https" 30 26 } 31 27 host := fmt.Sprintf("%s://%s", scheme, f.Knot) 32 - xrpcc := &indigoxrpc.Client{ 33 - Host: host, 34 - } 35 28 didSlashRepo := f.DidSlashRepo() 36 - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo) 37 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 38 - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) 29 + 30 + // build the xrpc url 31 + u, err := url.Parse(host) 32 + if err != nil { 33 + l.Error("failed to parse host URL", "err", err) 39 34 rp.pages.Error503(w) 40 35 return 41 36 } 42 - // Set headers for file download, just pass along whatever the knot specifies 43 - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") 44 - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) 45 - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) 46 - w.Header().Set("Content-Type", "application/gzip") 47 - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) 48 - // Write the archive data directly 49 - w.Write(archiveBytes) 37 + 38 + u.Path = "/xrpc/sh.tangled.repo.archive" 39 + query := url.Values{} 40 + query.Set("format", "tar.gz") 41 + query.Set("prefix", r.URL.Query().Get("prefix")) 42 + query.Set("ref", ref) 43 + query.Set("repo", didSlashRepo) 44 + u.RawQuery = query.Encode() 45 + 46 + xrpcURL := u.String() 47 + 48 + // make the get request 49 + resp, err := http.Get(xrpcURL) 50 + if err != nil { 51 + l.Error("failed to call XRPC repo.archive", "err", err) 52 + rp.pages.Error503(w) 53 + return 54 + } 55 + 56 + // pass through headers from upstream response 57 + if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" { 58 + w.Header().Set("Content-Disposition", contentDisposition) 59 + } 60 + if contentType := resp.Header.Get("Content-Type"); contentType != "" { 61 + w.Header().Set("Content-Type", contentType) 62 + } 63 + if contentLength := resp.Header.Get("Content-Length"); contentLength != "" { 64 + w.Header().Set("Content-Length", contentLength) 65 + } 66 + if link := resp.Header.Get("Link"); link != "" { 67 + if resolvedRef, err := extractImmutableLink(link); err == nil { 68 + newLink := fmt.Sprintf("<%s/%s/archive/%s.tar.gz>; rel=\"immutable\"", 69 + rp.config.Core.AppviewHost, f.DidSlashRepo(), resolvedRef) 70 + w.Header().Set("Link", newLink) 71 + } 72 + } 73 + 74 + // stream the archive data directly 75 + if _, err := io.Copy(w, resp.Body); err != nil { 76 + l.Error("failed to write response", "err", err) 77 + } 78 + } 79 + 80 + func extractImmutableLink(linkHeader string) (string, error) { 81 + trimmed := strings.TrimPrefix(linkHeader, "<") 82 + trimmed = strings.TrimSuffix(trimmed, ">; rel=\"immutable\"") 83 + 84 + parsedLink, err := url.Parse(trimmed) 85 + if err != nil { 86 + return "", err 87 + } 88 + 89 + resolvedRef := parsedLink.Query().Get("ref") 90 + if resolvedRef == "" { 91 + return "", fmt.Errorf("no ref found in link header") 92 + } 93 + 94 + return resolvedRef, nil 50 95 }

knotserver/git/git.go

··· 76 76 return &g, nil 77 77 } 78 78 79 + func (g *GitRepo) Hash() plumbing.Hash { 80 + return g.h 81 + } 82 + 79 83 // re-open a repository and update references 80 84 func (g *GitRepo) Refresh() error { 81 85 refreshed, err := PlainOpen(g.path)

+35

knotserver/xrpc/repo_archive.go

··· 4 4 "compress/gzip" 5 5 "fmt" 6 6 "net/http" 7 + "net/url" 7 8 "strings" 8 9 9 10 "github.com/go-git/go-git/v5/plumbing" 10 11 12 + "tangled.org/core/api/tangled" 11 13 "tangled.org/core/knotserver/git" 12 14 xrpcerr "tangled.org/core/xrpc/errors" 13 15 ) ··· 47 49 repoParts := strings.Split(repo, "/") 48 50 repoName := repoParts[len(repoParts)-1] 49 51 52 + immutableLink, err := x.buildImmutableLink(repo, format, gr.Hash().String(), prefix) 53 + if err != nil { 54 + x.Logger.Error( 55 + "failed to build immutable link", 56 + "err", err.Error(), 57 + "repo", repo, 58 + "format", format, 59 + "ref", gr.Hash().String(), 60 + "prefix", prefix, 61 + ) 62 + } 63 + 50 64 safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") 51 65 52 66 var archivePrefix string ··· 59 73 filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) 60 74 w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) 61 75 w.Header().Set("Content-Type", "application/gzip") 76 + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) 62 77 63 78 gw := gzip.NewWriter(w) 64 79 defer gw.Close() ··· 79 94 return 80 95 } 81 96 } 97 + 98 + func (x *Xrpc) buildImmutableLink(repo string, format string, ref string, prefix string) (string, error) { 99 + scheme := "https" 100 + if x.Config.Server.Dev { 101 + scheme = "http" 102 + } 103 + 104 + u, err := url.Parse(scheme + "://" + x.Config.Server.Hostname + "/xrpc/" + tangled.RepoArchiveNSID) 105 + if err != nil { 106 + return "", err 107 + } 108 + 109 + params := url.Values{} 110 + params.Set("repo", repo) 111 + params.Set("format", format) 112 + params.Set("ref", ref) 113 + params.Set("prefix", prefix) 114 + 115 + return fmt.Sprintf("%s?%s", u.String(), params.Encode()), nil 116 + }

History

2 rounds 2 comments

oppi.li submitted #1 4w

interdiff

1 commit

expand

1ad3ce54

appview,knotserver: support lockable http tarball protocol

the lockable http tarball protocol is meant to serve tarball flakes, by
emitting a stable `Link` header:

Link: <flakeref>; rel="immutable"

this patch now supports the new header in two places, on the appview, at
the `/archive/<ref>.tar.gz` endpoint:

λ nix flake metadata -v --refresh --no-write-lock-file 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz'
unpacking 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz' into the Git cache...
warning: not writing modified lock file of flake 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz':
• Added input 'nixpkgs':
'github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D' (2026-01-19)
Resolved URL: http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz
Locked URL: http://127.0.0.1:3000/did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14/archive/a63d945ae97b84812e394207f3cc80f6525c2082.tar.gz?narHash=sha256-IdKT88RIWvWrgQFx6c%2BX3cC7JFene%2BQI9yo2rKSGoA4%3D
Path: /nix/store/0k9pv83f0qn5cm0qy82j51plryk7szx7-source
Fingerprint: 9512ee4857b31a76c1112f05161bda5280d8596b866c4f78986c6c01c1d2f419
Inputs:
└───nixpkgs: github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D (2026-01-19 00:39:23)

and on the knotserver, when using the `/xrpc/sh.tangled.repo.archive`
endpoint:

λ nix flake metadata -v --refresh --no-write-lock-file "http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did%3Aplc%3Aqfpnj4og54vl56wngdriaxug%2Frepo-19-01-26-08-04-14"
unpacking 'http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14' into the Git cache...
warning: not writing modified lock file of flake 'http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14':
• Added input 'nixpkgs':
'github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D' (2026-01-19)
Resolved URL: http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14
Locked URL: http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&narHash=sha256-IdKT88RIWvWrgQFx6c%2BX3cC7JFene%2BQI9yo2rKSGoA4%3D&prefix=&ref=a63d945ae97b84812e394207f3cc80f6525c2082&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14
Path: /nix/store/0k9pv83f0qn5cm0qy82j51plryk7szx7-source
Fingerprint: 9512ee4857b31a76c1112f05161bda5280d8596b866c4f78986c6c01c1d2f419
Inputs:
└───nixpkgs: github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D (2026-01-19 00:39:23)

note that the "Resolved URL" includes a hash of the commit.

Co-authored-by: Seongmin Lee <git@boltless.me>
Signed-off-by: oppiliappan <me@oppi.li>

expand 1 comment

oppi.li 4w

adding @boltless.me as a co-author! largely inherits the link calculation logic from #741.

pull request successfully merged

oppi.li submitted #0 4w

diff

1 commit

expand

da304bf5

appview,knotserver: support lockable http tarball protocol

3/3 success

expand

expand 1 comment

oppi.li 4w

the difference between this PR and https://tangled.org/tangled.org/core/pulls/741 is that this one does not deprecate the sh.tangled.repo.archive endpoint.