boltless.me
Close: #231
Signed-off-by: Seongmin Lee git@boltless.me
-49
appview/repo/archive.go
-49
appview/repo/archive.go
···
1
-
package repo
2
-
3
-
import (
4
-
"fmt"
5
-
"net/http"
6
-
"net/url"
7
-
"strings"
8
-
9
-
"tangled.org/core/api/tangled"
10
-
xrpcclient "tangled.org/core/appview/xrpcclient"
11
-
12
-
indigoxrpc "github.com/bluesky-social/indigo/xrpc"
13
-
"github.com/go-chi/chi/v5"
14
-
"github.com/go-git/go-git/v5/plumbing"
15
-
)
16
-
17
-
func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) {
18
-
l := rp.logger.With("handler", "DownloadArchive")
19
-
ref := chi.URLParam(r, "ref")
20
-
ref, _ = url.PathUnescape(ref)
21
-
f, err := rp.repoResolver.Resolve(r)
22
-
if err != nil {
23
-
l.Error("failed to get repo and knot", "err", err)
24
-
return
25
-
}
26
-
scheme := "http"
27
-
if !rp.config.Core.Dev {
28
-
scheme = "https"
29
-
}
30
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
31
-
xrpcc := &indigoxrpc.Client{
32
-
Host: host,
33
-
}
34
-
didSlashRepo := f.DidSlashRepo()
35
-
archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo)
36
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
37
-
l.Error("failed to call XRPC repo.archive", "err", xrpcerr)
38
-
rp.pages.Error503(w)
39
-
return
40
-
}
41
-
// Set headers for file download, just pass along whatever the knot specifies
42
-
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
43
-
filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename)
44
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
45
-
w.Header().Set("Content-Type", "application/gzip")
46
-
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes)))
47
-
// Write the archive data directly
48
-
w.Write(archiveBytes)
49
-
}
-4
appview/repo/router.go
-4
appview/repo/router.go
···
40
40
r.Get("/blob/{ref}/*", rp.Blob)
41
41
r.Get("/raw/{ref}/*", rp.RepoBlobRaw)
42
42
43
-
// intentionally doesn't use /* as this isn't
44
-
// a file path
45
-
r.Get("/archive/{ref}", rp.DownloadArchive)
46
-
47
43
r.Route("/fork", func(r chi.Router) {
48
44
r.Use(middleware.AuthMiddleware(rp.oauth))
49
45
r.Get("/", rp.ForkRepo)
-97
appview/state/git_http.go
-97
appview/state/git_http.go
···
1
-
package state
2
-
3
-
import (
4
-
"fmt"
5
-
"io"
6
-
"maps"
7
-
"net/http"
8
-
9
-
"github.com/bluesky-social/indigo/atproto/identity"
10
-
"github.com/go-chi/chi/v5"
11
-
"tangled.org/core/appview/models"
12
-
)
13
-
14
-
func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) {
15
-
user := r.Context().Value("resolvedId").(identity.Identity)
16
-
repo := r.Context().Value("repo").(*models.Repo)
17
-
18
-
scheme := "https"
19
-
if s.config.Core.Dev {
20
-
scheme = "http"
21
-
}
22
-
23
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/info/refs?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
24
-
s.proxyRequest(w, r, targetURL)
25
-
26
-
}
27
-
28
-
func (s *State) UploadPack(w http.ResponseWriter, r *http.Request) {
29
-
user, ok := r.Context().Value("resolvedId").(identity.Identity)
30
-
if !ok {
31
-
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
32
-
return
33
-
}
34
-
repo := r.Context().Value("repo").(*models.Repo)
35
-
36
-
scheme := "https"
37
-
if s.config.Core.Dev {
38
-
scheme = "http"
39
-
}
40
-
41
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-upload-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
42
-
s.proxyRequest(w, r, targetURL)
43
-
}
44
-
45
-
func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) {
46
-
user, ok := r.Context().Value("resolvedId").(identity.Identity)
47
-
if !ok {
48
-
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
49
-
return
50
-
}
51
-
repo := r.Context().Value("repo").(*models.Repo)
52
-
53
-
scheme := "https"
54
-
if s.config.Core.Dev {
55
-
scheme = "http"
56
-
}
57
-
58
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-receive-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
59
-
s.proxyRequest(w, r, targetURL)
60
-
}
61
-
62
-
func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) {
63
-
client := &http.Client{}
64
-
65
-
// Create new request
66
-
proxyReq, err := http.NewRequest(r.Method, targetURL, r.Body)
67
-
if err != nil {
68
-
http.Error(w, err.Error(), http.StatusInternalServerError)
69
-
return
70
-
}
71
-
72
-
// Copy original headers
73
-
proxyReq.Header = r.Header
74
-
75
-
repoOwnerHandle := chi.URLParam(r, "user")
76
-
proxyReq.Header.Add("x-tangled-repo-owner-handle", repoOwnerHandle)
77
-
78
-
// Execute request
79
-
resp, err := client.Do(proxyReq)
80
-
if err != nil {
81
-
http.Error(w, err.Error(), http.StatusInternalServerError)
82
-
return
83
-
}
84
-
defer resp.Body.Close()
85
-
86
-
// Copy response headers
87
-
maps.Copy(w.Header(), resp.Header)
88
-
89
-
// Set response status code
90
-
w.WriteHeader(resp.StatusCode)
91
-
92
-
// Copy response body
93
-
if _, err := io.Copy(w, resp.Body); err != nil {
94
-
http.Error(w, err.Error(), http.StatusInternalServerError)
95
-
return
96
-
}
97
-
}
+168
appview/state/proxy_knot.go
+168
appview/state/proxy_knot.go
···
1
+
package state
2
+
3
+
import (
4
+
"fmt"
5
+
"io"
6
+
"maps"
7
+
"net/http"
8
+
"strings"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
indigoxrpc "github.com/bluesky-social/indigo/xrpc"
12
+
"github.com/go-chi/chi/v5"
13
+
"github.com/go-git/go-git/v5/plumbing"
14
+
"github.com/hashicorp/go-version"
15
+
"tangled.org/core/api/tangled"
16
+
"tangled.org/core/appview/models"
17
+
xrpcclient "tangled.org/core/appview/xrpcclient"
18
+
)
19
+
20
+
func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) {
21
+
user := r.Context().Value("resolvedId").(identity.Identity)
22
+
repo := r.Context().Value("repo").(*models.Repo)
23
+
24
+
scheme := "https"
25
+
if s.config.Core.Dev {
26
+
scheme = "http"
27
+
}
28
+
29
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/info/refs?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
30
+
s.proxyRequest(w, r, targetURL)
31
+
32
+
}
33
+
34
+
func (s *State) UploadPack(w http.ResponseWriter, r *http.Request) {
35
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
36
+
if !ok {
37
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
38
+
return
39
+
}
40
+
repo := r.Context().Value("repo").(*models.Repo)
41
+
42
+
scheme := "https"
43
+
if s.config.Core.Dev {
44
+
scheme = "http"
45
+
}
46
+
47
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-upload-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
48
+
s.proxyRequest(w, r, targetURL)
49
+
}
50
+
51
+
func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) {
52
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
53
+
if !ok {
54
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
55
+
return
56
+
}
57
+
repo := r.Context().Value("repo").(*models.Repo)
58
+
59
+
scheme := "https"
60
+
if s.config.Core.Dev {
61
+
scheme = "http"
62
+
}
63
+
64
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-receive-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
65
+
s.proxyRequest(w, r, targetURL)
66
+
}
67
+
68
+
var knotVersionDownloadArchiveConstraint = version.MustConstraints(version.NewConstraint(">= 1.12.0-alpha"))
69
+
70
+
func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) {
71
+
l := s.logger.With("handler", "DownloadArchive")
72
+
ref := chi.URLParam(r, "ref")
73
+
74
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
75
+
if !ok {
76
+
l.Error("failed to resolve user")
77
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
78
+
return
79
+
}
80
+
repo := r.Context().Value("repo").(*models.Repo)
81
+
82
+
scheme := "https"
83
+
if s.config.Core.Dev {
84
+
scheme = "http"
85
+
}
86
+
87
+
host := fmt.Sprintf("%s://%s", scheme, repo.Knot)
88
+
xrpcc := &indigoxrpc.Client{
89
+
Host: host,
90
+
}
91
+
l = l.With("knot", repo.Knot)
92
+
93
+
isCompatible := func() bool {
94
+
out, err := tangled.KnotVersion(r.Context(), xrpcc)
95
+
if err != nil {
96
+
l.Warn("failed to get knot version", "err", err)
97
+
return false
98
+
}
99
+
100
+
v, err := version.NewVersion(out.Version)
101
+
if err != nil {
102
+
l.Warn("failed to parse knot version", "version", out.Version, "err", err)
103
+
return false
104
+
}
105
+
106
+
if !knotVersionDownloadArchiveConstraint.Check(v) {
107
+
l.Warn("knot version incompatible.", "version", v)
108
+
return false
109
+
}
110
+
return true
111
+
}()
112
+
l.Debug("knot compatibility check", "isCompatible", isCompatible)
113
+
if isCompatible {
114
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref)
115
+
s.proxyRequest(w, r, targetURL)
116
+
} else {
117
+
l.Debug("requesting xrpc/sh.tangled.repo.archive")
118
+
archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo.DidSlashRepo())
119
+
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
120
+
l.Error("failed to call XRPC repo.archive", "err", xrpcerr)
121
+
s.pages.Error503(w)
122
+
return
123
+
}
124
+
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
125
+
filename := fmt.Sprintf("%s-%s.tar.gz", repo.Name, safeRefFilename)
126
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
127
+
w.Header().Set("Content-Type", "application/gzip")
128
+
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes)))
129
+
w.Write(archiveBytes)
130
+
}
131
+
}
132
+
133
+
func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) {
134
+
client := &http.Client{}
135
+
136
+
// Create new request
137
+
proxyReq, err := http.NewRequest(r.Method, targetURL, r.Body)
138
+
if err != nil {
139
+
http.Error(w, err.Error(), http.StatusInternalServerError)
140
+
return
141
+
}
142
+
143
+
// Copy original headers
144
+
proxyReq.Header = r.Header
145
+
146
+
repoOwnerHandle := chi.URLParam(r, "user")
147
+
proxyReq.Header.Add("x-tangled-repo-owner-handle", repoOwnerHandle)
148
+
149
+
// Execute request
150
+
resp, err := client.Do(proxyReq)
151
+
if err != nil {
152
+
http.Error(w, err.Error(), http.StatusInternalServerError)
153
+
return
154
+
}
155
+
defer resp.Body.Close()
156
+
157
+
// Copy response headers
158
+
maps.Copy(w.Header(), resp.Header)
159
+
160
+
// Set response status code
161
+
w.WriteHeader(resp.StatusCode)
162
+
163
+
// Copy response body
164
+
if _, err := io.Copy(w, resp.Body); err != nil {
165
+
http.Error(w, err.Error(), http.StatusInternalServerError)
166
+
return
167
+
}
168
+
}
+3
-1
appview/state/router.go
+3
-1
appview/state/router.go
···
103
103
r.Get("/info/refs", s.InfoRefs)
104
104
r.Post("/git-upload-pack", s.UploadPack)
105
105
r.Post("/git-receive-pack", s.ReceivePack)
106
-
106
+
// intentionally doesn't use /* as this isn't
107
+
// a file path
108
+
r.Get("/archive/{ref}", s.DownloadArchive)
107
109
})
108
110
})
109
111
+1
go.mod
+1
go.mod
···
131
131
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
132
132
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
133
133
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
134
+
github.com/hashicorp/go-version v1.8.0 // indirect
134
135
github.com/hashicorp/golang-lru v1.0.2 // indirect
135
136
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
136
137
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
+2
go.sum
+2
go.sum
···
264
264
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
265
265
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
266
266
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
267
+
github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
268
+
github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
267
269
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
268
270
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
269
271
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+69
knotserver/archive.go
+69
knotserver/archive.go
···
1
+
package knotserver
2
+
3
+
import (
4
+
"compress/gzip"
5
+
"fmt"
6
+
"net/http"
7
+
"strings"
8
+
9
+
securejoin "github.com/cyphar/filepath-securejoin"
10
+
"github.com/go-chi/chi/v5"
11
+
"github.com/go-git/go-git/v5/plumbing"
12
+
"tangled.org/core/knotserver/git"
13
+
)
14
+
15
+
func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) {
16
+
var (
17
+
did = chi.URLParam(r, "did")
18
+
name = chi.URLParam(r, "name")
19
+
ref = chi.URLParam(r, "ref")
20
+
)
21
+
repo, err := securejoin.SecureJoin(did, name)
22
+
if err != nil {
23
+
gitError(w, "repository not found", http.StatusNotFound)
24
+
h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err)
25
+
return
26
+
}
27
+
28
+
repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo)
29
+
if err != nil {
30
+
gitError(w, "repository not found", http.StatusNotFound)
31
+
h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err)
32
+
return
33
+
}
34
+
35
+
gr, err := git.Open(repoPath, ref)
36
+
37
+
immutableLink := fmt.Sprintf(
38
+
"https://%s/%s/%s/archive/%s",
39
+
h.c.Server.Hostname,
40
+
did,
41
+
name,
42
+
gr.Hash(),
43
+
)
44
+
45
+
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
46
+
filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename)
47
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
48
+
w.Header().Set("Content-Type", "application/gzip")
49
+
w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink))
50
+
51
+
gw := gzip.NewWriter(w)
52
+
defer gw.Close()
53
+
54
+
err = gr.WriteTar(gw, "")
55
+
if err != nil {
56
+
// once we start writing to the body we can't report error anymore
57
+
// so we are only left with logging the error
58
+
h.l.Error("writing tar file", "error", err)
59
+
return
60
+
}
61
+
62
+
err = gw.Flush()
63
+
if err != nil {
64
+
// once we start writing to the body we can't report error anymore
65
+
// so we are only left with logging the error
66
+
h.l.Error("flushing", "error", err.Error())
67
+
return
68
+
}
69
+
}
+4
knotserver/git/git.go
+4
knotserver/git/git.go
+2
knotserver/router.go
+2
knotserver/router.go
-81
knotserver/xrpc/repo_archive.go
-81
knotserver/xrpc/repo_archive.go
···
1
-
package xrpc
2
-
3
-
import (
4
-
"compress/gzip"
5
-
"fmt"
6
-
"net/http"
7
-
"strings"
8
-
9
-
"github.com/go-git/go-git/v5/plumbing"
10
-
11
-
"tangled.org/core/knotserver/git"
12
-
xrpcerr "tangled.org/core/xrpc/errors"
13
-
)
14
-
15
-
func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) {
16
-
repo := r.URL.Query().Get("repo")
17
-
repoPath, err := x.parseRepoParam(repo)
18
-
if err != nil {
19
-
writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest)
20
-
return
21
-
}
22
-
23
-
ref := r.URL.Query().Get("ref")
24
-
// ref can be empty (git.Open handles this)
25
-
26
-
format := r.URL.Query().Get("format")
27
-
if format == "" {
28
-
format = "tar.gz" // default
29
-
}
30
-
31
-
prefix := r.URL.Query().Get("prefix")
32
-
33
-
if format != "tar.gz" {
34
-
writeError(w, xrpcerr.NewXrpcError(
35
-
xrpcerr.WithTag("InvalidRequest"),
36
-
xrpcerr.WithMessage("only tar.gz format is supported"),
37
-
), http.StatusBadRequest)
38
-
return
39
-
}
40
-
41
-
gr, err := git.Open(repoPath, ref)
42
-
if err != nil {
43
-
writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound)
44
-
return
45
-
}
46
-
47
-
repoParts := strings.Split(repo, "/")
48
-
repoName := repoParts[len(repoParts)-1]
49
-
50
-
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
51
-
52
-
var archivePrefix string
53
-
if prefix != "" {
54
-
archivePrefix = prefix
55
-
} else {
56
-
archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename)
57
-
}
58
-
59
-
filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename)
60
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
61
-
w.Header().Set("Content-Type", "application/gzip")
62
-
63
-
gw := gzip.NewWriter(w)
64
-
defer gw.Close()
65
-
66
-
err = gr.WriteTar(gw, archivePrefix)
67
-
if err != nil {
68
-
// once we start writing to the body we can't report error anymore
69
-
// so we are only left with logging the error
70
-
x.Logger.Error("writing tar file", "error", err.Error())
71
-
return
72
-
}
73
-
74
-
err = gw.Flush()
75
-
if err != nil {
76
-
// once we start writing to the body we can't report error anymore
77
-
// so we are only left with logging the error
78
-
x.Logger.Error("flushing", "error", err.Error())
79
-
return
80
-
}
81
-
}
-1
knotserver/xrpc/xrpc.go
-1
knotserver/xrpc/xrpc.go
···
64
64
r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare)
65
65
r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch)
66
66
r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch)
67
-
r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive)
68
67
r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages)
69
68
70
69
// knot query endpoints (no auth required)
+1
lexicons/repo/archive.json
+1
lexicons/repo/archive.json
+3
nix/gomod2nix.toml
+3
nix/gomod2nix.toml
···
304
304
[mod."github.com/hashicorp/go-sockaddr"]
305
305
version = "v1.0.7"
306
306
hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs="
307
+
[mod."github.com/hashicorp/go-version"]
308
+
version = "v1.8.0"
309
+
hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8="
307
310
[mod."github.com/hashicorp/golang-lru"]
308
311
version = "v1.0.2"
309
312
hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48="