+18
src/backend.go
+18
src/backend.go
···
21
21
}
22
22
}
23
23
24
+
type BackendFeature string
25
+
26
+
const (
27
+
FeatureCheckDomainMarker BackendFeature = "check-domain-marker"
28
+
)
29
+
24
30
type GetManifestOptions struct {
25
31
BypassCache bool
26
32
}
27
33
28
34
type Backend interface {
35
+
// Returns true if the feature has been enabled for this store, false otherwise.
36
+
HasFeature(ctx context.Context, feature BackendFeature) bool
37
+
38
+
// Enables the feature for this store.
39
+
EnableFeature(ctx context.Context, feature BackendFeature) error
40
+
29
41
// Retrieve a blob. Returns `reader, size, mtime, err`.
30
42
GetBlob(ctx context.Context, name string) (reader io.ReadSeeker, size uint64, mtime time.Time, err error)
31
43
···
52
64
// Delete a manifest.
53
65
DeleteManifest(ctx context.Context, name string) error
54
66
67
+
// List all manifests.
68
+
ListManifests(ctx context.Context) (manifests []string, err error)
69
+
55
70
// Check whether a domain has any deployments.
56
71
CheckDomain(ctx context.Context, domain string) (found bool, err error)
72
+
73
+
// Creates a domain. This allows us to start serving content for the domain.
74
+
CreateDomain(ctx context.Context, domain string) error
57
75
}
58
76
59
77
var backend Backend
+39
src/backend_fs.go
+39
src/backend_fs.go
···
6
6
"errors"
7
7
"fmt"
8
8
"io"
9
+
"io/fs"
9
10
"os"
10
11
"path/filepath"
12
+
"strings"
11
13
"time"
12
14
)
13
15
···
66
68
67
69
func (fs *FSBackend) Backend() Backend {
68
70
return fs
71
+
}
72
+
73
+
func (fs *FSBackend) HasFeature(ctx context.Context, feature BackendFeature) bool {
74
+
switch feature {
75
+
case FeatureCheckDomainMarker:
76
+
return true
77
+
default:
78
+
return false
79
+
}
80
+
}
81
+
82
+
func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature) error {
83
+
switch feature {
84
+
case FeatureCheckDomainMarker:
85
+
return nil
86
+
default:
87
+
return fmt.Errorf("not implemented")
88
+
}
69
89
}
70
90
71
91
func (fs *FSBackend) GetBlob(
···
133
153
return fs.blobRoot.Remove(blobPath)
134
154
}
135
155
156
+
func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
157
+
err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error {
158
+
if strings.Count(path, "/") > 1 {
159
+
return fs.SkipDir
160
+
}
161
+
_, project, _ := strings.Cut(path, "/")
162
+
if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
163
+
return nil
164
+
}
165
+
manifests = append(manifests, path)
166
+
return nil
167
+
})
168
+
return
169
+
}
170
+
136
171
func (fs *FSBackend) GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) {
137
172
data, err := fs.siteRoot.ReadFile(name)
138
173
if errors.Is(err, os.ErrNotExist) {
···
201
236
return false, err
202
237
}
203
238
}
239
+
240
+
func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error {
241
+
return nil // no-op
242
+
}
+114
-15
src/backend_s3.go
+114
-15
src/backend_s3.go
···
9
9
"log"
10
10
"net/http"
11
11
"path"
12
+
"strings"
12
13
"time"
13
14
14
15
"github.com/c2h5oh/datasize"
···
118
119
func (c *CachedManifest) Weight() uint32 { return c.weight }
119
120
120
121
type S3Backend struct {
121
-
client *minio.Client
122
-
bucket string
123
-
blobCache *observedCache[string, *CachedBlob]
124
-
siteCache *observedCache[string, *CachedManifest]
122
+
client *minio.Client
123
+
bucket string
124
+
blobCache *observedCache[string, *CachedBlob]
125
+
siteCache *observedCache[string, *CachedManifest]
126
+
featureCache *otter.Cache[BackendFeature, bool]
125
127
}
126
128
127
129
var _ Backend = (*S3Backend)(nil)
···
200
202
return nil, err
201
203
}
202
204
203
-
return &S3Backend{client, bucket, blobCache, siteCache}, nil
205
+
featureCache, err := otter.New(&otter.Options[BackendFeature, bool]{
206
+
RefreshCalculator: otter.RefreshWriting[BackendFeature, bool](10 * time.Minute),
207
+
})
208
+
if err != nil {
209
+
return nil, err
210
+
}
211
+
212
+
return &S3Backend{client, bucket, blobCache, siteCache, featureCache}, nil
204
213
}
205
214
206
215
func (s3 *S3Backend) Backend() Backend {
···
211
220
return fmt.Sprintf("blob/%s", path.Join(splitBlobName(name)...))
212
221
}
213
222
223
+
func storeFeatureObjectName(feature BackendFeature) string {
224
+
return fmt.Sprintf("meta/feature/%s", feature)
225
+
}
226
+
227
+
func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) bool {
228
+
loader := func(ctx context.Context, feature BackendFeature) (bool, error) {
229
+
_, err := s3.client.StatObject(ctx, s3.bucket, storeFeatureObjectName(feature),
230
+
minio.StatObjectOptions{})
231
+
if err != nil {
232
+
if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
233
+
log.Printf("s3 feature %q: disabled", feature)
234
+
return false, nil
235
+
} else {
236
+
return false, err
237
+
}
238
+
}
239
+
log.Printf("s3 feature %q: enabled", feature)
240
+
return true, nil
241
+
}
242
+
243
+
isOn, err := s3.featureCache.Get(ctx, feature, otter.LoaderFunc[BackendFeature, bool](loader))
244
+
if err != nil {
245
+
err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err)
246
+
ObserveError(err)
247
+
log.Print(err)
248
+
return false
249
+
}
250
+
return isOn
251
+
}
252
+
253
+
func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) error {
254
+
_, err := s3.client.PutObject(ctx, s3.bucket, storeFeatureObjectName(feature),
255
+
&bytes.Reader{}, 0, minio.PutObjectOptions{})
256
+
return err
257
+
}
258
+
214
259
func (s3 *S3Backend) GetBlob(
215
260
ctx context.Context,
216
261
name string,
···
307
352
return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))
308
353
}
309
354
355
+
func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) {
356
+
log.Print("s3: list manifests")
357
+
358
+
ctx, cancel := context.WithCancel(ctx)
359
+
defer cancel()
360
+
361
+
prefix := manifestObjectName("")
362
+
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
363
+
Prefix: prefix,
364
+
Recursive: true,
365
+
}) {
366
+
if object.Err != nil {
367
+
return nil, object.Err
368
+
}
369
+
key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/")
370
+
if strings.Count(key, "/") > 1 {
371
+
continue
372
+
}
373
+
_, project, _ := strings.Cut(key, "/")
374
+
if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
375
+
continue
376
+
}
377
+
manifests = append(manifests, key)
378
+
}
379
+
380
+
return
381
+
}
382
+
310
383
type s3ManifestLoader struct {
311
384
s3 *S3Backend
312
385
}
···
426
499
return err
427
500
}
428
501
429
-
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (bool, error) {
502
+
func domainCheckObjectName(domain string) string {
503
+
return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
504
+
}
505
+
506
+
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) {
430
507
log.Printf("s3: check domain %s\n", domain)
431
508
432
-
ctx, cancel := context.WithCancel(ctx)
433
-
defer cancel()
509
+
_, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain),
510
+
minio.StatObjectOptions{})
511
+
if err != nil {
512
+
if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
513
+
exists, err = false, nil
514
+
}
515
+
} else {
516
+
exists = true
517
+
}
434
518
435
-
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
436
-
Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)),
437
-
}) {
438
-
if object.Err != nil {
439
-
return false, object.Err
519
+
if !exists && !s3.HasFeature(ctx, FeatureCheckDomainMarker) {
520
+
ctx, cancel := context.WithCancel(ctx)
521
+
defer cancel()
522
+
523
+
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
524
+
Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)),
525
+
}) {
526
+
if object.Err != nil {
527
+
return false, object.Err
528
+
}
529
+
return true, nil
440
530
}
441
-
return true, nil
531
+
return false, nil
442
532
}
443
-
return false, nil
533
+
534
+
return
535
+
}
536
+
537
+
func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
538
+
log.Printf("s3: create domain %s\n", domain)
539
+
540
+
_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
541
+
&bytes.Reader{}, 0, minio.PutObjectOptions{})
542
+
return err
444
543
}
+11
src/main.go
+11
src/main.go
···
78
78
"load configuration from `filename`")
79
79
noConfig := flag.Bool("no-config", false,
80
80
"run without configuration file (configure via environment variables)")
81
+
runMigration := flag.String("run-migration", "",
82
+
"run a specific store migration (available: \"create-domain-markers\")")
81
83
getManifest := flag.String("get-manifest", "",
82
84
"write manifest for `webroot` (either 'domain.tld' or 'domain.tld/dir') to stdout as ProtoJSON")
83
85
getBlob := flag.String("get-blob", "",
···
139
141
}
140
142
141
143
switch {
144
+
case *runMigration != "":
145
+
if err := ConfigureBackend(&config.Storage); err != nil {
146
+
log.Fatalln(err)
147
+
}
148
+
149
+
if err := RunMigration(context.Background(), *runMigration); err != nil {
150
+
log.Fatalln(err)
151
+
}
152
+
142
153
case *getManifest != "":
143
154
if err := ConfigureBackend(&config.Storage); err != nil {
144
155
log.Fatalln(err)
+49
src/migrate.go
+49
src/migrate.go
···
1
+
package git_pages
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log"
7
+
"slices"
8
+
"strings"
9
+
)
10
+
11
+
func RunMigration(ctx context.Context, name string) error {
12
+
switch name {
13
+
case "create-domain-markers":
14
+
return createDomainMarkers(ctx)
15
+
default:
16
+
return fmt.Errorf("unknown migration name (expected one of \"create-domain-markers\")")
17
+
}
18
+
}
19
+
20
+
func createDomainMarkers(ctx context.Context) error {
21
+
if backend.HasFeature(ctx, FeatureCheckDomainMarker) {
22
+
log.Print("store already has domain markers")
23
+
return nil
24
+
}
25
+
26
+
var manifests, domains []string
27
+
manifests, err := backend.ListManifests(ctx)
28
+
if err != nil {
29
+
return fmt.Errorf("list manifests: %w", err)
30
+
}
31
+
slices.Sort(manifests)
32
+
for _, manifest := range manifests {
33
+
domain, _, _ := strings.Cut(manifest, "/")
34
+
if len(domains) == 0 || domains[len(domains)-1] != domain {
35
+
domains = append(domains, domain)
36
+
}
37
+
}
38
+
for idx, domain := range domains {
39
+
log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain)
40
+
if err := backend.CreateDomain(ctx, domain); err != nil {
41
+
return fmt.Errorf("creating domain %s: %w", domain, err)
42
+
}
43
+
}
44
+
if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil {
45
+
return err
46
+
}
47
+
log.Printf("created markers for %d domains", len(domains))
48
+
return nil
49
+
}
+28
src/observe.go
+28
src/observe.go
···
266
266
return &observedBackend{inner: backend}
267
267
}
268
268
269
+
func (backend *observedBackend) HasFeature(ctx context.Context, feature BackendFeature) (isOn bool) {
270
+
span, ctx := ObserveFunction(ctx, "HasFeature")
271
+
isOn = backend.inner.HasFeature(ctx, feature)
272
+
span.Finish()
273
+
return
274
+
}
275
+
276
+
func (backend *observedBackend) EnableFeature(ctx context.Context, feature BackendFeature) (err error) {
277
+
span, ctx := ObserveFunction(ctx, "EnableFeature")
278
+
err = backend.inner.EnableFeature(ctx, feature)
279
+
span.Finish()
280
+
return
281
+
}
282
+
269
283
func (backend *observedBackend) GetBlob(
270
284
ctx context.Context,
271
285
name string,
···
302
316
return
303
317
}
304
318
319
+
func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
320
+
span, ctx := ObserveFunction(ctx, "ListManifests")
321
+
manifests, err = backend.inner.ListManifests(ctx)
322
+
span.Finish()
323
+
return
324
+
}
325
+
305
326
func (backend *observedBackend) GetManifest(
306
327
ctx context.Context,
307
328
name string,
···
345
366
span.Finish()
346
367
return
347
368
}
369
+
370
+
func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) {
371
+
span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain)
372
+
err = backend.inner.CreateDomain(ctx, domain)
373
+
span.Finish()
374
+
return
375
+
}
+5
src/update.go
+5
src/update.go
···
6
6
"fmt"
7
7
"io"
8
8
"log"
9
+
"strings"
9
10
)
10
11
11
12
type UpdateOutcome int
···
42
43
}
43
44
} else if err = PrepareManifest(ctx, manifest); err == nil {
44
45
newManifest, err = StoreManifest(ctx, webRoot, manifest)
46
+
if err == nil {
47
+
domain, _, _ := strings.Cut(webRoot, "/")
48
+
err = backend.CreateDomain(ctx, domain)
49
+
}
45
50
if err == nil {
46
51
if oldManifest == nil {
47
52
outcome = UpdateCreated