[mirror] Scalable static site server for Git forges (like GitHub Pages)

Use `s3:GetObject` instead of `s3:ListObjects` for `CheckDomain`.

miyuko aa965c5a 34db13e6

+18
src/backend.go
··· 21 21 } 22 22 } 23 23 24 + type BackendFeature string 25 + 26 + const ( 27 + FeatureCheckDomainMarker BackendFeature = "check-domain-marker" 28 + ) 29 + 24 30 type GetManifestOptions struct { 25 31 BypassCache bool 26 32 } 27 33 28 34 type Backend interface { 35 + // Returns true if the feature has been enabled for this store, false otherwise. 36 + HasFeature(ctx context.Context, feature BackendFeature) bool 37 + 38 + // Enables the feature for this store. 39 + EnableFeature(ctx context.Context, feature BackendFeature) error 40 + 29 41 // Retrieve a blob. Returns `reader, size, mtime, err`. 30 42 GetBlob(ctx context.Context, name string) (reader io.ReadSeeker, size uint64, mtime time.Time, err error) 31 43 ··· 52 64 // Delete a manifest. 53 65 DeleteManifest(ctx context.Context, name string) error 54 66 67 + // List all manifests. 68 + ListManifests(ctx context.Context) (manifests []string, err error) 69 + 55 70 // Check whether a domain has any deployments. 56 71 CheckDomain(ctx context.Context, domain string) (found bool, err error) 72 + 73 + // Creates a domain. This allows us to start serving content for the domain. 74 + CreateDomain(ctx context.Context, domain string) error 57 75 } 58 76 59 77 var backend Backend
+39
src/backend_fs.go
··· 6 6 "errors" 7 7 "fmt" 8 8 "io" 9 + "io/fs" 9 10 "os" 10 11 "path/filepath" 12 + "strings" 11 13 "time" 12 14 ) 13 15 ··· 66 68 67 69 func (fs *FSBackend) Backend() Backend { 68 70 return fs 71 + } 72 + 73 + func (fs *FSBackend) HasFeature(ctx context.Context, feature BackendFeature) bool { 74 + switch feature { 75 + case FeatureCheckDomainMarker: 76 + return true 77 + default: 78 + return false 79 + } 80 + } 81 + 82 + func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature) error { 83 + switch feature { 84 + case FeatureCheckDomainMarker: 85 + return nil 86 + default: 87 + return fmt.Errorf("not implemented") 88 + } 69 89 } 70 90 71 91 func (fs *FSBackend) GetBlob( ··· 133 153 return fs.blobRoot.Remove(blobPath) 134 154 } 135 155 156 + func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 157 + err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error { 158 + if strings.Count(path, "/") > 1 { 159 + return fs.SkipDir 160 + } 161 + _, project, _ := strings.Cut(path, "/") 162 + if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 163 + return nil 164 + } 165 + manifests = append(manifests, path) 166 + return nil 167 + }) 168 + return 169 + } 170 + 136 171 func (fs *FSBackend) GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) { 137 172 data, err := fs.siteRoot.ReadFile(name) 138 173 if errors.Is(err, os.ErrNotExist) { ··· 201 236 return false, err 202 237 } 203 238 } 239 + 240 + func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error { 241 + return nil // no-op 242 + }
+114 -15
src/backend_s3.go
··· 9 9 "log" 10 10 "net/http" 11 11 "path" 12 + "strings" 12 13 "time" 13 14 14 15 "github.com/c2h5oh/datasize" ··· 118 119 func (c *CachedManifest) Weight() uint32 { return c.weight } 119 120 120 121 type S3Backend struct { 121 - client *minio.Client 122 - bucket string 123 - blobCache *observedCache[string, *CachedBlob] 124 - siteCache *observedCache[string, *CachedManifest] 122 + client *minio.Client 123 + bucket string 124 + blobCache *observedCache[string, *CachedBlob] 125 + siteCache *observedCache[string, *CachedManifest] 126 + featureCache *otter.Cache[BackendFeature, bool] 125 127 } 126 128 127 129 var _ Backend = (*S3Backend)(nil) ··· 200 202 return nil, err 201 203 } 202 204 203 - return &S3Backend{client, bucket, blobCache, siteCache}, nil 205 + featureCache, err := otter.New(&otter.Options[BackendFeature, bool]{ 206 + RefreshCalculator: otter.RefreshWriting[BackendFeature, bool](10 * time.Minute), 207 + }) 208 + if err != nil { 209 + return nil, err 210 + } 211 + 212 + return &S3Backend{client, bucket, blobCache, siteCache, featureCache}, nil 204 213 } 205 214 206 215 func (s3 *S3Backend) Backend() Backend { ··· 211 220 return fmt.Sprintf("blob/%s", path.Join(splitBlobName(name)...)) 212 221 } 213 222 223 + func storeFeatureObjectName(feature BackendFeature) string { 224 + return fmt.Sprintf("meta/feature/%s", feature) 225 + } 226 + 227 + func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) bool { 228 + loader := func(ctx context.Context, feature BackendFeature) (bool, error) { 229 + _, err := s3.client.StatObject(ctx, s3.bucket, storeFeatureObjectName(feature), 230 + minio.StatObjectOptions{}) 231 + if err != nil { 232 + if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 233 + log.Printf("s3 feature %q: disabled", feature) 234 + return false, nil 235 + } else { 236 + return false, err 237 + } 238 + } 239 + log.Printf("s3 feature %q: enabled", feature) 240 + return true, nil 241 + } 242 + 243 + isOn, err := s3.featureCache.Get(ctx, feature, otter.LoaderFunc[BackendFeature, bool](loader)) 244 + if err != nil { 245 + err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err) 246 + ObserveError(err) 247 + log.Print(err) 248 + return false 249 + } 250 + return isOn 251 + } 252 + 253 + func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) error { 254 + _, err := s3.client.PutObject(ctx, s3.bucket, storeFeatureObjectName(feature), 255 + &bytes.Reader{}, 0, minio.PutObjectOptions{}) 256 + return err 257 + } 258 + 214 259 func (s3 *S3Backend) GetBlob( 215 260 ctx context.Context, 216 261 name string, ··· 307 352 return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData)) 308 353 } 309 354 355 + func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) { 356 + log.Print("s3: list manifests") 357 + 358 + ctx, cancel := context.WithCancel(ctx) 359 + defer cancel() 360 + 361 + prefix := manifestObjectName("") 362 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 363 + Prefix: prefix, 364 + Recursive: true, 365 + }) { 366 + if object.Err != nil { 367 + return nil, object.Err 368 + } 369 + key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/") 370 + if strings.Count(key, "/") > 1 { 371 + continue 372 + } 373 + _, project, _ := strings.Cut(key, "/") 374 + if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 375 + continue 376 + } 377 + manifests = append(manifests, key) 378 + } 379 + 380 + return 381 + } 382 + 310 383 type s3ManifestLoader struct { 311 384 s3 *S3Backend 312 385 } ··· 426 499 return err 427 500 } 428 501 429 - func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (bool, error) { 502 + func domainCheckObjectName(domain string) string { 503 + return manifestObjectName(fmt.Sprintf("%s/.exists", domain)) 504 + } 505 + 506 + func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) { 430 507 log.Printf("s3: check domain %s\n", domain) 431 508 432 - ctx, cancel := context.WithCancel(ctx) 433 - defer cancel() 509 + _, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain), 510 + minio.StatObjectOptions{}) 511 + if err != nil { 512 + if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 513 + exists, err = false, nil 514 + } 515 + } else { 516 + exists = true 517 + } 434 518 435 - for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 436 - Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)), 437 - }) { 438 - if object.Err != nil { 439 - return false, object.Err 519 + if !exists && !s3.HasFeature(ctx, FeatureCheckDomainMarker) { 520 + ctx, cancel := context.WithCancel(ctx) 521 + defer cancel() 522 + 523 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 524 + Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)), 525 + }) { 526 + if object.Err != nil { 527 + return false, object.Err 528 + } 529 + return true, nil 440 530 } 441 - return true, nil 531 + return false, nil 442 532 } 443 - return false, nil 533 + 534 + return 535 + } 536 + 537 + func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error { 538 + log.Printf("s3: create domain %s\n", domain) 539 + 540 + _, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain), 541 + &bytes.Reader{}, 0, minio.PutObjectOptions{}) 542 + return err 444 543 }
+11
src/main.go
··· 78 78 "load configuration from `filename`") 79 79 noConfig := flag.Bool("no-config", false, 80 80 "run without configuration file (configure via environment variables)") 81 + runMigration := flag.String("run-migration", "", 82 + "run a specific store migration (available: \"create-domain-markers\")") 81 83 getManifest := flag.String("get-manifest", "", 82 84 "write manifest for `webroot` (either 'domain.tld' or 'domain.tld/dir') to stdout as ProtoJSON") 83 85 getBlob := flag.String("get-blob", "", ··· 139 141 } 140 142 141 143 switch { 144 + case *runMigration != "": 145 + if err := ConfigureBackend(&config.Storage); err != nil { 146 + log.Fatalln(err) 147 + } 148 + 149 + if err := RunMigration(context.Background(), *runMigration); err != nil { 150 + log.Fatalln(err) 151 + } 152 + 142 153 case *getManifest != "": 143 154 if err := ConfigureBackend(&config.Storage); err != nil { 144 155 log.Fatalln(err)
+49
src/migrate.go
··· 1 + package git_pages 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "log" 7 + "slices" 8 + "strings" 9 + ) 10 + 11 + func RunMigration(ctx context.Context, name string) error { 12 + switch name { 13 + case "create-domain-markers": 14 + return createDomainMarkers(ctx) 15 + default: 16 + return fmt.Errorf("unknown migration name (expected one of \"create-domain-markers\")") 17 + } 18 + } 19 + 20 + func createDomainMarkers(ctx context.Context) error { 21 + if backend.HasFeature(ctx, FeatureCheckDomainMarker) { 22 + log.Print("store already has domain markers") 23 + return nil 24 + } 25 + 26 + var manifests, domains []string 27 + manifests, err := backend.ListManifests(ctx) 28 + if err != nil { 29 + return fmt.Errorf("list manifests: %w", err) 30 + } 31 + slices.Sort(manifests) 32 + for _, manifest := range manifests { 33 + domain, _, _ := strings.Cut(manifest, "/") 34 + if len(domains) == 0 || domains[len(domains)-1] != domain { 35 + domains = append(domains, domain) 36 + } 37 + } 38 + for idx, domain := range domains { 39 + log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain) 40 + if err := backend.CreateDomain(ctx, domain); err != nil { 41 + return fmt.Errorf("creating domain %s: %w", domain, err) 42 + } 43 + } 44 + if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil { 45 + return err 46 + } 47 + log.Printf("created markers for %d domains", len(domains)) 48 + return nil 49 + }
+28
src/observe.go
··· 266 266 return &observedBackend{inner: backend} 267 267 } 268 268 269 + func (backend *observedBackend) HasFeature(ctx context.Context, feature BackendFeature) (isOn bool) { 270 + span, ctx := ObserveFunction(ctx, "HasFeature") 271 + isOn = backend.inner.HasFeature(ctx, feature) 272 + span.Finish() 273 + return 274 + } 275 + 276 + func (backend *observedBackend) EnableFeature(ctx context.Context, feature BackendFeature) (err error) { 277 + span, ctx := ObserveFunction(ctx, "EnableFeature") 278 + err = backend.inner.EnableFeature(ctx, feature) 279 + span.Finish() 280 + return 281 + } 282 + 269 283 func (backend *observedBackend) GetBlob( 270 284 ctx context.Context, 271 285 name string, ··· 302 316 return 303 317 } 304 318 319 + func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 320 + span, ctx := ObserveFunction(ctx, "ListManifests") 321 + manifests, err = backend.inner.ListManifests(ctx) 322 + span.Finish() 323 + return 324 + } 325 + 305 326 func (backend *observedBackend) GetManifest( 306 327 ctx context.Context, 307 328 name string, ··· 345 366 span.Finish() 346 367 return 347 368 } 369 + 370 + func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) { 371 + span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain) 372 + err = backend.inner.CreateDomain(ctx, domain) 373 + span.Finish() 374 + return 375 + }
+5
src/update.go
··· 6 6 "fmt" 7 7 "io" 8 8 "log" 9 + "strings" 9 10 ) 10 11 11 12 type UpdateOutcome int ··· 42 43 } 43 44 } else if err = PrepareManifest(ctx, manifest); err == nil { 44 45 newManifest, err = StoreManifest(ctx, webRoot, manifest) 46 + if err == nil { 47 + domain, _, _ := strings.Cut(webRoot, "/") 48 + err = backend.CreateDomain(ctx, domain) 49 + } 45 50 if err == nil { 46 51 if oldManifest == nil { 47 52 outcome = UpdateCreated