[mirror] Scalable static site server for Git forges (like GitHub Pages)

Consolidate return values into `BlobMetadata`. NFC

+6 -1
src/backend.go
··· 31 FeatureCheckDomainMarker BackendFeature = "check-domain-marker" 32 ) 33 34 type GetManifestOptions struct { 35 // If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns 36 // a fresh object instead of revalidating in background and returning a stale object. ··· 77 78 // Retrieve a blob. Returns `reader, size, mtime, err`. 79 GetBlob(ctx context.Context, name string) ( 80 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 81 ) 82 83 // Store a blob. If a blob called `name` already exists, this function returns `nil` without
··· 31 FeatureCheckDomainMarker BackendFeature = "check-domain-marker" 32 ) 33 34 + type BlobMetadata struct { 35 + Size int64 36 + LastModified time.Time 37 + } 38 + 39 type GetManifestOptions struct { 40 // If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns 41 // a fresh object instead of revalidating in background and returning a stale object. ··· 82 83 // Retrieve a blob. Returns `reader, size, mtime, err`. 84 GetBlob(ctx context.Context, name string) ( 85 + reader io.ReadSeeker, metadata BlobMetadata, err error, 86 ) 87 88 // Store a blob. If a blob called `name` already exists, this function returns `nil` without
+2 -3
src/backend_fs.go
··· 11 "os" 12 "path/filepath" 13 "strings" 14 - "time" 15 ) 16 17 type FSBackend struct { ··· 118 func (fs *FSBackend) GetBlob( 119 ctx context.Context, name string, 120 ) ( 121 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 122 ) { 123 blobPath := filepath.Join(splitBlobName(name)...) 124 stat, err := fs.blobRoot.Stat(blobPath) ··· 134 err = fmt.Errorf("open: %w", err) 135 return 136 } 137 - return file, uint64(stat.Size()), stat.ModTime(), nil 138 } 139 140 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error {
··· 11 "os" 12 "path/filepath" 13 "strings" 14 ) 15 16 type FSBackend struct { ··· 117 func (fs *FSBackend) GetBlob( 118 ctx context.Context, name string, 119 ) ( 120 + reader io.ReadSeeker, metadata BlobMetadata, err error, 121 ) { 122 blobPath := filepath.Join(splitBlobName(name)...) 123 stat, err := fs.blobRoot.Stat(blobPath) ··· 133 err = fmt.Errorf("open: %w", err) 134 return 135 } 136 + return file, BlobMetadata{int64(stat.Size()), stat.ModTime()}, nil 137 } 138 139 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error {
+3 -3
src/backend_s3.go
··· 266 func (s3 *S3Backend) GetBlob( 267 ctx context.Context, name string, 268 ) ( 269 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 270 ) { 271 loader := func(ctx context.Context, name string) (*CachedBlob, error) { 272 logc.Printf(ctx, "s3: get blob %s\n", name) ··· 316 } 317 } else { 318 reader = bytes.NewReader(cached.blob) 319 - size = uint64(len(cached.blob)) 320 - mtime = cached.mtime 321 } 322 return 323 }
··· 266 func (s3 *S3Backend) GetBlob( 267 ctx context.Context, name string, 268 ) ( 269 + reader io.ReadSeeker, metadata BlobMetadata, err error, 270 ) { 271 loader := func(ctx context.Context, name string) (*CachedBlob, error) { 272 logc.Printf(ctx, "s3: get blob %s\n", name) ··· 316 } 317 } else { 318 reader = bytes.NewReader(cached.blob) 319 + metadata.Size = int64(len(cached.blob)) 320 + metadata.LastModified = cached.mtime 321 } 322 return 323 }
+7 -5
src/collect.go
··· 5 "context" 6 "fmt" 7 "io" 8 - "time" 9 ) 10 11 type Flusher interface { ··· 66 67 case Type_ExternalFile: 68 var blobReader io.Reader 69 - var blobMtime time.Time 70 var blobData []byte 71 - blobReader, _, blobMtime, err = backend.GetBlob(context, string(entry.Data)) 72 if err != nil { 73 return 74 } 75 - blobData, _ = io.ReadAll(blobReader) 76 header.Typeflag = tar.TypeReg 77 header.Mode = 0644 78 - header.ModTime = blobMtime 79 err = appendFile(&header, blobData, entry.GetTransform()) 80 81 case Type_Symlink:
··· 5 "context" 6 "fmt" 7 "io" 8 ) 9 10 type Flusher interface { ··· 65 66 case Type_ExternalFile: 67 var blobReader io.Reader 68 + var blobMetadata BlobMetadata 69 var blobData []byte 70 + blobReader, blobMetadata, err = backend.GetBlob(context, string(entry.Data)) 71 + if err != nil { 72 + return 73 + } 74 + blobData, err = io.ReadAll(blobReader) 75 if err != nil { 76 return 77 } 78 header.Typeflag = tar.TypeReg 79 header.Mode = 0644 80 + header.ModTime = blobMetadata.LastModified 81 err = appendFile(&header, blobData, entry.GetTransform()) 82 83 case Type_Symlink:
+1 -1
src/main.go
··· 287 logc.Fatalln(ctx, err) 288 } 289 290 - reader, _, _, err := backend.GetBlob(ctx, *getBlob) 291 if err != nil { 292 logc.Fatalln(ctx, err) 293 }
··· 287 logc.Fatalln(ctx, err) 288 } 289 290 + reader, _, err := backend.GetBlob(ctx, *getBlob) 291 if err != nil { 292 logc.Fatalln(ctx, err) 293 }
+4 -4
src/observe.go
··· 344 func (backend *observedBackend) GetBlob( 345 ctx context.Context, name string, 346 ) ( 347 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 348 ) { 349 span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name) 350 - if reader, size, mtime, err = backend.inner.GetBlob(ctx, name); err == nil { 351 - ObserveData(ctx, "blob.size", size) 352 blobsRetrievedCount.Inc() 353 - blobsRetrievedBytes.Add(float64(size)) 354 } 355 span.Finish() 356 return
··· 344 func (backend *observedBackend) GetBlob( 345 ctx context.Context, name string, 346 ) ( 347 + reader io.ReadSeeker, metadata BlobMetadata, err error, 348 ) { 349 span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name) 350 + if reader, metadata, err = backend.inner.GetBlob(ctx, name); err == nil { 351 + ObserveData(ctx, "blob.size", metadata.Size) 352 blobsRetrievedCount.Inc() 353 + blobsRetrievedBytes.Add(float64(metadata.Size)) 354 } 355 span.Finish() 356 return
+3 -1
src/pages.go
··· 292 w.WriteHeader(http.StatusNotModified) 293 return nil 294 } else { 295 - reader, _, mtime, err = backend.GetBlob(r.Context(), string(entry.Data)) 296 if err != nil { 297 ObserveError(err) // all storage errors must be reported 298 w.WriteHeader(http.StatusInternalServerError) 299 fmt.Fprintf(w, "internal server error: %s\n", err) 300 return err 301 } 302 w.Header().Set("ETag", etag) 303 } 304 } else if entry.GetType() == Type_Directory {
··· 292 w.WriteHeader(http.StatusNotModified) 293 return nil 294 } else { 295 + var metadata BlobMetadata 296 + reader, metadata, err = backend.GetBlob(r.Context(), string(entry.Data)) 297 if err != nil { 298 ObserveError(err) // all storage errors must be reported 299 w.WriteHeader(http.StatusInternalServerError) 300 fmt.Fprintf(w, "internal server error: %s\n", err) 301 return err 302 } 303 + mtime = metadata.LastModified 304 w.Header().Set("ETag", etag) 305 } 306 } else if entry.GetType() == Type_Directory {