[mirror] Scalable static site server for Git forges (like GitHub Pages)

Collect unresolved blob references in a dedicated error structure.

This will be used for incremental archive uploads.

Changed files
+30 -6
src
+27 -6
src/extract.go
··· 19 19 20 20 var ErrArchiveTooLarge = errors.New("archive too large") 21 21 22 - const BlobReferencePrefix = "/git/blobs/" 23 - 24 22 func boundArchiveStream(reader io.Reader) io.Reader { 25 23 return ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()), 26 24 fmt.Errorf("%w: %s limit exceeded", ErrArchiveTooLarge, config.Limits.MaxSiteSize.HR())) ··· 52 50 return next(ctx, boundArchiveStream(stream)) 53 51 } 54 52 53 + const BlobReferencePrefix = "/git/blobs/" 54 + 55 + type UnresolvedRefError struct { 56 + missing []string 57 + } 58 + 59 + func (err UnresolvedRefError) Error() string { 60 + return fmt.Sprintf("%d unresolved blob references", len(err.missing)) 61 + } 62 + 55 63 // Returns a map of git hash to entry. If `manifest` is nil, returns an empty map. 56 64 func indexManifestByGitHash(manifest *Manifest) map[string]*Entry { 57 65 index := map[string]*Entry{} ··· 68 76 } 69 77 70 78 func addSymlinkOrBlobReference( 71 - manifest *Manifest, fileName string, target string, index map[string]*Entry, 79 + manifest *Manifest, fileName string, target string, 80 + index map[string]*Entry, missing *[]string, 72 81 ) *Entry { 73 82 if hash, found := strings.CutPrefix(target, BlobReferencePrefix); found { 74 83 if entry, found := index[hash]; found { 75 84 manifest.Contents[fileName] = entry 76 85 return entry 77 86 } else { 78 - AddProblem(manifest, fileName, "unresolved reference: %s", target) 87 + *missing = append(*missing, hash) 79 88 return nil 80 89 } 81 90 } else { ··· 90 99 var dataBytesTransferred int64 91 100 92 101 index := indexManifestByGitHash(oldManifest) 102 + missing := []string{} 93 103 manifest := NewManifest() 94 104 for { 95 105 header, err := archive.Next() ··· 119 129 AddFile(manifest, fileName, fileData) 120 130 dataBytesTransferred += int64(len(fileData)) 121 131 case tar.TypeSymlink: 122 - entry := addSymlinkOrBlobReference(manifest, fileName, header.Linkname, index) 132 + entry := addSymlinkOrBlobReference( 133 + manifest, fileName, header.Linkname, index, &missing) 123 134 dataBytesRecycled += entry.GetOriginalSize() 124 135 case tar.TypeDir: 125 136 AddDirectory(manifest, fileName) ··· 127 138 AddProblem(manifest, fileName, "tar: unsupported type '%c'", header.Typeflag) 128 139 continue 129 140 } 141 + } 142 + 143 + if len(missing) > 0 { 144 + return nil, UnresolvedRefError{missing} 130 145 } 131 146 132 147 logc.Printf(ctx, ··· 166 181 var dataBytesTransferred int64 167 182 168 183 index := indexManifestByGitHash(oldManifest) 184 + missing := []string{} 169 185 manifest := NewManifest() 170 186 for _, file := range archive.File { 171 187 if strings.HasSuffix(file.Name, "/") { ··· 183 199 } 184 200 185 201 if file.Mode()&os.ModeSymlink != 0 { 186 - entry := addSymlinkOrBlobReference(manifest, file.Name, string(fileData), index) 202 + entry := addSymlinkOrBlobReference( 203 + manifest, file.Name, string(fileData), index, &missing) 187 204 dataBytesRecycled += entry.GetOriginalSize() 188 205 } else { 189 206 AddFile(manifest, file.Name, fileData) 190 207 dataBytesTransferred += int64(len(fileData)) 191 208 } 192 209 } 210 + } 211 + 212 + if len(missing) > 0 { 213 + return nil, UnresolvedRefError{missing} 193 214 } 194 215 195 216 logc.Printf(ctx,
+3
src/pages.go
··· 572 572 func reportUpdateResult(w http.ResponseWriter, result UpdateResult) error { 573 573 switch result.outcome { 574 574 case UpdateError: 575 + var unresolvedRefErr UnresolvedRefError 575 576 if errors.Is(result.err, ErrManifestTooLarge) { 576 577 w.WriteHeader(http.StatusRequestEntityTooLarge) 577 578 } else if errors.Is(result.err, errArchiveFormat) { ··· 586 587 w.WriteHeader(http.StatusConflict) 587 588 } else if errors.Is(result.err, ErrDomainFrozen) { 588 589 w.WriteHeader(http.StatusForbidden) 590 + } else if errors.As(result.err, &unresolvedRefErr) { 591 + w.WriteHeader(http.StatusUnprocessableEntity) 589 592 } else { 590 593 w.WriteHeader(http.StatusServiceUnavailable) 591 594 }