[mirror] Scalable static site server for Git forges (like GitHub Pages)

Warn when a Git repository is uploaded with Git LFS-tracked files.

miyuko d2b51441 34985c89

+139 -8
+18
src/fetch.go
··· 9 9 "net/url" 10 10 "os" 11 11 "slices" 12 + "strings" 12 13 13 14 "github.com/c2h5oh/datasize" 14 15 "github.com/go-git/go-billy/v6/osfs" ··· 209 210 datasize.ByteSize(dataBytesTransferred).HR(), 210 211 ) 211 212 213 + warnAboutGitLFS(ctx, manifest) 214 + 212 215 return manifest, nil 213 216 } 214 217 ··· 254 257 255 258 return nil 256 259 } 260 + 261 + func warnAboutGitLFS(ctx context.Context, manifest *Manifest) { 262 + gitattributes := ReadGitAttributes(ctx, manifest) 263 + for _, name := range slices.Sorted(maps.Keys(manifest.GetContents())) { 264 + entry := manifest.GetContents()[name] 265 + if !IsEntryRegularFile(entry) { 266 + continue 267 + } 268 + parts := strings.Split(name, "/") 269 + attrs, _ := gitattributes.Match(parts, nil) 270 + if attr, ok := attrs["filter"]; ok && attr.Value() == "lfs" { 271 + AddProblem(manifest, name, "git-pages does not support Git LFS; move this file into Git or use incremental uploads") 272 + } 273 + } 274 + }
+61
src/gitattributes.go
··· 1 + package git_pages 2 + 3 + import ( 4 + "bytes" 5 + "cmp" 6 + "context" 7 + "slices" 8 + "strings" 9 + 10 + "github.com/go-git/go-git/v6/plumbing/format/gitattributes" 11 + ) 12 + 13 + func ReadGitAttributes(ctx context.Context, manifest *Manifest) gitattributes.Matcher { 14 + type entryPair struct { 15 + parts []string 16 + entry *Entry 17 + } 18 + 19 + // Collect all .gitattributes files. 20 + var files []entryPair 21 + for name, entry := range manifest.GetContents() { 22 + switch entry.GetType() { 23 + case Type_InlineFile, Type_ExternalFile: 24 + parts := strings.Split(name, "/") 25 + if parts[len(parts)-1] == ".gitattributes" { 26 + files = append(files, entryPair{parts, entry}) 27 + } 28 + } 29 + } 30 + 31 + // Sort the file list by depth, then by name. 32 + slices.SortFunc(files, func(a entryPair, b entryPair) int { 33 + return cmp.Or( 34 + cmp.Compare(len(a.parts), len(b.parts)), 35 + slices.Compare(a.parts, b.parts), 36 + ) 37 + }) 38 + 39 + // Gather all .gitattributes rules, sorted by depth. 40 + var rules []gitattributes.MatchAttribute 41 + for _, pair := range files { 42 + parts, entry := pair.parts, pair.entry 43 + data, err := GetEntryContents(ctx, entry) 44 + if err != nil { 45 + continue 46 + } 47 + dirs := parts[:len(parts)-1] 48 + isRoot := len(parts) == 1 49 + newRules, err := gitattributes.ReadAttributes(bytes.NewReader(data), dirs, isRoot) 50 + if err != nil { 51 + AddProblem(manifest, strings.Join(parts, "/"), "parsing .gitattributes: %v", err) 52 + continue 53 + } 54 + rules = append(rules, newRules...) 55 + } 56 + 57 + // gitattributes.Matcher applies rules in reverse. 58 + slices.Reverse(rules) 59 + matcher := gitattributes.NewMatcher(rules) 60 + return matcher 61 + }
+9 -3
src/headers.go
··· 1 1 package git_pages 2 2 3 3 import ( 4 + "context" 4 5 "errors" 5 6 "fmt" 6 7 "net/http" ··· 85 86 } 86 87 87 88 // Parses redirects file and injects rules into the manifest. 88 - func ProcessHeadersFile(manifest *Manifest) error { 89 + func ProcessHeadersFile(ctx context.Context, manifest *Manifest) error { 89 90 headersEntry := manifest.Contents[HeadersFileName] 90 91 delete(manifest.Contents, HeadersFileName) 91 92 if headersEntry == nil { 92 93 return nil 93 - } else if headersEntry.GetType() != Type_InlineFile { 94 + } 95 + 96 + data, err := GetEntryContents(ctx, headersEntry) 97 + if errors.Is(err, ErrNotRegularFile) { 94 98 return AddProblem(manifest, HeadersFileName, 95 99 "not a regular file") 100 + } else if err != nil { 101 + return err 96 102 } 97 103 98 - rules, err := headers.ParseString(string(headersEntry.GetData())) 104 + rules, err := headers.ParseString(string(data)) 99 105 if err != nil { 100 106 return AddProblem(manifest, HeadersFileName, 101 107 "syntax error: %s", err)
+41 -2
src/manifest.go
··· 8 8 "crypto/sha256" 9 9 "errors" 10 10 "fmt" 11 + "io" 11 12 "mime" 12 13 "net/http" 13 14 "path" ··· 144 145 return fmt.Errorf("%s: %s", pathName, cause) 145 146 } 146 147 148 + func IsEntryRegularFile(entry *Entry) bool { 149 + return entry.GetType() == Type_InlineFile || 150 + entry.GetType() == Type_ExternalFile 151 + } 152 + 153 + var ErrNotRegularFile = errors.New("not a regular file") 154 + 155 + func GetEntryContents(ctx context.Context, entry *Entry) (data []byte, err error) { 156 + switch entry.GetType() { 157 + case Type_InlineFile: 158 + data = entry.GetData() 159 + case Type_ExternalFile: 160 + reader, _, err := backend.GetBlob(ctx, string(entry.GetData())) 161 + if err != nil { 162 + return nil, err 163 + } 164 + data, err = io.ReadAll(reader) 165 + if err != nil { 166 + return nil, err 167 + } 168 + default: 169 + return nil, ErrNotRegularFile 170 + } 171 + 172 + switch entry.GetTransform() { 173 + case Transform_Identity: 174 + case Transform_Zstd: 175 + data, err = zstdDecoder.DecodeAll(data, []byte{}) 176 + if err != nil { 177 + return nil, err 178 + } 179 + default: 180 + return nil, fmt.Errorf("unexpected transform") 181 + } 182 + 183 + return 184 + } 185 + 147 186 // EnsureLeadingDirectories adds directory entries for any parent directories 148 187 // that are implicitly referenced by files in the manifest but don't have 149 188 // explicit directory entries. (This can be the case if an archive is created ··· 275 314 // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?) 276 315 func PrepareManifest(ctx context.Context, manifest *Manifest) error { 277 316 // Parse Netlify-style `_redirects`. 278 - if err := ProcessRedirectsFile(manifest); err != nil { 317 + if err := ProcessRedirectsFile(ctx, manifest); err != nil { 279 318 logc.Printf(ctx, "redirects err: %s\n", err) 280 319 } else if len(manifest.Redirects) > 0 { 281 320 logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects)) ··· 285 324 LintRedirects(manifest) 286 325 287 326 // Parse Netlify-style `_headers`. 288 - if err := ProcessHeadersFile(manifest); err != nil { 327 + if err := ProcessHeadersFile(ctx, manifest); err != nil { 289 328 logc.Printf(ctx, "headers err: %s\n", err) 290 329 } else if len(manifest.Headers) > 0 { 291 330 logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers))
+10 -3
src/redirects.go
··· 1 1 package git_pages 2 2 3 3 import ( 4 + "context" 5 + "errors" 4 6 "fmt" 5 7 "net/http" 6 8 "net/url" ··· 96 98 } 97 99 98 100 // Parses redirects file and injects rules into the manifest. 99 - func ProcessRedirectsFile(manifest *Manifest) error { 101 + func ProcessRedirectsFile(ctx context.Context, manifest *Manifest) error { 100 102 redirectsEntry := manifest.Contents[RedirectsFileName] 101 103 delete(manifest.Contents, RedirectsFileName) 102 104 if redirectsEntry == nil { 103 105 return nil 104 - } else if redirectsEntry.GetType() != Type_InlineFile { 106 + } 107 + 108 + data, err := GetEntryContents(ctx, redirectsEntry) 109 + if errors.Is(err, ErrNotRegularFile) { 105 110 return AddProblem(manifest, RedirectsFileName, 106 111 "not a regular file") 112 + } else if err != nil { 113 + return err 107 114 } 108 115 109 - rules, err := redirects.ParseString(string(redirectsEntry.GetData())) 116 + rules, err := redirects.ParseString(string(data)) 110 117 if err != nil { 111 118 return AddProblem(manifest, RedirectsFileName, 112 119 "syntax error: %s", err)