[mirror] Scalable static site server for Git forges (like GitHub Pages)
whitequark.org
1package git_pages
2
3import (
4 "context"
5 "fmt"
6
7 "github.com/c2h5oh/datasize"
8 "github.com/dghubble/trie"
9)
10
11func trieReduce(data trie.Trier) (items, total int64) {
12 data.Walk(func(key string, value any) error {
13 items += 1
14 total += *value.(*int64)
15 return nil
16 })
17 return
18}
19
20func TraceGarbage(ctx context.Context) error {
21 allBlobs := trie.NewRuneTrie()
22 liveBlobs := trie.NewRuneTrie()
23
24 traceManifest := func(manifestName string, manifest *Manifest) error {
25 for _, entry := range manifest.GetContents() {
26 if entry.GetType() == Type_ExternalFile {
27 blobName := string(entry.Data)
28 if size := allBlobs.Get(blobName); size == nil {
29 return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
30 } else {
31 liveBlobs.Put(blobName, size)
32 }
33 }
34 }
35 return nil
36 }
37
38 // Enumerate all blobs.
39 for metadata, err := range backend.EnumerateBlobs(ctx) {
40 if err != nil {
41 return fmt.Errorf("trace blobs err: %w", err)
42 }
43 allBlobs.Put(metadata.Name, &metadata.Size)
44 }
45
46 // Enumerate blobs live via site manifests.
47 for metadata, err := range backend.EnumerateManifests(ctx) {
48 if err != nil {
49 return fmt.Errorf("trace sites err: %w", err)
50 }
51 manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
52 if err != nil {
53 return fmt.Errorf("trace sites err: %w", err)
54 }
55 err = traceManifest(metadata.Name, manifest)
56 if err != nil {
57 return fmt.Errorf("trace sites err: %w", err)
58 }
59 }
60
61 // Enumerate blobs live via audit records.
62 for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
63 if err != nil {
64 return fmt.Errorf("trace audit err: %w", err)
65 }
66 auditRecord, err := backend.QueryAuditLog(ctx, auditID)
67 if err != nil {
68 return fmt.Errorf("trace audit err: %w", err)
69 }
70 if auditRecord.Manifest != nil {
71 err = traceManifest(auditID.String(), auditRecord.Manifest)
72 if err != nil {
73 return fmt.Errorf("trace audit err: %w", err)
74 }
75 }
76 }
77
78 allBlobsCount, allBlobsSize := trieReduce(allBlobs)
79 liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
80 logc.Printf(ctx, "trace all: %d blobs, %s",
81 allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
82 logc.Printf(ctx, "trace live: %d blobs, %s",
83 liveBlobsCount, datasize.ByteSize(liveBlobsSize).HR())
84 logc.Printf(ctx, "trace dead: %d blobs, %s",
85 allBlobsCount-liveBlobsCount, datasize.ByteSize(allBlobsSize-liveBlobsSize).HR())
86
87 return nil
88}