[mirror] Scalable static site server for Git forges (like GitHub Pages)
whitequark.org
1package git_pages
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "iter"
9 "strings"
10 "time"
11)
12
13var ErrObjectNotFound = errors.New("not found")
14var ErrPreconditionFailed = errors.New("precondition failed")
15var ErrWriteConflict = errors.New("write conflict")
16var ErrDomainFrozen = errors.New("domain administratively frozen")
17
18func splitBlobName(name string) []string {
19 if algo, hash, found := strings.Cut(name, "-"); found {
20 return []string{algo, hash[0:2], hash[2:4], hash[4:]}
21 } else {
22 panic("malformed blob name")
23 }
24}
25
26func joinBlobName(parts []string) string {
27 return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], ""))
28}
29
30type BackendFeature string
31
32const (
33 FeatureCheckDomainMarker BackendFeature = "check-domain-marker"
34)
35
36type BlobMetadata struct {
37 Name string
38 Size int64
39 LastModified time.Time
40}
41
42type GetManifestOptions struct {
43 // If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns
44 // a fresh object instead of revalidating in background and returning a stale object.
45 BypassCache bool
46}
47
48type ManifestMetadata struct {
49 Name string
50 Size int64
51 LastModified time.Time
52 ETag string
53}
54
55type ModifyManifestOptions struct {
56 // If non-zero, the request will only succeed if the manifest hasn't been changed since
57 // the given time. Whether this is racy or not is can be determined via `HasAtomicCAS()`.
58 IfUnmodifiedSince time.Time
59 // If non-empty, the request will only succeed if the manifest hasn't changed from
60 // the state corresponding to the ETag. Whether this is racy or not is can be determined
61 // via `HasAtomicCAS()`.
62 IfMatch string
63}
64
65type SearchAuditLogOptions struct {
66 // Inclusive lower bound on returned audit records, per their Snowflake ID (which may differ
67 // slightly from the embedded timestamp). If zero, audit records are returned since beginning
68 // of time.
69 Since time.Time
70 // Inclusive upper bound on returned audit records, per their Snowflake ID (which may differ
71 // slightly from the embedded timestamp). If zero, audit records are returned until the end
72 // of time.
73 Until time.Time
74}
75
76type SearchAuditLogResult struct {
77 ID AuditID
78 Err error
79}
80
81type Backend interface {
82 // Returns true if the feature has been enabled for this store, false otherwise.
83 HasFeature(ctx context.Context, feature BackendFeature) bool
84
85 // Enables the feature for this store.
86 EnableFeature(ctx context.Context, feature BackendFeature) error
87
88 // Retrieve a blob. Returns `reader, size, mtime, err`.
89 GetBlob(ctx context.Context, name string) (
90 reader io.ReadSeeker, metadata BlobMetadata, err error,
91 )
92
93 // Store a blob. If a blob called `name` already exists, this function returns `nil` without
94 // regards to the old or new contents. It is expected that blobs are content-addressed, i.e.
95 // the `name` contains a cryptographic hash of `data`, but the backend is ignorant of this.
96 PutBlob(ctx context.Context, name string, data []byte) error
97
98 // Delete a blob. This is an unconditional operation that can break integrity of manifests.
99 DeleteBlob(ctx context.Context, name string) error
100
101 // Iterate through all blobs. Whether blobs that are newly added during iteration will appear
102 // in the results is unspecified.
103 EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error]
104
105 // Retrieve a manifest.
106 GetManifest(ctx context.Context, name string, opts GetManifestOptions) (
107 manifest *Manifest, metadata ManifestMetadata, err error,
108 )
109
110 // Stage a manifest. This operation stores a new version of a manifest, locking any blobs
111 // referenced from it in place (for garbage collection purposes) but without any other side
112 // effects.
113 StageManifest(ctx context.Context, manifest *Manifest) error
114
115 // Whether a compare-and-swap operation on a manifest is truly race-free, or only best-effort
116 // atomic with a small but non-zero window where two requests may race where the one committing
117 // first will have its update lost. (Plain swap operations are always guaranteed to be atomic.)
118 HasAtomicCAS(ctx context.Context) bool
119
120 // Commit a manifest. This is an atomic operation; `GetManifest` calls will return either
121 // the old version or the new version of the manifest, never anything else.
122 CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error
123
124 // Delete a manifest.
125 DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
126
127 // Iterate through all manifests. Whether manifests that are newly added during iteration
128 // will appear in the results is unspecified.
129 EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error]
130
131 // Check whether a domain has any deployments.
132 CheckDomain(ctx context.Context, domain string) (found bool, err error)
133
134 // Create a domain. This allows us to start serving content for the domain.
135 CreateDomain(ctx context.Context, domain string) error
136
137 // Freeze a domain. This allows a site to be administratively locked, e.g. if it
138 // is discovered serving abusive content.
139 FreezeDomain(ctx context.Context, domain string) error
140
141 // Thaw a domain. This removes the previously placed administrative lock (if any).
142 UnfreezeDomain(ctx context.Context, domain string) error
143
144 // Append a record to the audit log.
145 AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
146
147 // Retrieve a single record from the audit log.
148 QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error)
149
150 // Retrieve records from the audit log by time range.
151 SearchAuditLog(ctx context.Context, opts SearchAuditLogOptions) iter.Seq2[AuditID, error]
152}
153
154func CreateBackend(ctx context.Context, config *StorageConfig) (backend Backend, err error) {
155 switch config.Type {
156 case "fs":
157 if backend, err = NewFSBackend(ctx, &config.FS); err != nil {
158 err = fmt.Errorf("fs backend: %w", err)
159 }
160 case "s3":
161 if backend, err = NewS3Backend(ctx, &config.S3); err != nil {
162 err = fmt.Errorf("s3 backend: %w", err)
163 }
164 default:
165 err = fmt.Errorf("unknown backend: %s", config.Type)
166 }
167 backend = NewAuditedBackend(backend)
168 return
169}