+19
-17
src/manifest.go
+19
-17
src/manifest.go
···
167
167
span, _ := ObserveFunction(ctx, "CompressFiles")
168
168
defer span.Finish()
169
169
170
-
var originalSize, transformedSize int64
170
+
var originalSize, compressedSize int64
171
171
for _, entry := range manifest.Contents {
172
172
if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None {
173
173
mtype := getMediaType(entry.GetContentType())
···
181
181
entry.Size = proto.Int64(int64(len(entry.Data)))
182
182
entry.Transform = Transform_Zstandard.Enum()
183
183
}
184
-
transformedSize += entry.GetSize()
184
+
compressedSize += entry.GetSize()
185
185
}
186
186
}
187
+
manifest.OriginalSize = proto.Int64(originalSize)
188
+
manifest.CompressedSize = proto.Int64(compressedSize)
187
189
188
-
spaceSaving := (float64(originalSize) - float64(transformedSize)) / float64(originalSize)
190
+
spaceSaving := (float64(originalSize) - float64(compressedSize)) / float64(originalSize)
189
191
log.Printf("compress: saved %.2f percent (%s to %s)",
190
192
spaceSaving*100.0,
191
193
datasize.ByteSize(originalSize).HR(),
192
-
datasize.ByteSize(transformedSize).HR(),
194
+
datasize.ByteSize(compressedSize).HR(),
193
195
)
194
196
siteCompressionSpaceSaving.
195
197
Observe(spaceSaving)
···
232
234
233
235
// Replace inline files over certain size with references to external data.
234
236
extManifest := Manifest{
235
-
RepoUrl: manifest.RepoUrl,
236
-
Branch: manifest.Branch,
237
-
Commit: manifest.Commit,
238
-
Contents: make(map[string]*Entry),
239
-
Redirects: manifest.Redirects,
240
-
Headers: manifest.Headers,
241
-
Problems: manifest.Problems,
242
-
TotalSize: proto.Int64(0),
243
-
StoredSize: proto.Int64(0),
237
+
RepoUrl: manifest.RepoUrl,
238
+
Branch: manifest.Branch,
239
+
Commit: manifest.Commit,
240
+
Contents: make(map[string]*Entry),
241
+
Redirects: manifest.Redirects,
242
+
Headers: manifest.Headers,
243
+
Problems: manifest.Problems,
244
+
OriginalSize: manifest.OriginalSize,
245
+
CompressedSize: manifest.CompressedSize,
246
+
StoredSize: proto.Int64(0),
244
247
}
245
-
extObjectMap := make(map[string]int64)
248
+
extObjectSizes := make(map[string]int64)
246
249
for name, entry := range manifest.Contents {
247
250
cannotBeInlined := entry.GetType() == Type_InlineFile &&
248
251
entry.GetSize() > int64(config.Limits.MaxInlineFileSize.Bytes())
···
255
258
Transform: entry.Transform,
256
259
ContentType: entry.ContentType,
257
260
}
258
-
extObjectMap[string(dataHash[:])] = *entry.Size
261
+
extObjectSizes[string(dataHash[:])] = entry.GetSize()
259
262
} else {
260
263
extManifest.Contents[name] = entry
261
264
}
262
-
*extManifest.TotalSize += entry.GetSize()
263
265
}
264
266
// `extObjectMap` stores size once per object, deduplicating it
265
-
for _, storedSize := range extObjectMap {
267
+
for _, storedSize := range extObjectSizes {
266
268
*extManifest.StoredSize += storedSize
267
269
}
268
270
+22
-14
src/schema.pb.go
+22
-14
src/schema.pb.go
···
7
7
package git_pages
8
8
9
9
import (
10
+
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
11
+
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
10
12
reflect "reflect"
11
13
sync "sync"
12
14
unsafe "unsafe"
13
-
14
-
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
15
-
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
16
15
)
17
16
18
17
const (
···
452
451
Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
453
452
Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
454
453
// Contents
455
-
Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
456
-
TotalSize *int64 `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"` // simple sum of each `entry.size`
457
-
StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // external objects, after deduplication
454
+
Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
455
+
OriginalSize *int64 `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"` // total size of entries before compression
456
+
CompressedSize *int64 `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // simple sum of each `entry.size`
457
+
StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // total size of (deduplicated) external objects
458
458
// Netlify-style `_redirects` and `_headers`
459
459
Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
460
460
Headers []*HeaderRule `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
···
522
522
return nil
523
523
}
524
524
525
-
func (x *Manifest) GetTotalSize() int64 {
526
-
if x != nil && x.TotalSize != nil {
527
-
return *x.TotalSize
525
+
func (x *Manifest) GetOriginalSize() int64 {
526
+
if x != nil && x.OriginalSize != nil {
527
+
return *x.OriginalSize
528
+
}
529
+
return 0
530
+
}
531
+
532
+
func (x *Manifest) GetCompressedSize() int64 {
533
+
if x != nil && x.CompressedSize != nil {
534
+
return *x.CompressedSize
528
535
}
529
536
return 0
530
537
}
···
584
591
"header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"3\n" +
585
592
"\aProblem\x12\x12\n" +
586
593
"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
587
-
"\x05cause\x18\x02 \x01(\tR\x05cause\"\x89\x03\n" +
594
+
"\x05cause\x18\x02 \x01(\tR\x05cause\"\xb8\x03\n" +
588
595
"\bManifest\x12\x19\n" +
589
596
"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
590
597
"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
591
598
"\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" +
592
-
"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12\x1d\n" +
593
-
"\n" +
594
-
"total_size\x18\x05 \x01(\x03R\ttotalSize\x12\x1f\n" +
599
+
"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12#\n" +
600
+
"\roriginal_size\x18\n" +
601
+
" \x01(\x03R\foriginalSize\x12'\n" +
602
+
"\x0fcompressed_size\x18\x05 \x01(\x03R\x0ecompressedSize\x12\x1f\n" +
595
603
"\vstored_size\x18\b \x01(\x03R\n" +
596
604
"storedSize\x12+\n" +
597
605
"\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" +
···
609
617
"\aSymlink\x10\x04*$\n" +
610
618
"\tTransform\x12\b\n" +
611
619
"\x04None\x10\x00\x12\r\n" +
612
-
"\tZstandard\x10\x01B'Z%codeberg.org/git-pages/git-pages/mainb\beditionsp\xe8\a"
620
+
"\tZstandard\x10\x01B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"
613
621
614
622
var (
615
623
file_schema_proto_rawDescOnce sync.Once
+4
-3
src/schema.proto
+4
-3
src/schema.proto
···
1
1
edition = "2023";
2
2
3
-
option go_package = "codeberg.org/git-pages/git-pages/main";
3
+
option go_package = "codeberg.org/git-pages/git-pages/git_pages";
4
4
5
5
enum Type {
6
6
// Invalid entry.
···
75
75
76
76
// Contents
77
77
map<string, Entry> contents = 4;
78
-
int64 total_size = 5; // simple sum of each `entry.size`
79
-
int64 stored_size = 8; // external objects, after deduplication
78
+
int64 original_size = 10; // total size of entries before compression
79
+
int64 compressed_size = 5; // simple sum of each `entry.size`
80
+
int64 stored_size = 8; // total size of (deduplicated) external objects
80
81
81
82
// Netlify-style `_redirects` and `_headers`
82
83
repeated RedirectRule redirects = 6;