fork of whitequark.org/git-pages with mods for tangled

Add original (decompressed) size to site manifest.

This size is not used by git-pages itself, and is not representative of
storage needs, but may be used for estimating how large a site would
be if downloaded in its entirety.

Changed files
+45 -34
src
+19 -17
src/manifest.go
··· 167 167 span, _ := ObserveFunction(ctx, "CompressFiles") 168 168 defer span.Finish() 169 169 170 - var originalSize, transformedSize int64 170 + var originalSize, compressedSize int64 171 171 for _, entry := range manifest.Contents { 172 172 if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None { 173 173 mtype := getMediaType(entry.GetContentType()) ··· 181 181 entry.Size = proto.Int64(int64(len(entry.Data))) 182 182 entry.Transform = Transform_Zstandard.Enum() 183 183 } 184 - transformedSize += entry.GetSize() 184 + compressedSize += entry.GetSize() 185 185 } 186 186 } 187 + manifest.OriginalSize = proto.Int64(originalSize) 188 + manifest.CompressedSize = proto.Int64(compressedSize) 187 189 188 - spaceSaving := (float64(originalSize) - float64(transformedSize)) / float64(originalSize) 190 + spaceSaving := (float64(originalSize) - float64(compressedSize)) / float64(originalSize) 189 191 log.Printf("compress: saved %.2f percent (%s to %s)", 190 192 spaceSaving*100.0, 191 193 datasize.ByteSize(originalSize).HR(), 192 - datasize.ByteSize(transformedSize).HR(), 194 + datasize.ByteSize(compressedSize).HR(), 193 195 ) 194 196 siteCompressionSpaceSaving. 195 197 Observe(spaceSaving) ··· 232 234 233 235 // Replace inline files over certain size with references to external data. 234 236 extManifest := Manifest{ 235 - RepoUrl: manifest.RepoUrl, 236 - Branch: manifest.Branch, 237 - Commit: manifest.Commit, 238 - Contents: make(map[string]*Entry), 239 - Redirects: manifest.Redirects, 240 - Headers: manifest.Headers, 241 - Problems: manifest.Problems, 242 - TotalSize: proto.Int64(0), 243 - StoredSize: proto.Int64(0), 237 + RepoUrl: manifest.RepoUrl, 238 + Branch: manifest.Branch, 239 + Commit: manifest.Commit, 240 + Contents: make(map[string]*Entry), 241 + Redirects: manifest.Redirects, 242 + Headers: manifest.Headers, 243 + Problems: manifest.Problems, 244 + OriginalSize: manifest.OriginalSize, 245 + CompressedSize: manifest.CompressedSize, 246 + StoredSize: proto.Int64(0), 244 247 } 245 - extObjectMap := make(map[string]int64) 248 + extObjectSizes := make(map[string]int64) 246 249 for name, entry := range manifest.Contents { 247 250 cannotBeInlined := entry.GetType() == Type_InlineFile && 248 251 entry.GetSize() > int64(config.Limits.MaxInlineFileSize.Bytes()) ··· 255 258 Transform: entry.Transform, 256 259 ContentType: entry.ContentType, 257 260 } 258 - extObjectMap[string(dataHash[:])] = *entry.Size 261 + extObjectSizes[string(dataHash[:])] = entry.GetSize() 259 262 } else { 260 263 extManifest.Contents[name] = entry 261 264 } 262 - *extManifest.TotalSize += entry.GetSize() 263 265 } 264 266 // `extObjectMap` stores size once per object, deduplicating it 265 - for _, storedSize := range extObjectMap { 267 + for _, storedSize := range extObjectSizes { 266 268 *extManifest.StoredSize += storedSize 267 269 } 268 270
+22 -14
src/schema.pb.go
··· 7 7 package git_pages 8 8 9 9 import ( 10 + protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 + protoimpl "google.golang.org/protobuf/runtime/protoimpl" 10 12 reflect "reflect" 11 13 sync "sync" 12 14 unsafe "unsafe" 13 - 14 - protoreflect "google.golang.org/protobuf/reflect/protoreflect" 15 - protoimpl "google.golang.org/protobuf/runtime/protoimpl" 16 15 ) 17 16 18 17 const ( ··· 452 451 Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"` 453 452 Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"` 454 453 // Contents 455 - Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` 456 - TotalSize *int64 `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"` // simple sum of each `entry.size` 457 - StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // external objects, after deduplication 454 + Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` 455 + OriginalSize *int64 `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"` // total size of entries before compression 456 + CompressedSize *int64 `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // simple sum of each `entry.size` 457 + StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // total size of (deduplicated) external objects 458 458 // Netlify-style `_redirects` and `_headers` 459 459 Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"` 460 460 Headers []*HeaderRule `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"` ··· 522 522 return nil 523 523 } 524 524 525 - func (x *Manifest) GetTotalSize() int64 { 526 - if x != nil && x.TotalSize != nil { 527 - return *x.TotalSize 525 + func (x *Manifest) GetOriginalSize() int64 { 526 + if x != nil && x.OriginalSize != nil { 527 + return *x.OriginalSize 528 + } 529 + return 0 530 + } 531 + 532 + func (x *Manifest) GetCompressedSize() int64 { 533 + if x != nil && x.CompressedSize != nil { 534 + return *x.CompressedSize 528 535 } 529 536 return 0 530 537 } ··· 584 591 "header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"3\n" + 585 592 "\aProblem\x12\x12\n" + 586 593 "\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" + 587 - "\x05cause\x18\x02 \x01(\tR\x05cause\"\x89\x03\n" + 594 + "\x05cause\x18\x02 \x01(\tR\x05cause\"\xb8\x03\n" + 588 595 "\bManifest\x12\x19\n" + 589 596 "\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" + 590 597 "\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" + 591 598 "\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" + 592 - "\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12\x1d\n" + 593 - "\n" + 594 - "total_size\x18\x05 \x01(\x03R\ttotalSize\x12\x1f\n" + 599 + "\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12#\n" + 600 + "\roriginal_size\x18\n" + 601 + " \x01(\x03R\foriginalSize\x12'\n" + 602 + "\x0fcompressed_size\x18\x05 \x01(\x03R\x0ecompressedSize\x12\x1f\n" + 595 603 "\vstored_size\x18\b \x01(\x03R\n" + 596 604 "storedSize\x12+\n" + 597 605 "\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" + ··· 609 617 "\aSymlink\x10\x04*$\n" + 610 618 "\tTransform\x12\b\n" + 611 619 "\x04None\x10\x00\x12\r\n" + 612 - "\tZstandard\x10\x01B'Z%codeberg.org/git-pages/git-pages/mainb\beditionsp\xe8\a" 620 + "\tZstandard\x10\x01B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" 613 621 614 622 var ( 615 623 file_schema_proto_rawDescOnce sync.Once
+4 -3
src/schema.proto
··· 1 1 edition = "2023"; 2 2 3 - option go_package = "codeberg.org/git-pages/git-pages/main"; 3 + option go_package = "codeberg.org/git-pages/git-pages/git_pages"; 4 4 5 5 enum Type { 6 6 // Invalid entry. ··· 75 75 76 76 // Contents 77 77 map<string, Entry> contents = 4; 78 - int64 total_size = 5; // simple sum of each `entry.size` 79 - int64 stored_size = 8; // external objects, after deduplication 78 + int64 original_size = 10; // total size of entries before compression 79 + int64 compressed_size = 5; // simple sum of each `entry.size` 80 + int64 stored_size = 8; // total size of (deduplicated) external objects 80 81 81 82 // Netlify-style `_redirects` and `_headers` 82 83 repeated RedirectRule redirects = 6;