back interdiff of round #1 and #0

appview/pulls: upload patches as blobs #870

merged
opened by boltless.me targeting master from patches-as-blobs

By using blobs we massively increase our maximum patch size. When stored directly on the record patches can be at most 2 MiB. By moving it to a blob, we get a minimum of 50 MiB (smaller if the pds has set it but 50 is the default).

Signed-off-by: Samuel Shuert me@thecoded.prof Signed-off-by: Seongmin Lee git@boltless.me

files
api
appview
models
pulls
lexicons
pulls
ERROR
appview/models/pull.go

Failed to calculate interdiff for this file.

REBASED
appview/pulls/pulls.go

This patch was likely rebased, as context lines do not match.

NEW
api/tangled/cbor_gen.go
··· 7934 } 7935 7936 cw := cbg.NewCborWriter(w) 7937 - fieldCount := 9 7938 7939 if t.Body == nil { 7940 fieldCount-- ··· 7944 fieldCount-- 7945 } 7946 7947 if t.References == nil { 7948 fieldCount-- 7949 } ··· 8008 } 8009 8010 // t.Patch (string) (string) 8011 - if len("patch") > 1000000 { 8012 - return xerrors.Errorf("Value in field \"patch\" was too long") 8013 - } 8014 8015 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patch"))); err != nil { 8016 - return err 8017 - } 8018 - if _, err := cw.WriteString(string("patch")); err != nil { 8019 - return err 8020 - } 8021 8022 - if len(t.Patch) > 1000000 { 8023 - return xerrors.Errorf("Value in field t.Patch was too long") 8024 - } 8025 8026 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Patch))); err != nil { 8027 - return err 8028 - } 8029 - if _, err := cw.WriteString(string(t.Patch)); err != nil { 8030 - return err 8031 } 8032 8033 // t.Title (string) (string) ··· 8147 return err 8148 } 8149 8150 // t.References ([]string) (slice) 8151 if t.References != nil { 8152 ··· 8262 case "patch": 8263 8264 { 8265 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 8266 if err != nil { 8267 return err 8268 } 8269 8270 - t.Patch = string(sval) 8271 } 8272 // t.Title (string) (string) 8273 case "title": ··· 8371 8372 t.CreatedAt = string(sval) 8373 } 8374 // t.References ([]string) (slice) 8375 case "references": 8376
··· 7934 } 7935 7936 cw := cbg.NewCborWriter(w) 7937 + fieldCount := 10 7938 7939 if t.Body == nil { 7940 fieldCount-- ··· 7944 fieldCount-- 7945 } 7946 7947 + if t.Patch == nil { 7948 + fieldCount-- 7949 + } 7950 + 7951 if t.References == nil { 7952 fieldCount-- 7953 } ··· 8012 } 8013 8014 // t.Patch (string) (string) 8015 + if t.Patch != nil { 8016 8017 + if len("patch") > 1000000 { 8018 + return xerrors.Errorf("Value in field \"patch\" was too long") 8019 + } 8020 8021 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patch"))); err != nil { 8022 + return err 8023 + } 8024 + if _, err := cw.WriteString(string("patch")); err != nil { 8025 + return err 8026 + } 8027 8028 + if t.Patch == nil { 8029 + if _, err := cw.Write(cbg.CborNull); err != nil { 8030 + return err 8031 + } 8032 + } else { 8033 + if len(*t.Patch) > 1000000 { 8034 + return xerrors.Errorf("Value in field t.Patch was too long") 8035 + } 8036 + 8037 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Patch))); err != nil { 8038 + return err 8039 + } 8040 + if _, err := cw.WriteString(string(*t.Patch)); err != nil { 8041 + return err 8042 + } 8043 + } 8044 } 8045 8046 // t.Title (string) (string) ··· 8160 return err 8161 } 8162 8163 + // t.PatchBlob (util.LexBlob) (struct) 8164 + if len("patchBlob") > 1000000 { 8165 + return xerrors.Errorf("Value in field \"patchBlob\" was too long") 8166 + } 8167 + 8168 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patchBlob"))); err != nil { 8169 + return err 8170 + } 8171 + if _, err := cw.WriteString(string("patchBlob")); err != nil { 8172 + return err 8173 + } 8174 + 8175 + if err := t.PatchBlob.MarshalCBOR(cw); err != nil { 8176 + return err 8177 + } 8178 + 8179 // t.References ([]string) (slice) 8180 if t.References != nil { 8181 ··· 8291 case "patch": 8292 8293 { 8294 + b, err := cr.ReadByte() 8295 if err != nil { 8296 return err 8297 } 8298 + if b != cbg.CborNull[0] { 8299 + if err := cr.UnreadByte(); err != nil { 8300 + return err 8301 + } 8302 + 8303 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 8304 + if err != nil { 8305 + return err 8306 + } 8307 8308 + t.Patch = (*string)(&sval) 8309 + } 8310 } 8311 // t.Title (string) (string) 8312 case "title": ··· 8410 8411 t.CreatedAt = string(sval) 8412 } 8413 + // t.PatchBlob (util.LexBlob) (struct) 8414 + case "patchBlob": 8415 + 8416 + { 8417 + 8418 + b, err := cr.ReadByte() 8419 + if err != nil { 8420 + return err 8421 + } 8422 + if b != cbg.CborNull[0] { 8423 + if err := cr.UnreadByte(); err != nil { 8424 + return err 8425 + } 8426 + t.PatchBlob = new(util.LexBlob) 8427 + if err := t.PatchBlob.UnmarshalCBOR(cr); err != nil { 8428 + return xerrors.Errorf("unmarshaling t.PatchBlob pointer: %w", err) 8429 + } 8430 + } 8431 + 8432 + } 8433 // t.References ([]string) (slice) 8434 case "references": 8435
NEW
api/tangled/repopull.go
··· 17 } // 18 // RECORDTYPE: RepoPull 19 type RepoPull struct { 20 - LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull" cborgen:"$type,const=sh.tangled.repo.pull"` 21 - Body *string `json:"body,omitempty" cborgen:"body,omitempty"` 22 - CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 - Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` 24 - Patch string `json:"patch" cborgen:"patch"` 25 - References []string `json:"references,omitempty" cborgen:"references,omitempty"` 26 - Source *RepoPull_Source `json:"source,omitempty" cborgen:"source,omitempty"` 27 - Target *RepoPull_Target `json:"target" cborgen:"target"` 28 - Title string `json:"title" cborgen:"title"` 29 } 30 31 // RepoPull_Source is a "source" in the sh.tangled.repo.pull schema.
··· 17 } // 18 // RECORDTYPE: RepoPull 19 type RepoPull struct { 20 + LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull" cborgen:"$type,const=sh.tangled.repo.pull"` 21 + Body *string `json:"body,omitempty" cborgen:"body,omitempty"` 22 + CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` 24 + // patch: (deprecated) use patchBlob instead 25 + Patch *string `json:"patch,omitempty" cborgen:"patch,omitempty"` 26 + // patchBlob: patch content 27 + PatchBlob *util.LexBlob `json:"patchBlob" cborgen:"patchBlob"` 28 + References []string `json:"references,omitempty" cborgen:"references,omitempty"` 29 + Source *RepoPull_Source `json:"source,omitempty" cborgen:"source,omitempty"` 30 + Target *RepoPull_Target `json:"target" cborgen:"target"` 31 + Title string `json:"title" cborgen:"title"` 32 } 33 34 // RepoPull_Source is a "source" in the sh.tangled.repo.pull schema.
NEW
lexicons/pulls/pull.json
··· 12 "required": [ 13 "target", 14 "title", 15 - "patch", 16 "createdAt" 17 ], 18 "properties": { ··· 27 "type": "string" 28 }, 29 "patch": { 30 - "type": "string" 31 }, 32 "source": { 33 "type": "ref",
··· 12 "required": [ 13 "target", 14 "title", 15 + "patchBlob", 16 "createdAt" 17 ], 18 "properties": { ··· 27 "type": "string" 28 }, 29 "patch": { 30 + "type": "string", 31 + "description": "(deprecated) use patchBlob instead" 32 + }, 33 + "patchBlob": { 34 + "type": "blob", 35 + "accept": "text/x-patch", 36 + "description": "patch content" 37 }, 38 "source": { 39 "type": "ref",