lexicons,appview/pulls: upload patches as blobs

By using blobs we massively increase our maximum patch size. When stored
directly on the record patches can be at most 2 MiB. By moving it to a
blob, we get a minimum of 50 MiB (smaller if the pds has set it but 50
is the default).

legacy field name `patch` in `repo.pull` lexicon is preserved (but not
used) to support some level of backward compatibility

Signed-off-by: Samuel Shuert <[e-mail]>
Signed-off-by: Seongmin Lee <[e-mail]>

api/tangled/cbor_gen.go

···
 	}
 
 	cw := cbg.NewCborWriter(w)
-	fieldCount := 9
+	fieldCount := 10
 
 	if t.Body == nil {
 		fieldCount--
 	}
 
 	if t.Mentions == nil {
+		fieldCount--
+	}
+
+	if t.Patch == nil {
 		fieldCount--
 	}
 
···
 	}
 
 	// t.Patch (string) (string)
-	if len("patch") > 1000000 {
-		return xerrors.Errorf("Value in field \"patch\" was too long")
-	}
+	if t.Patch != nil {
 
-	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patch"))); err != nil {
-		return err
-	}
-	if _, err := cw.WriteString(string("patch")); err != nil {
-		return err
-	}
+		if len("patch") > 1000000 {
+			return xerrors.Errorf("Value in field \"patch\" was too long")
+		}
 
-	if len(t.Patch) > 1000000 {
-		return xerrors.Errorf("Value in field t.Patch was too long")
-	}
+		if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patch"))); err != nil {
+			return err
+		}
+		if _, err := cw.WriteString(string("patch")); err != nil {
+			return err
+		}
+
+		if t.Patch == nil {
+			if _, err := cw.Write(cbg.CborNull); err != nil {
+				return err
+			}
+		} else {
+			if len(*t.Patch) > 1000000 {
+				return xerrors.Errorf("Value in field t.Patch was too long")
+			}
 
-	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Patch))); err != nil {
-		return err
-	}
-	if _, err := cw.WriteString(string(t.Patch)); err != nil {
-		return err
+			if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Patch))); err != nil {
+				return err
+			}
+			if _, err := cw.WriteString(string(*t.Patch)); err != nil {
+				return err
+			}
+		}
 	}
 
 	// t.Title (string) (string)
···
 		return err
 	}
 
+	// t.PatchBlob (util.LexBlob) (struct)
+	if len("patchBlob") > 1000000 {
+		return xerrors.Errorf("Value in field \"patchBlob\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("patchBlob"))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string("patchBlob")); err != nil {
+		return err
+	}
+
+	if err := t.PatchBlob.MarshalCBOR(cw); err != nil {
+		return err
+	}
+
 	// t.References ([]string) (slice)
 	if t.References != nil {
 
···
 		case "patch":
 
 			{
-				sval, err := cbg.ReadStringWithMax(cr, 1000000)
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
 
-				t.Patch = string(sval)
+					sval, err := cbg.ReadStringWithMax(cr, 1000000)
+					if err != nil {
+						return err
+					}
+
+					t.Patch = (*string)(&sval)
+				}
 			}
 			// t.Title (string) (string)
 		case "title":
···
 				}
 
 				t.CreatedAt = string(sval)
+			}
+			// t.PatchBlob (util.LexBlob) (struct)
+		case "patchBlob":
+
+			{
+
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+					t.PatchBlob = new(util.LexBlob)
+					if err := t.PatchBlob.UnmarshalCBOR(cr); err != nil {
+						return xerrors.Errorf("unmarshaling t.PatchBlob pointer: %w", err)
+					}
+				}
+
 			}
 			// t.References ([]string) (slice)
 		case "references":

api/tangled/repopull.go

···
 } //
 // RECORDTYPE: RepoPull
 type RepoPull struct {
-	LexiconTypeID string           `json:"$type,const=sh.tangled.repo.pull" cborgen:"$type,const=sh.tangled.repo.pull"`
-	Body          *string          `json:"body,omitempty" cborgen:"body,omitempty"`
-	CreatedAt     string           `json:"createdAt" cborgen:"createdAt"`
-	Mentions      []string         `json:"mentions,omitempty" cborgen:"mentions,omitempty"`
-	Patch         string           `json:"patch" cborgen:"patch"`
-	References    []string         `json:"references,omitempty" cborgen:"references,omitempty"`
-	Source        *RepoPull_Source `json:"source,omitempty" cborgen:"source,omitempty"`
-	Target        *RepoPull_Target `json:"target" cborgen:"target"`
-	Title         string           `json:"title" cborgen:"title"`
+	LexiconTypeID string   `json:"$type,const=sh.tangled.repo.pull" cborgen:"$type,const=sh.tangled.repo.pull"`
+	Body          *string  `json:"body,omitempty" cborgen:"body,omitempty"`
+	CreatedAt     string   `json:"createdAt" cborgen:"createdAt"`
+	Mentions      []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"`
+	// patch: (deprecated) use patchBlob instead
+	Patch *string `json:"patch,omitempty" cborgen:"patch,omitempty"`
+	// patchBlob: patch content
+	PatchBlob  *util.LexBlob    `json:"patchBlob" cborgen:"patchBlob"`
+	References []string         `json:"references,omitempty" cborgen:"references,omitempty"`
+	Source     *RepoPull_Source `json:"source,omitempty" cborgen:"source,omitempty"`
+	Target     *RepoPull_Target `json:"target" cborgen:"target"`
+	Title      string           `json:"title" cborgen:"title"`
 }
 
 // RepoPull_Source is a "source" in the sh.tangled.repo.pull schema.

appview/models/pull.go

···
 	Repo   *Repo
 }
 
+// NOTE: This method does not include patch blob in returned atproto record
 func (p Pull) AsRecord() tangled.RepoPull {
 	var source *tangled.RepoPull_Source
 	if p.PullSource != nil {
···
 			Repo:   p.RepoAt.String(),
 			Branch: p.TargetBranch,
 		},
-		Patch:  p.LatestPatch(),
 		Source: source,
 	}
 	return record

appview/pulls/pulls.go

···
 		return
 	}
 
+	blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
+	if err != nil {
+		log.Println("failed to upload patch", err)
+		s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.")
+		return
+	}
+
 	_, err = comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
 		Collection: tangled.RepoPullNSID,
 		Repo:       user.Did,
···
 					Repo:   string(repo.RepoAt()),
 					Branch: targetBranch,
 				},
-				Patch:     patch,
+				PatchBlob: blob.Blob,
 				Source:    recordPullSource,
 				CreatedAt: time.Now().Format(time.RFC3339),
 			},
···
 	// apply all record creations at once
 	var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
 	for _, p := range stack {
+		blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(p.LatestPatch()))
+		if err != nil {
+			log.Println("failed to upload patch blob", err)
+			s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.")
+			return
+		}
+
 		record := p.AsRecord()
-		write := comatproto.RepoApplyWrites_Input_Writes_Elem{
+		record.PatchBlob = blob.Blob
+		writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
 			RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{
 				Collection: tangled.RepoPullNSID,
 				Rkey:       &p.Rkey,
···
 					Val: &record,
 				},
 			},
-		}
-		writes = append(writes, &write)
+		})
 	}
 	_, err = comatproto.RepoApplyWrites(r.Context(), client, &comatproto.RepoApplyWrites_Input{
 		Repo:   user.Did,
···
 		return
 	}
 
-	var recordPullSource *tangled.RepoPull_Source
-	if pull.IsBranchBased() {
-		recordPullSource = &tangled.RepoPull_Source{
-			Branch: pull.PullSource.Branch,
-			Sha:    sourceRev,
-		}
+	blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
+	if err != nil {
+		log.Println("failed to upload patch blob", err)
+		s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
+		return
 	}
-	if pull.IsForkBased() {
-		repoAt := pull.PullSource.RepoAt.String()
-		recordPullSource = &tangled.RepoPull_Source{
-			Branch: pull.PullSource.Branch,
-			Repo:   &repoAt,
-			Sha:    sourceRev,
-		}
-	}
+	record := pull.AsRecord()
+	record.PatchBlob = blob.Blob
+	record.CreatedAt = time.Now().Format(time.RFC3339)
 
 	_, err = comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
 		Collection: tangled.RepoPullNSID,
···
 		Rkey:       pull.Rkey,
 		SwapRecord: ex.Cid,
 		Record: &lexutil.LexiconTypeDecoder{
-			Val: &tangled.RepoPull{
-				Title: pull.Title,
-				Target: &tangled.RepoPull_Target{
-					Repo:   string(repo.RepoAt()),
-					Branch: pull.TargetBranch,
-				},
-				Patch:     patch, // new patch
-				Source:    recordPullSource,
-				CreatedAt: time.Now().Format(time.RFC3339),
-			},
+			Val: &record,
 		},
 	})
 	if err != nil {
···
 	}
 	defer tx.Rollback()
 
+	client, err := s.oauth.AuthorizedClient(r)
+	if err != nil {
+		log.Println("failed to authorize client")
+		s.pages.Notice(w, "resubmit-error", "Failed to create pull request. Try again later.")
+		return
+	}
+
 	// pds updates to make
 	var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
 
···
 			return
 		}
 
+		blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
+		if err != nil {
+			log.Println("failed to upload patch blob", err)
+			s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
+			return
+		}
 		record := p.AsRecord()
+		record.PatchBlob = blob.Blob
 		writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
 			RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{
 				Collection: tangled.RepoPullNSID,
···
 			return
 		}
 
+		blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
+		if err != nil {
+			log.Println("failed to upload patch blob", err)
+			s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
+			return
+		}
 		record := np.AsRecord()
-
+		record.PatchBlob = blob.Blob
 		writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
 			RepoApplyWrites_Update: &comatproto.RepoApplyWrites_Update{
 				Collection: tangled.RepoPullNSID,
···
 	if err != nil {
 		log.Println("failed to resubmit pull", err)
 		s.pages.Notice(w, "pull-resubmit-error", "Failed to resubmit pull request. Try again later.")
-		return
-	}
-
-	client, err := s.oauth.AuthorizedClient(r)
-	if err != nil {
-		log.Println("failed to authorize client")
-		s.pages.Notice(w, "resubmit-error", "Failed to create pull request. Try again later.")
 		return
 	}
 

lexicons/pulls/pull.json

···
         "required": [
           "target",
           "title",
-          "patch",
+          "patchBlob",
           "createdAt"
         ],
         "properties": {
···
             "type": "string"
           },
           "patch": {
-            "type": "string"
+            "type": "string",
+            "description": "(deprecated) use patchBlob instead"
+          },
+          "patchBlob": {
+            "type": "blob",
+            "accept": "text/x-patch",
+            "description": "patch content"
           },
           "source": {
             "type": "ref",