lex: refactor $blob, $link, and $bytes serialization

gen/main.go

···
 		panic(err)
 	}
 
-	if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.Blob{}); err != nil {
+	if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.LegacyBlob{}, lexutil.BlobSchema{}); err != nil {
 		panic(err)
 	}
 

lex/gen.go

···
 		return "util.LexiconTypeDecoder", nil
 	case "union":
 		return "*" + name + "_" + strings.Title(k), nil
-	case "image":
-		return "*util.Blob", nil
 	case "blob":
-		return "*util.Blob", nil
+		return "*util.LexBlob", nil
 	case "array":
 		subt, err := s.typeNameForField(name+"_"+strings.Title(k), "Elem", *v.Items)
 		if err != nil {
···
 
 		return "[]" + subt, nil
 	case "cid-link":
-		return "cid.Cid", nil
+		return "util.LexLink", nil
+	case "bytes":
+		return "util.LexBytes", nil
 	default:
 		return "", fmt.Errorf("field %q in %s has unsupported type name (%s)", k, name, v.Type)
 	}

lex/util/cbor_gen.go

···
 
 	return nil
 }
-func (t *Blob) MarshalCBOR(w io.Writer) error {
+func (t *LegacyBlob) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
···
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write([]byte{163}); err != nil {
+	if _, err := cw.Write([]byte{162}); err != nil {
+		return err
+	}
+
+	// t.Cid (string) (string)
+	if len("cid") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"cid\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("cid")); err != nil {
+		return err
+	}
+
+	if len(t.Cid) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.Cid was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.Cid)); err != nil {
+		return err
+	}
+
+	// t.MimeType (string) (string)
+	if len("mimeType") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"mimeType\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("mimeType")); err != nil {
 		return err
 	}
 
-	// t.Ref (cid.Cid) (struct)
+	if len(t.MimeType) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.MimeType was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.MimeType)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *LegacyBlob) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = LegacyBlob{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("LegacyBlob: map struct too large (%d)", extra)
+	}
+
+	var name string
+	n := extra
+
+	for i := uint64(0); i < n; i++ {
+
+		{
+			sval, err := cbg.ReadString(cr)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
+		}
+
+		switch name {
+		// t.Cid (string) (string)
+		case "cid":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.Cid = string(sval)
+			}
+			// t.MimeType (string) (string)
+		case "mimeType":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.MimeType = string(sval)
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
+		}
+	}
+
+	return nil
+}
+func (t *BlobSchema) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{164}); err != nil {
+		return err
+	}
+
+	// t.Ref (util.LexLink) (struct)
 	if len("ref") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"ref\" was too long")
 	}
···
 		return err
 	}
 
-	if err := cbg.WriteCid(cw, t.Ref); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
+	if err := t.Ref.MarshalCBOR(cw); err != nil {
+		return err
 	}
 
 	// t.Size (int64) (int64)
···
 	if _, err := io.WriteString(w, string(t.MimeType)); err != nil {
 		return err
 	}
+
+	// t.LexiconTypeID (string) (string)
+	if len("LexiconTypeID") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil {
+		return err
+	}
+
+	if len(t.LexiconTypeID) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.LexiconTypeID was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil {
+		return err
+	}
 	return nil
 }
 
-func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = Blob{}
+func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = BlobSchema{}
 
 	cr := cbg.NewCborReader(r)
 
···
 	}
 
 	if extra > cbg.MaxLength {
-		return fmt.Errorf("Blob: map struct too large (%d)", extra)
+		return fmt.Errorf("BlobSchema: map struct too large (%d)", extra)
 	}
 
 	var name string
···
 		}
 
 		switch name {
-		// t.Ref (cid.Cid) (struct)
+		// t.Ref (util.LexLink) (struct)
 		case "ref":
 
 			{
 
-				c, err := cbg.ReadCid(cr)
-				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
+				if err := t.Ref.UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.Ref: %w", err)
 				}
-
-				t.Ref = c
 
 			}
 			// t.Size (int64) (int64)
···
 				}
 
 				t.MimeType = string(sval)
+			}
+			// t.LexiconTypeID (string) (string)
+		case "LexiconTypeID":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.LexiconTypeID = string(sval)
 			}
 
 		default:

lex/util/lex_types.go

···
+package util
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+
+	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+)
+
+const (
+	// TODO: this is an arbitrary size. lexicons can set more realistic limits,
+	// and we should pass those limits through and only fall back to this when
+	// undefined.
+	MAX_BYTE_ARRAY_SIZE = 128 * 1024 * 1024
+)
+
+type LexLink cid.Cid
+
+type jsonLink struct {
+	Link string `json:"$type"`
+}
+
+// convenience helper
+func (ll LexLink) String() string {
+	return cid.Cid(ll).String()
+}
+
+// convenience helper
+func (ll LexLink) Defined() bool {
+	return cid.Cid(ll).Defined()
+}
+
+func (ll *LexLink) MarshalJSON() ([]byte, error) {
+	if ll == nil || !ll.Defined() {
+		return nil, xerrors.Errorf("tried to marshal nil or undefined cid-link")
+	}
+	jl := jsonLink{
+		Link: (*cid.Cid)(ll).String(),
+	}
+	return json.Marshal(jl)
+}
+
+func (ll *LexLink) UnmarshalJSON(raw []byte) error {
+	var jl jsonLink
+	err := json.Unmarshal(raw, &jl)
+	if err != nil {
+		return xerrors.Errorf("parsing cid-link JSON: %v", err)
+	}
+	c, err := cid.Decode(jl.Link)
+	if err != nil {
+		return xerrors.Errorf("parsing cid-link CID: %v", err)
+	}
+	*ll = LexLink(c)
+	return nil
+}
+
+func (ll *LexLink) MarshalCBOR(w io.Writer) error {
+	if ll == nil || !ll.Defined() {
+		return xerrors.Errorf("tried to marshal nil or undefined cid-link")
+	}
+	cw := cbg.NewCborWriter(w)
+	if err := cbg.WriteCid(cw, cid.Cid(*ll)); err != nil {
+		return xerrors.Errorf("failed to write cid-link as CBOR: %w", err)
+	}
+	return nil
+}
+
+func (ll *LexLink) UnmarshalCBOR(r io.Reader) error {
+	cr := cbg.NewCborReader(r)
+	c, err := cbg.ReadCid(cr)
+	if err != nil {
+		return xerrors.Errorf("failed to read cid-link from CBOR: %w", err)
+	}
+	*ll = LexLink(c)
+	return nil
+}
+
+type LexBytes []byte
+
+type jsonBytes struct {
+	Bytes string `json:"$bytes"`
+}
+
+func (lb *LexBytes) MarshalJSON() ([]byte, error) {
+	if lb == nil {
+		return nil, xerrors.Errorf("tried to marshal nil $bytes")
+	}
+	jb := jsonBytes{
+		Bytes: base64.StdEncoding.EncodeToString([]byte(*lb)),
+	}
+	return json.Marshal(jb)
+}
+
+func (lb *LexBytes) UnmarshalJSON(raw []byte) error {
+	var jb jsonBytes
+	err := json.Unmarshal(raw, &jb)
+	if err != nil {
+		return xerrors.Errorf("parsing $bytes JSON: %v", err)
+	}
+	out, err := base64.StdEncoding.DecodeString(jb.Bytes)
+	if err != nil {
+		return xerrors.Errorf("parsing $bytes base64: %v", err)
+	}
+	*lb = LexBytes(out)
+	return nil
+}
+
+func (lb *LexBytes) MarshalCBOR(w io.Writer) error {
+	if lb == nil {
+		return xerrors.Errorf("tried to marshal nil or undefined $bytes")
+	}
+	cw := cbg.NewCborWriter(w)
+	if err := cbg.WriteByteArray(cw, ([]byte)(*lb)); err != nil {
+		return xerrors.Errorf("failed to write $bytes as CBOR: %w", err)
+	}
+	return nil
+}
+
+func (lb *LexBytes) UnmarshalCBOR(r io.Reader) error {
+	cr := cbg.NewCborReader(r)
+	b, err := cbg.ReadByteArray(cr, MAX_BYTE_ARRAY_SIZE)
+	if err != nil {
+		return xerrors.Errorf("failed to read $bytes from CBOR: %w", err)
+	}
+	*lb = LexBytes(b)
+	return nil
+}
+
+// used in schemas, and can represent either a legacy blob or a "new" (lex
+// refactor) blob. size=-1 indicates that this is (and should be serialized as)
+// a legacy blob (string CID, no size, etc).
+type LexBlob struct {
+	Ref      LexLink `json:"ref" cborgen:"ref"`
+	MimeType string  `json:"mimeType" cborgen:"mimeType"`
+	Size     int64   `json:"size" cborgen:"size"`
+}
+
+type LegacyBlob struct {
+	Cid      string `json:"cid" cborgen:"cid"`
+	MimeType string `json:"mimeType" cborgen:"mimeType"`
+}
+
+type BlobSchema struct {
+	LexiconTypeID string  `json:"$type,omitempty"`
+	Ref           LexLink `json:"ref" cborgen:"ref"`
+	MimeType      string  `json:"mimeType" cborgen:"mimeType"`
+	Size          int64   `json:"size" cborgen:"size"`
+}
+
+func (b *LexBlob) MarshalJSON() ([]byte, error) {
+	if b.Size < 0 {
+		lb := LegacyBlob{
+			Cid:      b.Ref.String(),
+			MimeType: b.MimeType,
+		}
+		return json.Marshal(lb)
+	} else {
+		nb := BlobSchema{
+			LexiconTypeID: "blob",
+			Ref:           b.Ref,
+			MimeType:      b.MimeType,
+			Size:          b.Size,
+		}
+		return json.Marshal(nb)
+	}
+}
+
+func (b *LexBlob) UnmarshalJSON(raw []byte) error {
+	typ, err := TypeExtract(raw)
+	if err != nil {
+		return xerrors.Errorf("parsing blob: %v", err)
+	}
+
+	if typ == "blob" {
+		var bs BlobSchema
+		err := json.Unmarshal(raw, &bs)
+		if err != nil {
+			return xerrors.Errorf("parsing blob JSON: %v", err)
+		}
+		b.Ref = bs.Ref
+		b.MimeType = bs.MimeType
+		b.Size = bs.Size
+		if bs.Size < 0 {
+			return xerrors.Errorf("parsing blob: negative size: %d", bs.Size)
+		}
+	} else {
+		var legacy *LegacyBlob
+		err := json.Unmarshal(raw, legacy)
+		if err != nil {
+			return xerrors.Errorf("parsing legacy blob: %v", err)
+		}
+		refCid, err := cid.Decode(legacy.Cid)
+		if err != nil {
+			return xerrors.Errorf("parsing CID in legacy blob: %v", err)
+		}
+		b.Ref = LexLink(refCid)
+		b.MimeType = legacy.MimeType
+		b.Size = -1
+	}
+	return nil
+}
+
+func (b *LexBlob) MarshalCBOR(w io.Writer) error {
+	if b == nil {
+		return nil
+	}
+	if b.Size < 0 {
+		lb := LegacyBlob{
+			Cid:      b.Ref.String(),
+			MimeType: b.MimeType,
+		}
+		return lb.MarshalCBOR(w)
+	} else {
+		bs := BlobSchema{
+			LexiconTypeID: "blob",
+			Ref:           b.Ref,
+			MimeType:      b.MimeType,
+			Size:          b.Size,
+		}
+		return bs.MarshalCBOR(w)
+	}
+}
+
+func (lb *LexBlob) UnmarshalCBOR(r io.Reader) error {
+	typ, b, err := CborTypeExtractReader(r)
+	if err != nil {
+		return xerrors.Errorf("parsing $blob CBOR type: %w", err)
+	}
+	*lb = LexBlob{}
+	if typ == "blob" {
+		var bs BlobSchema
+		err := bs.UnmarshalCBOR(bytes.NewReader(b))
+		if err != nil {
+			return xerrors.Errorf("parsing $blob CBOR: %v", err)
+		}
+		lb.Ref = bs.Ref
+		lb.MimeType = bs.MimeType
+		lb.Size = bs.Size
+		if bs.Size < 0 {
+			return xerrors.Errorf("parsing $blob CBOR: negative size: %d", bs.Size)
+		}
+	} else {
+		legacy := LegacyBlob{}
+		err := legacy.UnmarshalCBOR(bytes.NewReader(b))
+		if err != nil {
+			return xerrors.Errorf("parsing legacy blob CBOR: %v", err)
+		}
+		refCid, err := cid.Decode(legacy.Cid)
+		if err != nil {
+			return xerrors.Errorf("parsing CID in legacy blob CBOR: %v", err)
+		}
+		lb.Ref = LexLink(refCid)
+		lb.MimeType = legacy.MimeType
+		lb.Size = -1
+	}
+
+	return nil
+}

lex/util/util.go

···
 	"encoding/json"
 	"fmt"
 	"io"
-
-	"github.com/ipfs/go-cid"
 )
 
 type typeExtractor struct {
···
 	}
 
 	return te.Type, nil
-}
-
-type LegacyBlob struct {
-	Cid      string `json:"cid" cborgen:"cid"`
-	MimeType string `json:"mimeType" cborgen:"mimeType"`
-}
-
-type CidLink struct {
-	Cid string `json:"$link"`
-}
-
-type NewBlob struct {
-	LexiconTypeID string  `json:"$type,omitempty"`
-	Ref           CidLink `json:"ref" cborgen:"ref"`
-	MimeType      string  `json:"mimeType" cborgen:"mimeType"`
-	Size          int64   `json:"size" cborgen:"size"`
-}
-
-type Blob struct {
-	Ref      cid.Cid `json:"ref" cborgen:"ref"`
-	MimeType string  `json:"mimeType" cborgen:"mimeType"`
-	Size     int64   `json:"size" cborgen:"size"`
-}
-
-func (b *Blob) MarshalJSON() ([]byte, error) {
-	nb := NewBlob{
-		LexiconTypeID: "blob",
-		Ref:           CidLink{b.Ref.String()},
-		MimeType:      b.MimeType,
-		Size:          b.Size,
-	}
-	return json.Marshal(nb)
-}
-
-func (b *Blob) UnmarshalJSON(raw []byte) error {
-	typ, err := TypeExtract(raw)
-	if err != nil {
-		return fmt.Errorf("parsing blob: %v", err)
-	}
-
-	if typ == "blob" {
-		var nb NewBlob
-		err := json.Unmarshal(raw, &nb)
-		if err != nil {
-			return fmt.Errorf("parsing blob JSON: %v", err)
-		}
-		b.Ref, err = cid.Decode(nb.Ref.Cid)
-		if err != nil {
-			return fmt.Errorf("parsing blob CID: %v", err)
-		}
-		b.MimeType = nb.MimeType
-		b.Size = nb.Size
-	} else {
-		var legacy *LegacyBlob
-		err := json.Unmarshal(raw, legacy)
-		if err != nil {
-			return fmt.Errorf("parsing legacy blob: %v", err)
-		}
-		b.Ref, err = cid.Decode(legacy.Cid)
-		if err != nil {
-			return fmt.Errorf("parsing CID in legacy blob: %v", err)
-		}
-		b.MimeType = legacy.MimeType
-		// TODO: copying the -1 here from atproto behavior. should verify if it
-		// should be *size instead
-		b.Size = -1
-	}
-	return nil
 }
 
 type CborChecker struct {