appview,lexicons: atprotate the mentions & references #761

merged
opened by boltless.me targeting master from feat/mentions

Storing references parsed from the markdown body in atproto record and DB. There can be lots of reference types considering the from/to types so storing both as AT-URIs

Using sql.Tx more to combine multiple DB query to single recoverable operation.

Note: Pulls don't have mentinos/references yet

Signed-off-by: Seongmin Lee git@boltless.me

+487 -6
api/tangled/cbor_gen.go
··· 6744 6744 } 6745 6745 6746 6746 cw := cbg.NewCborWriter(w) 6747 - fieldCount := 5 6747 + fieldCount := 7 6748 6748 6749 6749 if t.Body == nil { 6750 6750 fieldCount-- 6751 6751 } 6752 6752 6753 + if t.Mentions == nil { 6754 + fieldCount-- 6755 + } 6756 + 6757 + if t.References == nil { 6758 + fieldCount-- 6759 + } 6760 + 6753 6761 if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 6754 6762 return err 6755 6763 } ··· 6851 6859 return err 6852 6860 } 6853 6861 6862 + // t.Mentions ([]string) (slice) 6863 + if t.Mentions != nil { 6864 + 6865 + if len("mentions") > 1000000 { 6866 + return xerrors.Errorf("Value in field \"mentions\" was too long") 6867 + } 6868 + 6869 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { 6870 + return err 6871 + } 6872 + if _, err := cw.WriteString(string("mentions")); err != nil { 6873 + return err 6874 + } 6875 + 6876 + if len(t.Mentions) > 8192 { 6877 + return xerrors.Errorf("Slice value in field t.Mentions was too long") 6878 + } 6879 + 6880 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { 6881 + return err 6882 + } 6883 + for _, v := range t.Mentions { 6884 + if len(v) > 1000000 { 6885 + return xerrors.Errorf("Value in field v was too long") 6886 + } 6887 + 6888 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 6889 + return err 6890 + } 6891 + if _, err := cw.WriteString(string(v)); err != nil { 6892 + return err 6893 + } 6894 + 6895 + } 6896 + } 6897 + 6854 6898 // t.CreatedAt (string) (string) 6855 6899 if len("createdAt") > 1000000 { 6856 6900 return xerrors.Errorf("Value in field \"createdAt\" was too long") ··· 6873 6917 if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { 6874 6918 return err 6875 6919 } 6920 + 6921 + // t.References ([]string) (slice) 6922 + if t.References != nil { 6923 + 6924 + if len("references") > 1000000 { 6925 + return xerrors.Errorf("Value in field \"references\" was too long") 6926 + } 6927 + 6928 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { 6929 + return err 6930 + } 6931 + if _, err := cw.WriteString(string("references")); err != nil { 6932 + return err 6933 + } 6934 + 6935 + if len(t.References) > 8192 { 6936 + return xerrors.Errorf("Slice value in field t.References was too long") 6937 + } 6938 + 6939 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { 6940 + return err 6941 + } 6942 + for _, v := range t.References { 6943 + if len(v) > 1000000 { 6944 + return xerrors.Errorf("Value in field v was too long") 6945 + } 6946 + 6947 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 6948 + return err 6949 + } 6950 + if _, err := cw.WriteString(string(v)); err != nil { 6951 + return err 6952 + } 6953 + 6954 + } 6955 + } 6876 6956 return nil 6877 6957 } 6878 6958 ··· 6901 6981 6902 6982 n := extra 6903 6983 6904 - nameBuf := make([]byte, 9) 6984 + nameBuf := make([]byte, 10) 6905 6985 for i := uint64(0); i < n; i++ { 6906 6986 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 6907 6987 if err != nil { ··· 6971 7051 6972 7052 t.Title = string(sval) 6973 7053 } 7054 + // t.Mentions ([]string) (slice) 7055 + case "mentions": 7056 + 7057 + maj, extra, err = cr.ReadHeader() 7058 + if err != nil { 7059 + return err 7060 + } 7061 + 7062 + if extra > 8192 { 7063 + return fmt.Errorf("t.Mentions: array too large (%d)", extra) 7064 + } 7065 + 7066 + if maj != cbg.MajArray { 7067 + return fmt.Errorf("expected cbor array") 7068 + } 7069 + 7070 + if extra > 0 { 7071 + t.Mentions = make([]string, extra) 7072 + } 7073 + 7074 + for i := 0; i < int(extra); i++ { 7075 + { 7076 + var maj byte 7077 + var extra uint64 7078 + var err error 7079 + _ = maj 7080 + _ = extra 7081 + _ = err 7082 + 7083 + { 7084 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 7085 + if err != nil { 7086 + return err 7087 + } 7088 + 7089 + t.Mentions[i] = string(sval) 7090 + } 7091 + 7092 + } 7093 + } 6974 7094 // t.CreatedAt (string) (string) 6975 7095 case "createdAt": 6976 7096 ··· 6982 7102 6983 7103 t.CreatedAt = string(sval) 6984 7104 } 7105 + // t.References ([]string) (slice) 7106 + case "references": 7107 + 7108 + maj, extra, err = cr.ReadHeader() 7109 + if err != nil { 7110 + return err 7111 + } 7112 + 7113 + if extra > 8192 { 7114 + return fmt.Errorf("t.References: array too large (%d)", extra) 7115 + } 7116 + 7117 + if maj != cbg.MajArray { 7118 + return fmt.Errorf("expected cbor array") 7119 + } 7120 + 7121 + if extra > 0 { 7122 + t.References = make([]string, extra) 7123 + } 7124 + 7125 + for i := 0; i < int(extra); i++ { 7126 + { 7127 + var maj byte 7128 + var extra uint64 7129 + var err error 7130 + _ = maj 7131 + _ = extra 7132 + _ = err 7133 + 7134 + { 7135 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 7136 + if err != nil { 7137 + return err 7138 + } 7139 + 7140 + t.References[i] = string(sval) 7141 + } 7142 + 7143 + } 7144 + } 6985 7145 6986 7146 default: 6987 7147 // Field doesn't exist on this type, so ignore it ··· 7000 7160 } 7001 7161 7002 7162 cw := cbg.NewCborWriter(w) 7003 - fieldCount := 5 7163 + fieldCount := 7 7164 + 7165 + if t.Mentions == nil { 7166 + fieldCount-- 7167 + } 7168 + 7169 + if t.References == nil { 7170 + fieldCount-- 7171 + } 7004 7172 7005 7173 if t.ReplyTo == nil { 7006 7174 fieldCount-- ··· 7107 7275 } 7108 7276 } 7109 7277 7278 + // t.Mentions ([]string) (slice) 7279 + if t.Mentions != nil { 7280 + 7281 + if len("mentions") > 1000000 { 7282 + return xerrors.Errorf("Value in field \"mentions\" was too long") 7283 + } 7284 + 7285 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { 7286 + return err 7287 + } 7288 + if _, err := cw.WriteString(string("mentions")); err != nil { 7289 + return err 7290 + } 7291 + 7292 + if len(t.Mentions) > 8192 { 7293 + return xerrors.Errorf("Slice value in field t.Mentions was too long") 7294 + } 7295 + 7296 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { 7297 + return err 7298 + } 7299 + for _, v := range t.Mentions { 7300 + if len(v) > 1000000 { 7301 + return xerrors.Errorf("Value in field v was too long") 7302 + } 7303 + 7304 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 7305 + return err 7306 + } 7307 + if _, err := cw.WriteString(string(v)); err != nil { 7308 + return err 7309 + } 7310 + 7311 + } 7312 + } 7313 + 7110 7314 // t.CreatedAt (string) (string) 7111 7315 if len("createdAt") > 1000000 { 7112 7316 return xerrors.Errorf("Value in field \"createdAt\" was too long") ··· 7129 7333 if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { 7130 7334 return err 7131 7335 } 7336 + 7337 + // t.References ([]string) (slice) 7338 + if t.References != nil { 7339 + 7340 + if len("references") > 1000000 { 7341 + return xerrors.Errorf("Value in field \"references\" was too long") 7342 + } 7343 + 7344 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { 7345 + return err 7346 + } 7347 + if _, err := cw.WriteString(string("references")); err != nil { 7348 + return err 7349 + } 7350 + 7351 + if len(t.References) > 8192 { 7352 + return xerrors.Errorf("Slice value in field t.References was too long") 7353 + } 7354 + 7355 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { 7356 + return err 7357 + } 7358 + for _, v := range t.References { 7359 + if len(v) > 1000000 { 7360 + return xerrors.Errorf("Value in field v was too long") 7361 + } 7362 + 7363 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 7364 + return err 7365 + } 7366 + if _, err := cw.WriteString(string(v)); err != nil { 7367 + return err 7368 + } 7369 + 7370 + } 7371 + } 7132 7372 return nil 7133 7373 } 7134 7374 ··· 7157 7397 7158 7398 n := extra 7159 7399 7160 - nameBuf := make([]byte, 9) 7400 + nameBuf := make([]byte, 10) 7161 7401 for i := uint64(0); i < n; i++ { 7162 7402 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 7163 7403 if err != nil { ··· 7227 7467 t.ReplyTo = (*string)(&sval) 7228 7468 } 7229 7469 } 7470 + // t.Mentions ([]string) (slice) 7471 + case "mentions": 7472 + 7473 + maj, extra, err = cr.ReadHeader() 7474 + if err != nil { 7475 + return err 7476 + } 7477 + 7478 + if extra > 8192 { 7479 + return fmt.Errorf("t.Mentions: array too large (%d)", extra) 7480 + } 7481 + 7482 + if maj != cbg.MajArray { 7483 + return fmt.Errorf("expected cbor array") 7484 + } 7485 + 7486 + if extra > 0 { 7487 + t.Mentions = make([]string, extra) 7488 + } 7489 + 7490 + for i := 0; i < int(extra); i++ { 7491 + { 7492 + var maj byte 7493 + var extra uint64 7494 + var err error 7495 + _ = maj 7496 + _ = extra 7497 + _ = err 7498 + 7499 + { 7500 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 7501 + if err != nil { 7502 + return err 7503 + } 7504 + 7505 + t.Mentions[i] = string(sval) 7506 + } 7507 + 7508 + } 7509 + } 7230 7510 // t.CreatedAt (string) (string) 7231 7511 case "createdAt": 7232 7512 ··· 7238 7518 7239 7519 t.CreatedAt = string(sval) 7240 7520 } 7521 + // t.References ([]string) (slice) 7522 + case "references": 7523 + 7524 + maj, extra, err = cr.ReadHeader() 7525 + if err != nil { 7526 + return err 7527 + } 7528 + 7529 + if extra > 8192 { 7530 + return fmt.Errorf("t.References: array too large (%d)", extra) 7531 + } 7532 + 7533 + if maj != cbg.MajArray { 7534 + return fmt.Errorf("expected cbor array") 7535 + } 7536 + 7537 + if extra > 0 { 7538 + t.References = make([]string, extra) 7539 + } 7540 + 7541 + for i := 0; i < int(extra); i++ { 7542 + { 7543 + var maj byte 7544 + var extra uint64 7545 + var err error 7546 + _ = maj 7547 + _ = extra 7548 + _ = err 7549 + 7550 + { 7551 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 7552 + if err != nil { 7553 + return err 7554 + } 7555 + 7556 + t.References[i] = string(sval) 7557 + } 7558 + 7559 + } 7560 + } 7241 7561 7242 7562 default: 7243 7563 // Field doesn't exist on this type, so ignore it ··· 7755 8075 } 7756 8076 7757 8077 cw := cbg.NewCborWriter(w) 8078 + fieldCount := 6 7758 8079 7759 - if _, err := cw.Write([]byte{164}); err != nil { 8080 + if t.Mentions == nil { 8081 + fieldCount-- 8082 + } 8083 + 8084 + if t.References == nil { 8085 + fieldCount-- 8086 + } 8087 + 8088 + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 7760 8089 return err 7761 8090 } 7762 8091 ··· 7825 8154 return err 7826 8155 } 7827 8156 8157 + // t.Mentions ([]string) (slice) 8158 + if t.Mentions != nil { 8159 + 8160 + if len("mentions") > 1000000 { 8161 + return xerrors.Errorf("Value in field \"mentions\" was too long") 8162 + } 8163 + 8164 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { 8165 + return err 8166 + } 8167 + if _, err := cw.WriteString(string("mentions")); err != nil { 8168 + return err 8169 + } 8170 + 8171 + if len(t.Mentions) > 8192 { 8172 + return xerrors.Errorf("Slice value in field t.Mentions was too long") 8173 + } 8174 + 8175 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { 8176 + return err 8177 + } 8178 + for _, v := range t.Mentions { 8179 + if len(v) > 1000000 { 8180 + return xerrors.Errorf("Value in field v was too long") 8181 + } 8182 + 8183 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 8184 + return err 8185 + } 8186 + if _, err := cw.WriteString(string(v)); err != nil { 8187 + return err 8188 + } 8189 + 8190 + } 8191 + } 8192 + 7828 8193 // t.CreatedAt (string) (string) 7829 8194 if len("createdAt") > 1000000 { 7830 8195 return xerrors.Errorf("Value in field \"createdAt\" was too long") ··· 7847 8212 if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { 7848 8213 return err 7849 8214 } 8215 + 8216 + // t.References ([]string) (slice) 8217 + if t.References != nil { 8218 + 8219 + if len("references") > 1000000 { 8220 + return xerrors.Errorf("Value in field \"references\" was too long") 8221 + } 8222 + 8223 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { 8224 + return err 8225 + } 8226 + if _, err := cw.WriteString(string("references")); err != nil { 8227 + return err 8228 + } 8229 + 8230 + if len(t.References) > 8192 { 8231 + return xerrors.Errorf("Slice value in field t.References was too long") 8232 + } 8233 + 8234 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { 8235 + return err 8236 + } 8237 + for _, v := range t.References { 8238 + if len(v) > 1000000 { 8239 + return xerrors.Errorf("Value in field v was too long") 8240 + } 8241 + 8242 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 8243 + return err 8244 + } 8245 + if _, err := cw.WriteString(string(v)); err != nil { 8246 + return err 8247 + } 8248 + 8249 + } 8250 + } 7850 8251 return nil 7851 8252 } 7852 8253 ··· 7875 8276 7876 8277 n := extra 7877 8278 7878 - nameBuf := make([]byte, 9) 8279 + nameBuf := make([]byte, 10) 7879 8280 for i := uint64(0); i < n; i++ { 7880 8281 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 7881 8282 if err != nil { ··· 7924 8325 7925 8326 t.LexiconTypeID = string(sval) 7926 8327 } 8328 + // t.Mentions ([]string) (slice) 8329 + case "mentions": 8330 + 8331 + maj, extra, err = cr.ReadHeader() 8332 + if err != nil { 8333 + return err 8334 + } 8335 + 8336 + if extra > 8192 { 8337 + return fmt.Errorf("t.Mentions: array too large (%d)", extra) 8338 + } 8339 + 8340 + if maj != cbg.MajArray { 8341 + return fmt.Errorf("expected cbor array") 8342 + } 8343 + 8344 + if extra > 0 { 8345 + t.Mentions = make([]string, extra) 8346 + } 8347 + 8348 + for i := 0; i < int(extra); i++ { 8349 + { 8350 + var maj byte 8351 + var extra uint64 8352 + var err error 8353 + _ = maj 8354 + _ = extra 8355 + _ = err 8356 + 8357 + { 8358 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 8359 + if err != nil { 8360 + return err 8361 + } 8362 + 8363 + t.Mentions[i] = string(sval) 8364 + } 8365 + 8366 + } 8367 + } 7927 8368 // t.CreatedAt (string) (string) 7928 8369 case "createdAt": 7929 8370 ··· 7935 8376 7936 8377 t.CreatedAt = string(sval) 7937 8378 } 8379 + // t.References ([]string) (slice) 8380 + case "references": 8381 + 8382 + maj, extra, err = cr.ReadHeader() 8383 + if err != nil { 8384 + return err 8385 + } 8386 + 8387 + if extra > 8192 { 8388 + return fmt.Errorf("t.References: array too large (%d)", extra) 8389 + } 8390 + 8391 + if maj != cbg.MajArray { 8392 + return fmt.Errorf("expected cbor array") 8393 + } 8394 + 8395 + if extra > 0 { 8396 + t.References = make([]string, extra) 8397 + } 8398 + 8399 + for i := 0; i < int(extra); i++ { 8400 + { 8401 + var maj byte 8402 + var extra uint64 8403 + var err error 8404 + _ = maj 8405 + _ = extra 8406 + _ = err 8407 + 8408 + { 8409 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 8410 + if err != nil { 8411 + return err 8412 + } 8413 + 8414 + t.References[i] = string(sval) 8415 + } 8416 + 8417 + } 8418 + } 7938 8419 7939 8420 default: 7940 8421 // Field doesn't exist on this type, so ignore it
+7 -5
api/tangled/issuecomment.go
··· 17 17 } // 18 18 // RECORDTYPE: RepoIssueComment 19 19 type RepoIssueComment struct { 20 - LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue.comment" cborgen:"$type,const=sh.tangled.repo.issue.comment"` 21 - Body string `json:"body" cborgen:"body"` 22 - CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 - Issue string `json:"issue" cborgen:"issue"` 24 - ReplyTo *string `json:"replyTo,omitempty" cborgen:"replyTo,omitempty"` 20 + LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue.comment" cborgen:"$type,const=sh.tangled.repo.issue.comment"` 21 + Body string `json:"body" cborgen:"body"` 22 + CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 + Issue string `json:"issue" cborgen:"issue"` 24 + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` 25 + References []string `json:"references,omitempty" cborgen:"references,omitempty"` 26 + ReplyTo *string `json:"replyTo,omitempty" cborgen:"replyTo,omitempty"` 25 27 }
+6 -4
api/tangled/pullcomment.go
··· 17 17 } // 18 18 // RECORDTYPE: RepoPullComment 19 19 type RepoPullComment struct { 20 - LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull.comment" cborgen:"$type,const=sh.tangled.repo.pull.comment"` 21 - Body string `json:"body" cborgen:"body"` 22 - CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 - Pull string `json:"pull" cborgen:"pull"` 20 + LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull.comment" cborgen:"$type,const=sh.tangled.repo.pull.comment"` 21 + Body string `json:"body" cborgen:"body"` 22 + CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` 24 + Pull string `json:"pull" cborgen:"pull"` 25 + References []string `json:"references,omitempty" cborgen:"references,omitempty"` 24 26 }
+7 -5
api/tangled/repoissue.go
··· 17 17 } // 18 18 // RECORDTYPE: RepoIssue 19 19 type RepoIssue struct { 20 - LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue" cborgen:"$type,const=sh.tangled.repo.issue"` 21 - Body *string `json:"body,omitempty" cborgen:"body,omitempty"` 22 - CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 - Repo string `json:"repo" cborgen:"repo"` 24 - Title string `json:"title" cborgen:"title"` 20 + LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue" cborgen:"$type,const=sh.tangled.repo.issue"` 21 + Body *string `json:"body,omitempty" cborgen:"body,omitempty"` 22 + CreatedAt string `json:"createdAt" cborgen:"createdAt"` 23 + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` 24 + References []string `json:"references,omitempty" cborgen:"references,omitempty"` 25 + Repo string `json:"repo" cborgen:"repo"` 26 + Title string `json:"title" cborgen:"title"` 25 27 }
+9
appview/db/db.go
··· 561 561 email_notifications integer not null default 0 562 562 ); 563 563 564 + create table if not exists references ( 565 + id integer primary key autoincrement, 566 + from_at text not null, 567 + to_at text not null, 568 + unique (from, to) 569 + ); 570 + 564 571 create table if not exists migrations ( 565 572 id integer primary key autoincrement, 566 573 name text unique ··· 571 578 create index if not exists idx_notifications_recipient_read on notifications(recipient_did, read); 572 579 create index if not exists idx_stars_created on stars(created); 573 580 create index if not exists idx_stars_repo_at_created on stars(repo_at, created); 581 + create index if not exists idx_references_from_at on references(from_at); 582 + create index if not exists idx_references_to_at on references(to_at); 574 583 `) 575 584 if err != nil { 576 585 return nil, err
+73 -18
appview/db/issues.go
··· 10 10 "time" 11 11 12 12 "github.com/bluesky-social/indigo/atproto/syntax" 13 + "tangled.org/core/api/tangled" 13 14 "tangled.org/core/appview/models" 14 15 "tangled.org/core/appview/pagination" 15 16 ) ··· 69 70 returning rowid, issue_id 70 71 `, issue.RepoAt, issue.Did, issue.Rkey, newIssueId, issue.Title, issue.Body) 71 72 72 - return row.Scan(&issue.Id, &issue.IssueId) 73 + err = row.Scan(&issue.Id, &issue.IssueId) 74 + if err != nil { 75 + return fmt.Errorf("scan row: %w", err) 76 + } 77 + 78 + if err := putReferences(tx, issue.AtUri(), issue.References); err != nil { 79 + return fmt.Errorf("put references: %w", err) 80 + } 81 + return nil 73 82 } 74 83 75 84 func updateIssue(tx *sql.Tx, issue *models.Issue) error { ··· 79 88 set title = ?, body = ?, edited = ? 80 89 where did = ? and rkey = ? 81 90 `, issue.Title, issue.Body, time.Now().Format(time.RFC3339), issue.Did, issue.Rkey) 82 - return err 91 + if err != nil { 92 + return err 93 + } 94 + 95 + if err := putReferences(tx, issue.AtUri(), issue.References); err != nil { 96 + return fmt.Errorf("put references: %w", err) 97 + } 98 + return nil 83 99 } 84 100 85 101 func GetIssuesPaginated(e Execer, page pagination.Page, filters ...filter) ([]models.Issue, error) { ··· 234 250 } 235 251 } 236 252 253 + // collect references for each issue 254 + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", issueAts)) 255 + if err != nil { 256 + return nil, fmt.Errorf("failed to query references: %w", err) 257 + } 258 + for issueAt, references := range allReferencs { 259 + if issue, ok := issueMap[issueAt.String()]; ok { 260 + issue.References = references 261 + } 262 + } 263 + 237 264 var issues []models.Issue 238 265 for _, i := range issueMap { 239 266 issues = append(issues, *i) ··· 323 350 return ids, nil 324 351 } 325 352 326 - func AddIssueComment(e Execer, c models.IssueComment) (int64, error) { 327 - result, err := e.Exec( 353 + func AddIssueComment(tx *sql.Tx, c models.IssueComment) (int64, error) { 354 + result, err := tx.Exec( 328 355 `insert into issue_comments ( 329 356 did, 330 357 rkey, ··· 358 385 return 0, err 359 386 } 360 387 388 + if err := putReferences(tx, c.AtUri(), c.References); err != nil { 389 + return 0, fmt.Errorf("put references: %w", err) 390 + } 391 + 361 392 id, err := result.LastInsertId() 362 393 if err != nil { 363 394 return 0, err ··· 386 417 } 387 418 388 419 func GetIssueComments(e Execer, filters ...filter) ([]models.IssueComment, error) { 389 - var comments []models.IssueComment 420 + commentMap := make(map[string]*models.IssueComment) 390 421 391 422 var conditions []string 392 423 var args []any ··· 465 496 comment.ReplyTo = &replyTo.V 466 497 } 467 498 468 - comments = append(comments, comment) 499 + atUri := comment.AtUri().String() 500 + commentMap[atUri] = &comment 469 501 } 470 502 471 503 if err = rows.Err(); err != nil { 472 504 return nil, err 473 505 } 474 506 507 + // collect references for each comments 508 + commentAts := slices.Collect(maps.Keys(commentMap)) 509 + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", commentAts)) 510 + if err != nil { 511 + return nil, fmt.Errorf("failed to query references: %w", err) 512 + } 513 + for commentAt, references := range allReferencs { 514 + if comment, ok := commentMap[commentAt.String()]; ok { 515 + comment.References = references 516 + } 517 + } 518 + 519 + var comments []models.IssueComment 520 + for _, c := range commentMap { 521 + comments = append(comments, *c) 522 + } 523 + 524 + sort.Slice(comments, func(i, j int) bool { 525 + return comments[i].Created.After(comments[j].Created) 526 + }) 527 + 475 528 return comments, nil 476 529 } 477 530 478 - func DeleteIssues(e Execer, filters ...filter) error { 479 - var conditions []string 480 - var args []any 481 - for _, filter := range filters { 482 - conditions = append(conditions, filter.Condition()) 483 - args = append(args, filter.Arg()...) 531 + func DeleteIssues(tx *sql.Tx, did, rkey string) error { 532 + _, err := tx.Exec( 533 + `delete from issues 534 + where did = ? and rkey = ?`, 535 + did, 536 + rkey, 537 + ) 538 + if err != nil { 539 + return fmt.Errorf("delete issue: %w", err) 484 540 } 485 541 486 - whereClause := "" 487 - if conditions != nil { 488 - whereClause = " where " + strings.Join(conditions, " and ") 542 + uri := syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", did, tangled.RepoIssueNSID, rkey)) 543 + err = deleteReferences(tx, uri) 544 + if err != nil { 545 + return fmt.Errorf("delete references: %w", err) 489 546 } 490 547 491 - query := fmt.Sprintf(`delete from issues %s`, whereClause) 492 - _, err := e.Exec(query, args...) 493 - return err 548 + return nil 494 549 } 495 550 496 551 func CloseIssues(e Execer, filters ...filter) error {
+30 -4
appview/db/pulls.go
··· 492 492 } 493 493 defer rows.Close() 494 494 495 - var comments []models.PullComment 495 + commentMap := make(map[string]*models.PullComment) 496 496 for rows.Next() { 497 497 var comment models.PullComment 498 498 var createdAt string ··· 514 514 comment.Created = t 515 515 } 516 516 517 - comments = append(comments, comment) 517 + atUri := comment.AtUri().String() 518 + commentMap[atUri] = &comment 518 519 } 519 520 520 521 if err := rows.Err(); err != nil { 521 522 return nil, err 522 523 } 523 524 525 + // collect references for each comments 526 + commentAts := slices.Collect(maps.Keys(commentMap)) 527 + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", commentAts)) 528 + if err != nil { 529 + return nil, fmt.Errorf("failed to query references: %w", err) 530 + } 531 + for commentAt, references := range allReferencs { 532 + if comment, ok := commentMap[commentAt.String()]; ok { 533 + comment.References = references 534 + } 535 + } 536 + 537 + var comments []models.PullComment 538 + for _, c := range commentMap { 539 + comments = append(comments, *c) 540 + } 541 + 542 + sort.Slice(comments, func(i, j int) bool { 543 + return comments[i].Created.After(comments[j].Created) 544 + }) 545 + 524 546 return comments, nil 525 547 } 526 548 ··· 600 622 return pulls, nil 601 623 } 602 624 603 - func NewPullComment(e Execer, comment *models.PullComment) (int64, error) { 625 + func NewPullComment(tx *sql.Tx, comment *models.PullComment) (int64, error) { 604 626 query := `insert into pull_comments (owner_did, repo_at, submission_id, comment_at, pull_id, body) values (?, ?, ?, ?, ?, ?)` 605 - res, err := e.Exec( 627 + res, err := tx.Exec( 606 628 query, 607 629 comment.OwnerDid, 608 630 comment.RepoAt, ··· 620 642 return 0, err 621 643 } 622 644 645 + if err := putReferences(tx, comment.AtUri(), comment.References); err != nil { 646 + return 0, fmt.Errorf("put references: %w", err) 647 + } 648 + 623 649 return i, nil 624 650 } 625 651
+78
appview/db/reference.go
··· 101 101 } 102 102 uris = append(uris, uri) 103 103 } 104 + if err := rows.Err(); err != nil { 105 + return nil, fmt.Errorf("iterate rows: %w", err) 106 + } 107 + 104 108 return uris, nil 105 109 } 106 110 ··· 170 174 } 171 175 return uris, nil 172 176 } 177 + 178 + func putReferences(tx *sql.Tx, fromAt syntax.ATURI, references []syntax.ATURI) error { 179 + err := deleteReferences(tx, fromAt) 180 + if err != nil { 181 + return fmt.Errorf("delete old references: %w", err) 182 + } 183 + 184 + values := make([]string, 0, len(references)) 185 + args := make([]any, 0, len(references)*2) 186 + for _, ref := range references { 187 + values = append(values, "(?, ?)") 188 + args = append(args, fromAt, ref) 189 + } 190 + _, err = tx.Exec( 191 + fmt.Sprintf( 192 + `insert into references (from, at) 193 + values %s`, 194 + strings.Join(values, ","), 195 + ), 196 + args..., 197 + ) 198 + if err != nil { 199 + return fmt.Errorf("insert new references: %w", err) 200 + } 201 + return nil 202 + } 203 + 204 + func deleteReferences(tx *sql.Tx, fromAt syntax.ATURI) error { 205 + _, err := tx.Exec(`delete from references where from_at = ?`, fromAt) 206 + return err 207 + } 208 + 209 + func GetReferencesAll(e Execer, filters ...filter) (map[syntax.ATURI][]syntax.ATURI, error) { 210 + var ( 211 + conditions []string 212 + args []any 213 + ) 214 + for _, filter := range filters { 215 + conditions = append(conditions, filter.Condition()) 216 + args = append(args, filter.Arg()...) 217 + } 218 + 219 + whereClause := "" 220 + if conditions != nil { 221 + whereClause = " where " + strings.Join(conditions, " and ") 222 + } 223 + 224 + rows, err := e.Query( 225 + fmt.Sprintf( 226 + `select from_at, to_at from references %s`, 227 + whereClause, 228 + ), 229 + ) 230 + if err != nil { 231 + return nil, fmt.Errorf("query references: %w", err) 232 + } 233 + defer rows.Close() 234 + 235 + result := make(map[syntax.ATURI][]syntax.ATURI) 236 + 237 + for rows.Next() { 238 + var from, to syntax.ATURI 239 + if err := rows.Scan(&from, &to); err != nil { 240 + return nil, fmt.Errorf("scan row: %w", err) 241 + } 242 + 243 + result[from] = append(result[from], to) 244 + } 245 + if err := rows.Err(); err != nil { 246 + return nil, fmt.Errorf("iterate rows: %w", err) 247 + } 248 + 249 + return result, nil 250 + }
+22 -5
appview/ingester.go
··· 841 841 return nil 842 842 843 843 case jmodels.CommitOperationDelete: 844 + tx, err := ddb.BeginTx(ctx, nil) 845 + if err != nil { 846 + l.Error("failed to begin transaction", "err", err) 847 + return err 848 + } 849 + defer tx.Rollback() 850 + 844 851 if err := db.DeleteIssues( 845 - ddb, 846 - db.FilterEq("did", did), 847 - db.FilterEq("rkey", rkey), 852 + tx, 853 + did, 854 + rkey, 848 855 ); err != nil { 849 856 l.Error("failed to delete", "err", err) 850 857 return fmt.Errorf("failed to delete issue record: %w", err) 851 858 } 859 + if err := tx.Commit(); err != nil { 860 + l.Error("failed to commit txn", "err", err) 861 + return err 862 + } 852 863 853 864 return nil 854 865 } ··· 888 899 return fmt.Errorf("failed to validate comment: %w", err) 889 900 } 890 901 891 - _, err = db.AddIssueComment(ddb, *comment) 902 + tx, err := ddb.Begin() 903 + if err != nil { 904 + return fmt.Errorf("failed to start transaction: %w", err) 905 + } 906 + defer tx.Rollback() 907 + 908 + _, err = db.AddIssueComment(tx, *comment) 892 909 if err != nil { 893 910 return fmt.Errorf("failed to create issue comment: %w", err) 894 911 } 895 912 896 - return nil 913 + return tx.Commit() 897 914 898 915 case jmodels.CommitOperationDelete: 899 916 if err := db.DeleteIssueComments(
+49 -19
appview/issues/issues.go
··· 241 241 } 242 242 l = l.With("did", issue.Did, "rkey", issue.Rkey) 243 243 244 + tx, err := rp.db.Begin() 245 + if err != nil { 246 + l.Error("failed to start transaction", "err", err) 247 + rp.pages.Notice(w, "issue-comment", "Failed to create comment, try again later.") 248 + return 249 + } 250 + defer tx.Rollback() 251 + 244 252 // delete from PDS 245 253 client, err := rp.oauth.AuthorizedClient(r) 246 254 if err != nil { ··· 261 269 } 262 270 263 271 // delete from db 264 - if err := db.DeleteIssues(rp.db, db.FilterEq("id", issue.Id)); err != nil { 272 + if err := db.DeleteIssues(tx, issue.Did, issue.Rkey); err != nil { 265 273 l.Error("failed to delete issue", "err", err) 266 274 rp.pages.Notice(w, noticeId, "Failed to delete issue.") 267 275 return 268 276 } 277 + tx.Commit() 269 278 270 279 rp.notifier.DeleteIssue(r.Context(), issue) 271 280 ··· 402 411 replyTo = &replyToUri 403 412 } 404 413 405 - mentions, _ := rp.refResolver.Resolve(r.Context(), body) 414 + mentions, references := rp.refResolver.Resolve(r.Context(), body) 406 415 407 416 comment := models.IssueComment{ 408 - Did: user.Did, 409 - Rkey: tid.TID(), 410 - IssueAt: issue.AtUri().String(), 411 - ReplyTo: replyTo, 412 - Body: body, 413 - Created: time.Now(), 417 + Did: user.Did, 418 + Rkey: tid.TID(), 419 + IssueAt: issue.AtUri().String(), 420 + ReplyTo: replyTo, 421 + Body: body, 422 + Created: time.Now(), 423 + Mentions: mentions, 424 + References: references, 414 425 } 415 426 if err = rp.validator.ValidateIssueComment(&comment); err != nil { 416 427 l.Error("failed to validate comment", "err", err) ··· 447 458 } 448 459 }() 449 460 450 - commentId, err := db.AddIssueComment(rp.db, comment) 461 + tx, err := rp.db.Begin() 462 + if err != nil { 463 + l.Error("failed to start transaction", "err", err) 464 + rp.pages.Notice(w, "issue-comment", "Failed to create comment, try again later.") 465 + return 466 + } 467 + defer tx.Rollback() 468 + 469 + commentId, err := db.AddIssueComment(tx, comment) 451 470 if err != nil { 452 471 l.Error("failed to create comment", "err", err) 453 472 rp.pages.Notice(w, "issue-comment", "Failed to create comment.") ··· 569 588 newComment.Edited = &now 570 589 record := newComment.AsRecord() 571 590 572 - _, err = db.AddIssueComment(rp.db, newComment) 591 + tx, err := rp.db.Begin() 592 + if err != nil { 593 + l.Error("failed to start transaction", "err", err) 594 + rp.pages.Notice(w, "repo-notice", "Failed to update description, try again later.") 595 + return 596 + } 597 + defer tx.Rollback() 598 + 599 + _, err = db.AddIssueComment(tx, newComment) 573 600 if err != nil { 574 601 l.Error("failed to perferom update-description query", "err", err) 575 602 rp.pages.Notice(w, "repo-notice", "Failed to update description, try again later.") 576 603 return 577 604 } 605 + tx.Commit() 578 606 579 607 // rkey is optional, it was introduced later 580 608 if newComment.Rkey != "" { ··· 881 909 }) 882 910 case http.MethodPost: 883 911 body := r.FormValue("body") 884 - mentions, _ := rp.refResolver.Resolve(r.Context(), body) 912 + mentions, references := rp.refResolver.Resolve(r.Context(), body) 885 913 886 914 issue := &models.Issue{ 887 - RepoAt: f.RepoAt(), 888 - Rkey: tid.TID(), 889 - Title: r.FormValue("title"), 890 - Body: body, 891 - Open: true, 892 - Did: user.Did, 893 - Created: time.Now(), 894 - Repo: &f.Repo, 915 + RepoAt: f.RepoAt(), 916 + Rkey: tid.TID(), 917 + Title: r.FormValue("title"), 918 + Body: body, 919 + Open: true, 920 + Did: user.Did, 921 + Created: time.Now(), 922 + Mentions: mentions, 923 + References: references, 924 + Repo: &f.Repo, 895 925 } 896 926 897 927 if err := rp.validator.ValidateIssue(issue); err != nil {
+52 -28
appview/models/issue.go
··· 10 10 ) 11 11 12 12 type Issue struct { 13 - Id int64 14 - Did string 15 - Rkey string 16 - RepoAt syntax.ATURI 17 - IssueId int 18 - Created time.Time 19 - Edited *time.Time 20 - Deleted *time.Time 21 - Title string 22 - Body string 23 - Open bool 13 + Id int64 14 + Did string 15 + Rkey string 16 + RepoAt syntax.ATURI 17 + IssueId int 18 + Created time.Time 19 + Edited *time.Time 20 + Deleted *time.Time 21 + Title string 22 + Body string 23 + Open bool 24 + Mentions []syntax.DID 25 + References []syntax.ATURI 24 26 25 27 // optionally, populate this when querying for reverse mappings 26 28 // like comment counts, parent repo etc. ··· 34 36 } 35 37 36 38 func (i *Issue) AsRecord() tangled.RepoIssue { 39 + mentions := make([]string, len(i.Mentions)) 40 + for i, did := range i.Mentions { 41 + mentions[i] = string(did) 42 + } 43 + references := make([]string, len(i.References)) 44 + for i, uri := range i.References { 45 + references[i] = string(uri) 46 + } 37 47 return tangled.RepoIssue{ 38 - Repo: i.RepoAt.String(), 39 - Title: i.Title, 40 - Body: &i.Body, 41 - CreatedAt: i.Created.Format(time.RFC3339), 48 + Repo: i.RepoAt.String(), 49 + Title: i.Title, 50 + Body: &i.Body, 51 + Mentions: mentions, 52 + References: references, 53 + CreatedAt: i.Created.Format(time.RFC3339), 42 54 } 43 55 } 44 56 ··· 161 173 } 162 174 163 175 type IssueComment struct { 164 - Id int64 165 - Did string 166 - Rkey string 167 - IssueAt string 168 - ReplyTo *string 169 - Body string 170 - Created time.Time 171 - Edited *time.Time 172 - Deleted *time.Time 176 + Id int64 177 + Did string 178 + Rkey string 179 + IssueAt string 180 + ReplyTo *string 181 + Body string 182 + Created time.Time 183 + Edited *time.Time 184 + Deleted *time.Time 185 + Mentions []syntax.DID 186 + References []syntax.ATURI 173 187 } 174 188 175 189 func (i *IssueComment) AtUri() syntax.ATURI { ··· 177 191 } 178 192 179 193 func (i *IssueComment) AsRecord() tangled.RepoIssueComment { 194 + mentions := make([]string, len(i.Mentions)) 195 + for i, did := range i.Mentions { 196 + mentions[i] = string(did) 197 + } 198 + references := make([]string, len(i.References)) 199 + for i, uri := range i.References { 200 + references[i] = string(uri) 201 + } 180 202 return tangled.RepoIssueComment{ 181 - Body: i.Body, 182 - Issue: i.IssueAt, 183 - CreatedAt: i.Created.Format(time.RFC3339), 184 - ReplyTo: i.ReplyTo, 203 + Body: i.Body, 204 + Issue: i.IssueAt, 205 + CreatedAt: i.Created.Format(time.RFC3339), 206 + ReplyTo: i.ReplyTo, 207 + Mentions: mentions, 208 + References: references, 185 209 } 186 210 } 187 211
+26
appview/models/pull.go
··· 147 147 // content 148 148 Body string 149 149 150 + // meta 151 + Mentions []syntax.DID 152 + References []syntax.ATURI 153 + 150 154 // meta 151 155 Created time.Time 152 156 } 153 157 158 + func (p *PullComment) AtUri() syntax.ATURI { 159 + return syntax.ATURI(p.CommentAt) 160 + } 161 + 162 + // func (p *PullComment) AsRecord() tangled.RepoPullComment { 163 + // mentions := make([]string, len(p.Mentions)) 164 + // for i, did := range p.Mentions { 165 + // mentions[i] = string(did) 166 + // } 167 + // references := make([]string, len(p.References)) 168 + // for i, uri := range p.References { 169 + // references[i] = string(uri) 170 + // } 171 + // return tangled.RepoPullComment{ 172 + // Pull: p.PullAt, 173 + // Body: p.Body, 174 + // Mentions: mentions, 175 + // References: references, 176 + // CreatedAt: p.Created.Format(time.RFC3339), 177 + // } 178 + // } 179 + 154 180 func (p *Pull) LastRoundNumber() int { 155 181 return len(p.Submissions) - 1 156 182 }
+3 -1
appview/pulls/pulls.go
··· 733 733 return 734 734 } 735 735 736 - mentions, _ := s.refResolver.Resolve(r.Context(), body) 736 + mentions, references := s.refResolver.Resolve(r.Context(), body) 737 737 738 738 // Start a transaction 739 739 tx, err := s.db.BeginTx(r.Context(), nil) ··· 777 777 Body: body, 778 778 CommentAt: atResp.Uri, 779 779 SubmissionId: pull.Submissions[roundNumber].ID, 780 + Mentions: mentions, 781 + References: references, 780 782 } 781 783 782 784 // Create the pull comment in the database with the commentAt field
+14
lexicons/issue/comment.json
··· 29 29 "replyTo": { 30 30 "type": "string", 31 31 "format": "at-uri" 32 + }, 33 + "mentions": { 34 + "type": "array", 35 + "items": { 36 + "type": "string", 37 + "format": "did" 38 + } 39 + }, 40 + "references": { 41 + "type": "array", 42 + "items": { 43 + "type": "string", 44 + "format": "at-uri" 45 + } 32 46 } 33 47 } 34 48 }
+14
lexicons/issue/issue.json
··· 24 24 "createdAt": { 25 25 "type": "string", 26 26 "format": "datetime" 27 + }, 28 + "mentions": { 29 + "type": "array", 30 + "items": { 31 + "type": "string", 32 + "format": "did" 33 + } 34 + }, 35 + "references": { 36 + "type": "array", 37 + "items": { 38 + "type": "string", 39 + "format": "at-uri" 40 + } 27 41 } 28 42 } 29 43 }
+14
lexicons/pulls/comment.json
··· 25 25 "createdAt": { 26 26 "type": "string", 27 27 "format": "datetime" 28 + }, 29 + "mentions": { 30 + "type": "array", 31 + "items": { 32 + "type": "string", 33 + "format": "did" 34 + } 35 + }, 36 + "references": { 37 + "type": "array", 38 + "items": { 39 + "type": "string", 40 + "format": "at-uri" 41 + } 28 42 } 29 43 } 30 44 }