appview: backlinks

currently all backlinks are parsed with markdown parser.
So only explict urls like below are supported:

- <https://tangled.org/owner.com/repo-name/issues/123>
- [full url](https://tangled.org/owner.com/repo-name/issues/123)
- [absolute path](/owner.com/repo-name/issues/123)

Also `#comment-123` fragment is supported too.

All references in issue/pull/comment records are stored in at-uri
format. we do have a `mentions` field, but it isn't used yet. mentions
are parsed on rendering and aren't stored anywhere for now.

Signed-off-by: Seongmin Lee <[e-mail]>

appview/db/pulls.go

···
 		insert into pull_submissions (pull_at, round_number, patch, combined, source_rev)
 		values (?, ?, ?, ?, ?)
 	`, pull.AtUri(), 0, pull.Submissions[0].Patch, pull.Submissions[0].Combined, pull.Submissions[0].SourceRev)
-	return err
+	if err != nil {
+		return err
+	}
+
+	if err := putReferences(tx, pull.AtUri(), pull.References); err != nil {
+		return fmt.Errorf("put reference_links: %w", err)
+	}
+
+	return nil
 }
 
 func GetPullAt(e Execer, repoAt syntax.ATURI, pullId int) (syntax.ATURI, error) {
···
 			if sourceRepo, ok := sourceRepoMap[*p.PullSource.RepoAt]; ok {
 				p.PullSource.Repo = sourceRepo
 			}
+		}
+	}
+
+	allReferences, err := GetReferencesAll(e, FilterIn("from_at", pullAts))
+	if err != nil {
+		return nil, fmt.Errorf("failed to query reference_links: %w", err)
+	}
+	for pullAt, references := range allReferences {
+		if pull, ok := pulls[pullAt]; ok {
+			pull.References = references
 		}
 	}
 

appview/db/reference.go

···
 
 	return result, nil
 }
+
+func GetBacklinks(e Execer, target syntax.ATURI) ([]models.RichReferenceLink, error) {
+	rows, err := e.Query(
+		`select from_at from reference_links
+		where to_at = ?`,
+		target,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("query backlinks: %w", err)
+	}
+	defer rows.Close()
+
+	var (
+		backlinks    []models.RichReferenceLink
+		backlinksMap = make(map[string][]syntax.ATURI)
+	)
+	for rows.Next() {
+		var from syntax.ATURI
+		if err := rows.Scan(&from); err != nil {
+			return nil, fmt.Errorf("scan row: %w", err)
+		}
+		nsid := from.Collection().String()
+		backlinksMap[nsid] = append(backlinksMap[nsid], from)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+
+	var ls []models.RichReferenceLink
+	ls, err = getIssueBacklinks(e, backlinksMap[tangled.RepoIssueNSID])
+	if err != nil {
+		return nil, fmt.Errorf("get issue backlinks: %w", err)
+	}
+	backlinks = append(backlinks, ls...)
+	ls, err = getIssueCommentBacklinks(e, backlinksMap[tangled.RepoIssueCommentNSID])
+	if err != nil {
+		return nil, fmt.Errorf("get issue_comment backlinks: %w", err)
+	}
+	backlinks = append(backlinks, ls...)
+	ls, err = getPullBacklinks(e, backlinksMap[tangled.RepoPullNSID])
+	if err != nil {
+		return nil, fmt.Errorf("get pull backlinks: %w", err)
+	}
+	backlinks = append(backlinks, ls...)
+	ls, err = getPullCommentBacklinks(e, backlinksMap[tangled.RepoPullCommentNSID])
+	if err != nil {
+		return nil, fmt.Errorf("get pull_comment backlinks: %w", err)
+	}
+	backlinks = append(backlinks, ls...)
+
+	return backlinks, nil
+}
+
+func getIssueBacklinks(e Execer, aturis []syntax.ATURI) ([]models.RichReferenceLink, error) {
+	if len(aturis) == 0 {
+		return nil, nil
+	}
+	vals := make([]string, len(aturis))
+	args := make([]any, 0, len(aturis)*2)
+	for i, aturi := range aturis {
+		vals[i] = "(?, ?)"
+		did := aturi.Authority().String()
+		rkey := aturi.RecordKey().String()
+		args = append(args, did, rkey)
+	}
+	rows, err := e.Query(
+		fmt.Sprintf(
+			`select r.did, r.name, i.issue_id, i.title, i.open
+			from issues i
+			join repos r
+				on r.at_uri = i.repo_at
+			where (i.did, i.rkey) in (%s)`,
+			strings.Join(vals, ","),
+		),
+		args...,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var refLinks []models.RichReferenceLink
+	for rows.Next() {
+		var l models.RichReferenceLink
+		l.Kind = models.RefKindIssue
+		if err := rows.Scan(&l.Handle, &l.Repo, &l.SubjectId, &l.Title, &l.State); err != nil {
+			return nil, err
+		}
+		refLinks = append(refLinks, l)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+	return refLinks, nil
+}
+
+func getIssueCommentBacklinks(e Execer, aturis []syntax.ATURI) ([]models.RichReferenceLink, error) {
+	if len(aturis) == 0 {
+		return nil, nil
+	}
+	filter := FilterIn("c.at_uri", aturis)
+	rows, err := e.Query(
+		fmt.Sprintf(
+			`select r.did, r.name, i.issue_id, c.id, i.title, i.open
+			from issue_comments c
+			join issues i
+				on i.at_uri = c.issue_at
+			join repos r
+				on r.at_uri = i.repo_at
+			where %s`,
+			filter.Condition(),
+		),
+		filter.Arg()...,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var refLinks []models.RichReferenceLink
+	for rows.Next() {
+		var l models.RichReferenceLink
+		l.Kind = models.RefKindIssue
+		l.CommentId = new(int)
+		if err := rows.Scan(&l.Handle, &l.Repo, &l.SubjectId, l.CommentId, &l.Title, &l.State); err != nil {
+			return nil, err
+		}
+		refLinks = append(refLinks, l)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+	return refLinks, nil
+}
+
+func getPullBacklinks(e Execer, aturis []syntax.ATURI) ([]models.RichReferenceLink, error) {
+	if len(aturis) == 0 {
+		return nil, nil
+	}
+	vals := make([]string, len(aturis))
+	args := make([]any, 0, len(aturis)*2)
+	for i, aturi := range aturis {
+		vals[i] = "(?, ?)"
+		did := aturi.Authority().String()
+		rkey := aturi.RecordKey().String()
+		args = append(args, did, rkey)
+	}
+	rows, err := e.Query(
+		fmt.Sprintf(
+			`select r.did, r.name, p.pull_id, p.title, p.state
+			from pulls p
+			join repos r
+				on r.at_uri = p.repo_at
+			where (p.owner_did, p.rkey) in (%s)`,
+			strings.Join(vals, ","),
+		),
+		args...,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var refLinks []models.RichReferenceLink
+	for rows.Next() {
+		var l models.RichReferenceLink
+		l.Kind = models.RefKindPull
+		if err := rows.Scan(&l.Handle, &l.Repo, &l.SubjectId, &l.Title, &l.State); err != nil {
+			return nil, err
+		}
+		refLinks = append(refLinks, l)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+	return refLinks, nil
+}
+
+func getPullCommentBacklinks(e Execer, aturis []syntax.ATURI) ([]models.RichReferenceLink, error) {
+	if len(aturis) == 0 {
+		return nil, nil
+	}
+	filter := FilterIn("c.comment_at", aturis)
+	rows, err := e.Query(
+		fmt.Sprintf(
+			`select r.did, r.name, p.pull_id, c.id, p.title, p.state
+			from repos r
+			join pulls p
+				on r.at_uri = p.repo_at
+			join pull_comments c
+				on r.at_uri = c.repo_at and p.pull_id = c.pull_id
+			where %s`,
+			filter.Condition(),
+		),
+		filter.Arg()...,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var refLinks []models.RichReferenceLink
+	for rows.Next() {
+		var l models.RichReferenceLink
+		l.Kind = models.RefKindPull
+		l.CommentId = new(int)
+		if err := rows.Scan(&l.Handle, &l.Repo, &l.SubjectId, l.CommentId, &l.Title, &l.State); err != nil {
+			return nil, err
+		}
+		refLinks = append(refLinks, l)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+	return refLinks, nil
+}

appview/issues/issues.go

···
 		userReactions = db.GetReactionStatusMap(rp.db, user.Did, issue.AtUri())
 	}
 
+	backlinks, err := db.GetBacklinks(rp.db, issue.AtUri())
+	if err != nil {
+		l.Error("failed to fetch backlinks", "err", err)
+		rp.pages.Error503(w)
+		return
+	}
+
 	labelDefs, err := db.GetLabelDefinitions(
 		rp.db,
 		db.FilterIn("at_uri", f.Labels),
···
 		RepoInfo:             rp.repoResolver.GetRepoInfo(r, user),
 		Issue:                issue,
 		CommentList:          issue.CommentList(),
+		Backlinks:            backlinks,
 		OrderedReactionKinds: models.OrderedReactionKinds,
 		Reactions:            reactionMap,
 		UserReacted:          userReactions,
···
 		newIssue := issue
 		newIssue.Title = r.FormValue("title")
 		newIssue.Body = r.FormValue("body")
+		newIssue.Mentions, newIssue.References = rp.refResolver.Resolve(r.Context(), newIssue.Body)
 
 		if err := rp.validator.ValidateIssue(newIssue); err != nil {
 			l.Error("validation error", "err", err)
···
 		newComment := comment
 		newComment.Body = newBody
 		newComment.Edited = &now
+		newComment.Mentions, newComment.References = rp.refResolver.Resolve(r.Context(), newBody)
+
 		record := newComment.AsRecord()
 
 		tx, err := rp.db.Begin()

appview/models/pull.go

···
 	TargetBranch string
 	State        PullState
 	Submissions  []*PullSubmission
+	Mentions     []syntax.DID
+	References   []syntax.ATURI
 
 	// stacking
 	StackId        string // nullable string
···
 			source.Repo = &s
 		}
 	}
+	mentions := make([]string, len(p.Mentions))
+	for i, did := range p.Mentions {
+		mentions[i] = string(did)
+	}
+	references := make([]string, len(p.References))
+	for i, uri := range p.References {
+		references[i] = string(uri)
+	}
 
 	record := tangled.RepoPull{
-		Title:     p.Title,
-		Body:      &p.Body,
-		CreatedAt: p.Created.Format(time.RFC3339),
+		Title:      p.Title,
+		Body:       &p.Body,
+		Mentions:   mentions,
+		References: references,
+		CreatedAt:  p.Created.Format(time.RFC3339),
 		Target: &tangled.RepoPull_Target{
 			Repo:   p.RepoAt.String(),
 			Branch: p.TargetBranch,

appview/models/reference.go

···
 package models
 
+import "fmt"
+
 type RefKind int
 
 const (
···
 	RefKindPull
 )
 
+func (k RefKind) String() string {
+	if k == RefKindIssue {
+		return "issues"
+	} else {
+		return "pulls"
+	}
+}
+
 // /@alice.com/cool-proj/issues/123
 // /@alice.com/cool-proj/issues/123#comment-321
 type ReferenceLink struct {
···
 	SubjectId int
 	CommentId *int
 }
+
+func (l ReferenceLink) String() string {
+	comment := ""
+	if l.CommentId != nil {
+		comment = fmt.Sprintf("#comment-%d", *l.CommentId)
+	}
+	return fmt.Sprintf("/%s/%s/%s/%d%s",
+		l.Handle,
+		l.Repo,
+		l.Kind.String(),
+		l.SubjectId,
+		comment,
+	)
+}
+
+type RichReferenceLink struct {
+	ReferenceLink
+	Title string
+	// reusing PullState for both issue & PR
+	State PullState
+}

appview/pages/pages.go

···
 	Active       string
 	Issue        *models.Issue
 	CommentList  []models.CommentListItem
+	Backlinks    []models.RichReferenceLink
 	LabelDefs    map[string]*models.LabelDefinition
 
 	OrderedReactionKinds []models.ReactionKind
···
 	Pull               *models.Pull
 	Stack              models.Stack
 	AbandonedPulls     []*models.Pull
+	Backlinks          []models.RichReferenceLink
 	BranchDeleteStatus *models.BranchDeleteStatus
 	MergeCheck         types.MergeCheckResponse
 	ResubmitCheck      ResubmitResult

appview/pages/templates/repo/fragments/backlinks.html

···
+{{ define "repo/fragments/backlinks" }}
+  {{ if .Backlinks }}
+  <div id="at-uri-panel" class="px-2 md:px-0">
+    <div>
+      <span class="text-sm py-1 font-bold text-gray-500 dark:text-gray-400">Referenced by</span>
+    </div>
+    <ul>
+      {{ range .Backlinks }}
+      <li>
+        {{ $repoOwner := resolve .Handle }}
+        {{ $repoName := .Repo }}
+        {{ $repoUrl := printf "%s/%s" $repoOwner $repoName }}
+        <div class="flex flex-col">
+          <div class="flex gap-2 items-center">
+            {{ if .State.IsClosed }}
+              <span class="text-gray-500 dark:text-gray-400">
+                {{ i "ban" "w-4 h-4" }}
+              </span>
+            {{ else if eq .Kind.String "issues" }}
+              <span class="text-green-600 dark:text-green-500">
+                {{ i "circle-dot" "w-4 h-4" }}
+              </span>
+            {{ else if .State.IsOpen }}
+              <span class="text-green-600 dark:text-green-500">
+                {{ i "git-pull-request" "w-4 h-4" }}
+              </span>
+            {{ else if .State.IsMerged }}
+              <span class="text-purple-600 dark:text-purple-500">
+                {{ i "git-merge" "w-4 h-4" }}
+              </span>
+            {{ else }}
+              <span class="text-gray-600 dark:text-gray-300">
+                {{ i "git-pull-request-closed" "w-4 h-4" }}
+              </span>
+            {{ end }}
+            <a href="{{ . }}"><span class="text-gray-500 dark:text-gray-400">#{{ .SubjectId }}</span> {{ .Title }}</a>
+          </div>
+          {{ if not (eq $.RepoInfo.FullName $repoUrl) }}
+          <div>
+            <span>on <a href="/{{ $repoUrl }}">{{ $repoUrl }}</a></span>
+          </div>
+          {{ end }}
+        </div>
+      </li>
+      {{ end }}
+    </ul>
+  </div>
+  {{ end }}
+{{ end }}

appview/pages/templates/repo/issues/issue.html

···
               "Subject" $.Issue.AtUri
               "State" $.Issue.Labels) }}
       {{ template "repo/fragments/participants" $.Issue.Participants }}
+      {{ template "repo/fragments/backlinks"
+        (dict "RepoInfo" $.RepoInfo
+              "Backlinks" $.Backlinks) }}
       {{ template "repo/fragments/externalLinkPanel" $.Issue.AtUri }}
     </div>
   </div>

appview/pages/templates/repo/pulls/pull.html

···
               "Subject" $.Pull.AtUri
               "State" $.Pull.Labels) }}
       {{ template "repo/fragments/participants" $.Pull.Participants }}
+      {{ template "repo/fragments/backlinks"
+        (dict "RepoInfo" $.RepoInfo
+              "Backlinks" $.Backlinks) }}
       {{ template "repo/fragments/externalLinkPanel" $.Pull.AtUri }}
     </div>
   </div>

appview/pulls/pulls.go

···
 package pulls
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"errors"
···
 		return
 	}
 
+	backlinks, err := db.GetBacklinks(s.db, pull.AtUri())
+	if err != nil {
+		log.Println("failed to get pull backlinks", err)
+		s.pages.Notice(w, "pull-error", "Failed to get pull. Try again later.")
+		return
+	}
+
 	// can be nil  if this pull is not stacked
 	stack, _ := r.Context().Value("stack").(models.Stack)
 	abandonedPulls, _ := r.Context().Value("abandonedPulls").([]*models.Pull)
···
 		Pull:               pull,
 		Stack:              stack,
 		AbandonedPulls:     abandonedPulls,
+		Backlinks:          backlinks,
 		BranchDeleteStatus: branchDeleteStatus,
 		MergeCheck:         mergeCheckResponse,
 		ResubmitCheck:      resubmitResult,
···
 		}
 	}
 
+	mentions, references := s.refResolver.Resolve(r.Context(), body)
+
 	rkey := tid.TID()
 	initialSubmission := models.PullSubmission{
 		Patch:     patch,
···
 		OwnerDid:     user.Did,
 		RepoAt:       repo.RepoAt(),
 		Rkey:         rkey,
+		Mentions:     mentions,
+		References:   references,
 		Submissions: []*models.PullSubmission{
 			&initialSubmission,
 		},
···
 
 	// build a stack out of this patch
 	stackId := uuid.New()
-	stack, err := newStack(repo, user, targetBranch, patch, pullSource, stackId.String())
+	stack, err := s.newStack(r.Context(), repo, user, targetBranch, patch, pullSource, stackId.String())
 	if err != nil {
 		log.Println("failed to create stack", err)
 		s.pages.Notice(w, "pull", fmt.Sprintf("Failed to create stack: %v", err))
···
 	targetBranch := pull.TargetBranch
 
 	origStack, _ := r.Context().Value("stack").(models.Stack)
-	newStack, err := newStack(repo, user, targetBranch, patch, pull.PullSource, stackId)
+	newStack, err := s.newStack(r.Context(), repo, user, targetBranch, patch, pull.PullSource, stackId)
 	if err != nil {
 		log.Println("failed to create resubmitted stack", err)
 		s.pages.Notice(w, "pull-merge-error", "Failed to merge pull request. Try again later.")
···
 	s.pages.HxLocation(w, fmt.Sprintf("/%s/pulls/%d", ownerSlashRepo, pull.PullId))
 }
 
-func newStack(repo *models.Repo, user *oauth.User, targetBranch, patch string, pullSource *models.PullSource, stackId string) (models.Stack, error) {
+func (s *Pulls) newStack(ctx context.Context, repo *models.Repo, user *oauth.User, targetBranch, patch string, pullSource *models.PullSource, stackId string) (models.Stack, error) {
 	formatPatches, err := patchutil.ExtractPatches(patch)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to extract patches: %v", err)
···
 		body := fp.Body
 		rkey := tid.TID()
 
+		mentions, references := s.refResolver.Resolve(ctx, body)
+
 		initialSubmission := models.PullSubmission{
 			Patch:     fp.Raw,
 			SourceRev: fp.SHA,
···
 			OwnerDid:     user.Did,
 			RepoAt:       repo.RepoAt(),
 			Rkey:         rkey,
+			Mentions:     mentions,
+			References:   references,
 			Submissions: []*models.PullSubmission{
 				&initialSubmission,
 			},