tangled.org
the previous Go-based opengraph rendering made it difficult to handle dynamic content and was cumbersome to update when redesigning images. this moves rendering to a typescript/satori service on cloudflare workers, making design changes easier while offloading image processing from the main application.
+2
.gitignore
+2
.gitignore
+4
-4
.tangled/workflows/deploy-blog.yml
+4
-4
.tangled/workflows/deploy-blog.yml
···
16
16
mkdir -p appview/pages/static
17
17
touch appview/pages/static/x
18
18
19
+
- name: generate css
20
+
command: |
21
+
tailwindcss -i input.css -o appview/pages/static/tw.css
22
+
19
23
- name: build blog cmd
20
24
command: |
21
25
go build -o blog.out ./cmd/blog
22
-
23
-
- name: generate css
24
-
command: |
25
-
tailwindcss -i input.css -o appview/pages/static/tw.css
26
26
27
27
- name: build static site
28
28
command: |
+50
api/tangled/tempanalyzeMerge.go
+50
api/tangled/tempanalyzeMerge.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.analyzeMerge
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempAnalyzeMergeNSID = "sh.tangled.git.temp.analyzeMerge"
15
+
)
16
+
17
+
// GitTempAnalyzeMerge_ConflictInfo is a "conflictInfo" in the sh.tangled.git.temp.analyzeMerge schema.
18
+
type GitTempAnalyzeMerge_ConflictInfo struct {
19
+
// filename: Name of the conflicted file
20
+
Filename string `json:"filename" cborgen:"filename"`
21
+
// reason: Reason for the conflict
22
+
Reason string `json:"reason" cborgen:"reason"`
23
+
}
24
+
25
+
// GitTempAnalyzeMerge_Output is the output of a sh.tangled.git.temp.analyzeMerge call.
26
+
type GitTempAnalyzeMerge_Output struct {
27
+
// conflicts: List of files with merge conflicts
28
+
Conflicts []*GitTempAnalyzeMerge_ConflictInfo `json:"conflicts,omitempty" cborgen:"conflicts,omitempty"`
29
+
// is_conflicted: Whether the merge has conflicts
30
+
Is_conflicted bool `json:"is_conflicted" cborgen:"is_conflicted"`
31
+
}
32
+
33
+
// GitTempAnalyzeMerge calls the XRPC method "sh.tangled.git.temp.analyzeMerge".
34
+
//
35
+
// branch: Target branch to merge into
36
+
// patch: Patch or pull request to check for merge conflicts
37
+
// repo: AT-URI of the repository
38
+
func GitTempAnalyzeMerge(ctx context.Context, c util.LexClient, branch string, patch string, repo string) (*GitTempAnalyzeMerge_Output, error) {
39
+
var out GitTempAnalyzeMerge_Output
40
+
41
+
params := map[string]interface{}{}
42
+
params["branch"] = branch
43
+
params["patch"] = patch
44
+
params["repo"] = repo
45
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.analyzeMerge", params, nil, &out); err != nil {
46
+
return nil, err
47
+
}
48
+
49
+
return &out, nil
50
+
}
+71
api/tangled/tempdefs.go
+71
api/tangled/tempdefs.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.defs
6
+
7
+
import (
8
+
"github.com/bluesky-social/indigo/lex/util"
9
+
)
10
+
11
+
const ()
12
+
13
+
// GitTempDefs_Blob is a "blob" in the sh.tangled.git.temp.defs schema.
14
+
//
15
+
// blob metadata. This object doesn't include the blob content
16
+
type GitTempDefs_Blob struct {
17
+
LastCommit *GitTempDefs_Commit `json:"lastCommit" cborgen:"lastCommit"`
18
+
Mode string `json:"mode" cborgen:"mode"`
19
+
// name: The file name
20
+
Name string `json:"name" cborgen:"name"`
21
+
// size: File size in bytes
22
+
Size int64 `json:"size" cborgen:"size"`
23
+
// submodule: Submodule information if path is a submodule
24
+
Submodule *GitTempDefs_Submodule `json:"submodule,omitempty" cborgen:"submodule,omitempty"`
25
+
}
26
+
27
+
// GitTempDefs_Branch is a "branch" in the sh.tangled.git.temp.defs schema.
28
+
type GitTempDefs_Branch struct {
29
+
// commit: hydrated commit object
30
+
Commit *GitTempDefs_Commit `json:"commit" cborgen:"commit"`
31
+
// name: branch name
32
+
Name string `json:"name" cborgen:"name"`
33
+
}
34
+
35
+
// GitTempDefs_Commit is a "commit" in the sh.tangled.git.temp.defs schema.
36
+
type GitTempDefs_Commit struct {
37
+
Author *GitTempDefs_Signature `json:"author" cborgen:"author"`
38
+
Committer *GitTempDefs_Signature `json:"committer" cborgen:"committer"`
39
+
Hash *string `json:"hash" cborgen:"hash"`
40
+
Message string `json:"message" cborgen:"message"`
41
+
Tree *string `json:"tree" cborgen:"tree"`
42
+
}
43
+
44
+
// GitTempDefs_Signature is a "signature" in the sh.tangled.git.temp.defs schema.
45
+
type GitTempDefs_Signature struct {
46
+
// email: Person email
47
+
Email string `json:"email" cborgen:"email"`
48
+
// name: Person name
49
+
Name string `json:"name" cborgen:"name"`
50
+
// when: Timestamp of the signature
51
+
When string `json:"when" cborgen:"when"`
52
+
}
53
+
54
+
// GitTempDefs_Submodule is a "submodule" in the sh.tangled.git.temp.defs schema.
55
+
type GitTempDefs_Submodule struct {
56
+
// branch: Branch to track in the submodule
57
+
Branch *string `json:"branch,omitempty" cborgen:"branch,omitempty"`
58
+
// name: Submodule name
59
+
Name string `json:"name" cborgen:"name"`
60
+
// url: Submodule repository URL
61
+
Url string `json:"url" cborgen:"url"`
62
+
}
63
+
64
+
// GitTempDefs_Tag is a "tag" in the sh.tangled.git.temp.defs schema.
65
+
type GitTempDefs_Tag struct {
66
+
Message *string `json:"message,omitempty" cborgen:"message,omitempty"`
67
+
// name: tag name
68
+
Name string `json:"name" cborgen:"name"`
69
+
Tagger *GitTempDefs_Signature `json:"tagger" cborgen:"tagger"`
70
+
Target *util.LexiconTypeDecoder `json:"target" cborgen:"target"`
71
+
}
+41
api/tangled/tempgetArchive.go
+41
api/tangled/tempgetArchive.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getArchive
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempGetArchiveNSID = "sh.tangled.git.temp.getArchive"
16
+
)
17
+
18
+
// GitTempGetArchive calls the XRPC method "sh.tangled.git.temp.getArchive".
19
+
//
20
+
// format: Archive format
21
+
// prefix: Prefix for files in the archive
22
+
// ref: Git reference (branch, tag, or commit SHA)
23
+
// repo: AT-URI of the repository
24
+
func GitTempGetArchive(ctx context.Context, c util.LexClient, format string, prefix string, ref string, repo string) ([]byte, error) {
25
+
buf := new(bytes.Buffer)
26
+
27
+
params := map[string]interface{}{}
28
+
if format != "" {
29
+
params["format"] = format
30
+
}
31
+
if prefix != "" {
32
+
params["prefix"] = prefix
33
+
}
34
+
params["ref"] = ref
35
+
params["repo"] = repo
36
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getArchive", params, nil, buf); err != nil {
37
+
return nil, err
38
+
}
39
+
40
+
return buf.Bytes(), nil
41
+
}
+37
api/tangled/tempgetBlob.go
+37
api/tangled/tempgetBlob.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getBlob
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempGetBlobNSID = "sh.tangled.git.temp.getBlob"
16
+
)
17
+
18
+
// GitTempGetBlob calls the XRPC method "sh.tangled.git.temp.getBlob".
19
+
//
20
+
// path: Path within the repository tree
21
+
// ref: Git reference (branch, tag, or commit SHA)
22
+
// repo: AT-URI of the repository
23
+
func GitTempGetBlob(ctx context.Context, c util.LexClient, path string, ref string, repo string) ([]byte, error) {
24
+
buf := new(bytes.Buffer)
25
+
26
+
params := map[string]interface{}{}
27
+
params["path"] = path
28
+
if ref != "" {
29
+
params["ref"] = ref
30
+
}
31
+
params["repo"] = repo
32
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getBlob", params, nil, buf); err != nil {
33
+
return nil, err
34
+
}
35
+
36
+
return buf.Bytes(), nil
37
+
}
+45
api/tangled/tempgetBranch.go
+45
api/tangled/tempgetBranch.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getBranch
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempGetBranchNSID = "sh.tangled.git.temp.getBranch"
15
+
)
16
+
17
+
// GitTempGetBranch_Output is the output of a sh.tangled.git.temp.getBranch call.
18
+
type GitTempGetBranch_Output struct {
19
+
Author *GitTempDefs_Signature `json:"author,omitempty" cborgen:"author,omitempty"`
20
+
// hash: Latest commit hash on this branch
21
+
Hash string `json:"hash" cborgen:"hash"`
22
+
// message: Latest commit message
23
+
Message *string `json:"message,omitempty" cborgen:"message,omitempty"`
24
+
// name: Branch name
25
+
Name string `json:"name" cborgen:"name"`
26
+
// when: Timestamp of latest commit
27
+
When string `json:"when" cborgen:"when"`
28
+
}
29
+
30
+
// GitTempGetBranch calls the XRPC method "sh.tangled.git.temp.getBranch".
31
+
//
32
+
// name: Branch name to get information for
33
+
// repo: AT-URI of the repository
34
+
func GitTempGetBranch(ctx context.Context, c util.LexClient, name string, repo string) (*GitTempGetBranch_Output, error) {
35
+
var out GitTempGetBranch_Output
36
+
37
+
params := map[string]interface{}{}
38
+
params["name"] = name
39
+
params["repo"] = repo
40
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getBranch", params, nil, &out); err != nil {
41
+
return nil, err
42
+
}
43
+
44
+
return &out, nil
45
+
}
+32
api/tangled/tempgetCommit.go
+32
api/tangled/tempgetCommit.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getCommit
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempGetCommitNSID = "sh.tangled.git.temp.getCommit"
15
+
)
16
+
17
+
// GitTempGetCommit calls the XRPC method "sh.tangled.git.temp.getCommit".
18
+
//
19
+
// ref: reference name to resolve
20
+
// repo: AT-URI of the repository
21
+
func GitTempGetCommit(ctx context.Context, c util.LexClient, ref string, repo string) (*GitTempDefs_Commit, error) {
22
+
var out GitTempDefs_Commit
23
+
24
+
params := map[string]interface{}{}
25
+
params["ref"] = ref
26
+
params["repo"] = repo
27
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getCommit", params, nil, &out); err != nil {
28
+
return nil, err
29
+
}
30
+
31
+
return &out, nil
32
+
}
+35
api/tangled/tempgetDiff.go
+35
api/tangled/tempgetDiff.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getDiff
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempGetDiffNSID = "sh.tangled.git.temp.getDiff"
16
+
)
17
+
18
+
// GitTempGetDiff calls the XRPC method "sh.tangled.git.temp.getDiff".
19
+
//
20
+
// repo: AT-URI of the repository
21
+
// rev1: First revision (commit, branch, or tag)
22
+
// rev2: Second revision (commit, branch, or tag)
23
+
func GitTempGetDiff(ctx context.Context, c util.LexClient, repo string, rev1 string, rev2 string) ([]byte, error) {
24
+
buf := new(bytes.Buffer)
25
+
26
+
params := map[string]interface{}{}
27
+
params["repo"] = repo
28
+
params["rev1"] = rev1
29
+
params["rev2"] = rev2
30
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getDiff", params, nil, buf); err != nil {
31
+
return nil, err
32
+
}
33
+
34
+
return buf.Bytes(), nil
35
+
}
+36
api/tangled/tempgetEntity.go
+36
api/tangled/tempgetEntity.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getEntity
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempGetEntityNSID = "sh.tangled.git.temp.getEntity"
15
+
)
16
+
17
+
// GitTempGetEntity calls the XRPC method "sh.tangled.git.temp.getEntity".
18
+
//
19
+
// path: path of the entity
20
+
// ref: Git reference (branch, tag, or commit SHA)
21
+
// repo: AT-URI of the repository
22
+
func GitTempGetEntity(ctx context.Context, c util.LexClient, path string, ref string, repo string) (*GitTempDefs_Blob, error) {
23
+
var out GitTempDefs_Blob
24
+
25
+
params := map[string]interface{}{}
26
+
params["path"] = path
27
+
if ref != "" {
28
+
params["ref"] = ref
29
+
}
30
+
params["repo"] = repo
31
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getEntity", params, nil, &out); err != nil {
32
+
return nil, err
33
+
}
34
+
35
+
return &out, nil
36
+
}
+30
api/tangled/tempgetHead.go
+30
api/tangled/tempgetHead.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getHead
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempGetHeadNSID = "sh.tangled.git.temp.getHead"
15
+
)
16
+
17
+
// GitTempGetHead calls the XRPC method "sh.tangled.git.temp.getHead".
18
+
//
19
+
// repo: AT-URI of the repository
20
+
func GitTempGetHead(ctx context.Context, c util.LexClient, repo string) (*GitTempDefs_Branch, error) {
21
+
var out GitTempDefs_Branch
22
+
23
+
params := map[string]interface{}{}
24
+
params["repo"] = repo
25
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getHead", params, nil, &out); err != nil {
26
+
return nil, err
27
+
}
28
+
29
+
return &out, nil
30
+
}
+33
api/tangled/tempgetTag.go
+33
api/tangled/tempgetTag.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getTag
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempGetTagNSID = "sh.tangled.git.temp.getTag"
16
+
)
17
+
18
+
// GitTempGetTag calls the XRPC method "sh.tangled.git.temp.getTag".
19
+
//
20
+
// repo: AT-URI of the repository
21
+
// tag: Name of tag, such as v1.3.0
22
+
func GitTempGetTag(ctx context.Context, c util.LexClient, repo string, tag string) ([]byte, error) {
23
+
buf := new(bytes.Buffer)
24
+
25
+
params := map[string]interface{}{}
26
+
params["repo"] = repo
27
+
params["tag"] = tag
28
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getTag", params, nil, buf); err != nil {
29
+
return nil, err
30
+
}
31
+
32
+
return buf.Bytes(), nil
33
+
}
+90
api/tangled/tempgetTree.go
+90
api/tangled/tempgetTree.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.getTree
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempGetTreeNSID = "sh.tangled.git.temp.getTree"
15
+
)
16
+
17
+
// GitTempGetTree_LastCommit is a "lastCommit" in the sh.tangled.git.temp.getTree schema.
18
+
type GitTempGetTree_LastCommit struct {
19
+
Author *GitTempGetTree_Signature `json:"author,omitempty" cborgen:"author,omitempty"`
20
+
// hash: Commit hash
21
+
Hash string `json:"hash" cborgen:"hash"`
22
+
// message: Commit message
23
+
Message string `json:"message" cborgen:"message"`
24
+
// when: Commit timestamp
25
+
When string `json:"when" cborgen:"when"`
26
+
}
27
+
28
+
// GitTempGetTree_Output is the output of a sh.tangled.git.temp.getTree call.
29
+
type GitTempGetTree_Output struct {
30
+
// dotdot: Parent directory path
31
+
Dotdot *string `json:"dotdot,omitempty" cborgen:"dotdot,omitempty"`
32
+
Files []*GitTempGetTree_TreeEntry `json:"files" cborgen:"files"`
33
+
LastCommit *GitTempGetTree_LastCommit `json:"lastCommit,omitempty" cborgen:"lastCommit,omitempty"`
34
+
// parent: The parent path in the tree
35
+
Parent *string `json:"parent,omitempty" cborgen:"parent,omitempty"`
36
+
// readme: Readme for this file tree
37
+
Readme *GitTempGetTree_Readme `json:"readme,omitempty" cborgen:"readme,omitempty"`
38
+
// ref: The git reference used
39
+
Ref string `json:"ref" cborgen:"ref"`
40
+
}
41
+
42
+
// GitTempGetTree_Readme is a "readme" in the sh.tangled.git.temp.getTree schema.
43
+
type GitTempGetTree_Readme struct {
44
+
// contents: Contents of the readme file
45
+
Contents string `json:"contents" cborgen:"contents"`
46
+
// filename: Name of the readme file
47
+
Filename string `json:"filename" cborgen:"filename"`
48
+
}
49
+
50
+
// GitTempGetTree_Signature is a "signature" in the sh.tangled.git.temp.getTree schema.
51
+
type GitTempGetTree_Signature struct {
52
+
// email: Author email
53
+
Email string `json:"email" cborgen:"email"`
54
+
// name: Author name
55
+
Name string `json:"name" cborgen:"name"`
56
+
// when: Author timestamp
57
+
When string `json:"when" cborgen:"when"`
58
+
}
59
+
60
+
// GitTempGetTree_TreeEntry is a "treeEntry" in the sh.tangled.git.temp.getTree schema.
61
+
type GitTempGetTree_TreeEntry struct {
62
+
Last_commit *GitTempGetTree_LastCommit `json:"last_commit,omitempty" cborgen:"last_commit,omitempty"`
63
+
// mode: File mode
64
+
Mode string `json:"mode" cborgen:"mode"`
65
+
// name: Relative file or directory name
66
+
Name string `json:"name" cborgen:"name"`
67
+
// size: File size in bytes
68
+
Size int64 `json:"size" cborgen:"size"`
69
+
}
70
+
71
+
// GitTempGetTree calls the XRPC method "sh.tangled.git.temp.getTree".
72
+
//
73
+
// path: Path within the repository tree
74
+
// ref: Git reference (branch, tag, or commit SHA)
75
+
// repo: AT-URI of the repository
76
+
func GitTempGetTree(ctx context.Context, c util.LexClient, path string, ref string, repo string) (*GitTempGetTree_Output, error) {
77
+
var out GitTempGetTree_Output
78
+
79
+
params := map[string]interface{}{}
80
+
if path != "" {
81
+
params["path"] = path
82
+
}
83
+
params["ref"] = ref
84
+
params["repo"] = repo
85
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.getTree", params, nil, &out); err != nil {
86
+
return nil, err
87
+
}
88
+
89
+
return &out, nil
90
+
}
+39
api/tangled/templistBranches.go
+39
api/tangled/templistBranches.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.listBranches
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempListBranchesNSID = "sh.tangled.git.temp.listBranches"
16
+
)
17
+
18
+
// GitTempListBranches calls the XRPC method "sh.tangled.git.temp.listBranches".
19
+
//
20
+
// cursor: Pagination cursor
21
+
// limit: Maximum number of branches to return
22
+
// repo: AT-URI of the repository
23
+
func GitTempListBranches(ctx context.Context, c util.LexClient, cursor string, limit int64, repo string) ([]byte, error) {
24
+
buf := new(bytes.Buffer)
25
+
26
+
params := map[string]interface{}{}
27
+
if cursor != "" {
28
+
params["cursor"] = cursor
29
+
}
30
+
if limit != 0 {
31
+
params["limit"] = limit
32
+
}
33
+
params["repo"] = repo
34
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.listBranches", params, nil, buf); err != nil {
35
+
return nil, err
36
+
}
37
+
38
+
return buf.Bytes(), nil
39
+
}
+43
api/tangled/templistCommits.go
+43
api/tangled/templistCommits.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.listCommits
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempListCommitsNSID = "sh.tangled.git.temp.listCommits"
16
+
)
17
+
18
+
// GitTempListCommits calls the XRPC method "sh.tangled.git.temp.listCommits".
19
+
//
20
+
// cursor: Pagination cursor (commit SHA)
21
+
// limit: Maximum number of commits to return
22
+
// ref: Git reference (branch, tag, or commit SHA)
23
+
// repo: AT-URI of the repository
24
+
func GitTempListCommits(ctx context.Context, c util.LexClient, cursor string, limit int64, ref string, repo string) ([]byte, error) {
25
+
buf := new(bytes.Buffer)
26
+
27
+
params := map[string]interface{}{}
28
+
if cursor != "" {
29
+
params["cursor"] = cursor
30
+
}
31
+
if limit != 0 {
32
+
params["limit"] = limit
33
+
}
34
+
if ref != "" {
35
+
params["ref"] = ref
36
+
}
37
+
params["repo"] = repo
38
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.listCommits", params, nil, buf); err != nil {
39
+
return nil, err
40
+
}
41
+
42
+
return buf.Bytes(), nil
43
+
}
+61
api/tangled/templistLanguages.go
+61
api/tangled/templistLanguages.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.listLanguages
6
+
7
+
import (
8
+
"context"
9
+
10
+
"github.com/bluesky-social/indigo/lex/util"
11
+
)
12
+
13
+
const (
14
+
GitTempListLanguagesNSID = "sh.tangled.git.temp.listLanguages"
15
+
)
16
+
17
+
// GitTempListLanguages_Language is a "language" in the sh.tangled.git.temp.listLanguages schema.
18
+
type GitTempListLanguages_Language struct {
19
+
// color: Hex color code for this language
20
+
Color *string `json:"color,omitempty" cborgen:"color,omitempty"`
21
+
// extensions: File extensions associated with this language
22
+
Extensions []string `json:"extensions,omitempty" cborgen:"extensions,omitempty"`
23
+
// fileCount: Number of files in this language
24
+
FileCount *int64 `json:"fileCount,omitempty" cborgen:"fileCount,omitempty"`
25
+
// name: Programming language name
26
+
Name string `json:"name" cborgen:"name"`
27
+
// percentage: Percentage of total codebase (0-100)
28
+
Percentage int64 `json:"percentage" cborgen:"percentage"`
29
+
// size: Total size of files in this language (bytes)
30
+
Size int64 `json:"size" cborgen:"size"`
31
+
}
32
+
33
+
// GitTempListLanguages_Output is the output of a sh.tangled.git.temp.listLanguages call.
34
+
type GitTempListLanguages_Output struct {
35
+
Languages []*GitTempListLanguages_Language `json:"languages" cborgen:"languages"`
36
+
// ref: The git reference used
37
+
Ref string `json:"ref" cborgen:"ref"`
38
+
// totalFiles: Total number of files analyzed
39
+
TotalFiles *int64 `json:"totalFiles,omitempty" cborgen:"totalFiles,omitempty"`
40
+
// totalSize: Total size of all analyzed files in bytes
41
+
TotalSize *int64 `json:"totalSize,omitempty" cborgen:"totalSize,omitempty"`
42
+
}
43
+
44
+
// GitTempListLanguages calls the XRPC method "sh.tangled.git.temp.listLanguages".
45
+
//
46
+
// ref: Git reference (branch, tag, or commit SHA)
47
+
// repo: AT-URI of the repository
48
+
func GitTempListLanguages(ctx context.Context, c util.LexClient, ref string, repo string) (*GitTempListLanguages_Output, error) {
49
+
var out GitTempListLanguages_Output
50
+
51
+
params := map[string]interface{}{}
52
+
if ref != "" {
53
+
params["ref"] = ref
54
+
}
55
+
params["repo"] = repo
56
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.listLanguages", params, nil, &out); err != nil {
57
+
return nil, err
58
+
}
59
+
60
+
return &out, nil
61
+
}
+39
api/tangled/templistTags.go
+39
api/tangled/templistTags.go
···
1
+
// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
2
+
3
+
package tangled
4
+
5
+
// schema: sh.tangled.git.temp.listTags
6
+
7
+
import (
8
+
"bytes"
9
+
"context"
10
+
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
)
13
+
14
+
const (
15
+
GitTempListTagsNSID = "sh.tangled.git.temp.listTags"
16
+
)
17
+
18
+
// GitTempListTags calls the XRPC method "sh.tangled.git.temp.listTags".
19
+
//
20
+
// cursor: Pagination cursor
21
+
// limit: Maximum number of tags to return
22
+
// repo: AT-URI of the repository
23
+
func GitTempListTags(ctx context.Context, c util.LexClient, cursor string, limit int64, repo string) ([]byte, error) {
24
+
buf := new(bytes.Buffer)
25
+
26
+
params := map[string]interface{}{}
27
+
if cursor != "" {
28
+
params["cursor"] = cursor
29
+
}
30
+
if limit != 0 {
31
+
params["limit"] = limit
32
+
}
33
+
params["repo"] = repo
34
+
if err := c.LexDo(ctx, util.Query, "", "sh.tangled.git.temp.listTags", params, nil, buf); err != nil {
35
+
return nil, err
36
+
}
37
+
38
+
return buf.Bytes(), nil
39
+
}
+22
-17
appview/config/config.go
+22
-17
appview/config/config.go
···
46
46
PLCURL string `env:"URL, default=https://plc.directory"`
47
47
}
48
48
49
+
type KnotMirrorConfig struct {
50
+
Url string `env:"URL, required"`
51
+
}
52
+
49
53
type JetstreamConfig struct {
50
54
Endpoint string `env:"ENDPOINT, default=wss://jetstream1.us-east.bsky.network/subscribe"`
51
55
}
···
154
158
}
155
159
156
160
type Config struct {
157
-
Core CoreConfig `env:",prefix=TANGLED_"`
158
-
Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"`
159
-
Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"`
160
-
Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"`
161
-
Resend ResendConfig `env:",prefix=TANGLED_RESEND_"`
162
-
Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"`
163
-
Camo CamoConfig `env:",prefix=TANGLED_CAMO_"`
164
-
Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"`
165
-
OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"`
166
-
Redis RedisConfig `env:",prefix=TANGLED_REDIS_"`
167
-
Plc PlcConfig `env:",prefix=TANGLED_PLC_"`
168
-
Pds PdsConfig `env:",prefix=TANGLED_PDS_"`
169
-
Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"`
170
-
Label LabelConfig `env:",prefix=TANGLED_LABEL_"`
171
-
Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"`
172
-
Sites SitesConfig `env:",prefix=TANGLED_SITES_"`
173
-
Ogcard OgcardConfig `env:",prefix=TANGLED_OGCARD_"`
161
+
Core CoreConfig `env:",prefix=TANGLED_"`
162
+
Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"`
163
+
Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"`
164
+
Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"`
165
+
Resend ResendConfig `env:",prefix=TANGLED_RESEND_"`
166
+
Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"`
167
+
Camo CamoConfig `env:",prefix=TANGLED_CAMO_"`
168
+
Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"`
169
+
OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"`
170
+
Redis RedisConfig `env:",prefix=TANGLED_REDIS_"`
171
+
Plc PlcConfig `env:",prefix=TANGLED_PLC_"`
172
+
Pds PdsConfig `env:",prefix=TANGLED_PDS_"`
173
+
Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"`
174
+
Label LabelConfig `env:",prefix=TANGLED_LABEL_"`
175
+
Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"`
176
+
Sites SitesConfig `env:",prefix=TANGLED_SITES_"`
177
+
KnotMirror KnotMirrorConfig `env:",prefix=TANGLED_KNOTMIRROR_"`
178
+
Ogcard OgcardConfig `env:",prefix=TANGLED_OGCARD_"`
174
179
}
175
180
176
181
func LoadConfig(ctx context.Context) (*Config, error) {
+2
-2
appview/issues/issues.go
+2
-2
appview/issues/issues.go
···
10
10
"time"
11
11
12
12
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
13
+
"github.com/bluesky-social/indigo/atproto/atclient"
14
14
"github.com/bluesky-social/indigo/atproto/syntax"
15
15
lexutil "github.com/bluesky-social/indigo/lex/util"
16
16
"github.com/go-chi/chi/v5"
···
1101
1101
// this is used to rollback changes made to the PDS
1102
1102
//
1103
1103
// it is a no-op if the provided ATURI is empty
1104
-
func rollbackRecord(ctx context.Context, aturi string, client *atpclient.APIClient) error {
1104
+
func rollbackRecord(ctx context.Context, aturi string, client *atclient.APIClient) error {
1105
1105
if aturi == "" {
1106
1106
return nil
1107
1107
}
appview/issues/opengraph.go
appview/issues/opengraph.go
This file has not been changed.
+2
-2
appview/labels/labels.go
+2
-2
appview/labels/labels.go
···
22
22
"tangled.org/core/tid"
23
23
24
24
comatproto "github.com/bluesky-social/indigo/api/atproto"
25
+
"github.com/bluesky-social/indigo/atproto/atclient"
25
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
26
26
"github.com/bluesky-social/indigo/atproto/syntax"
27
27
lexutil "github.com/bluesky-social/indigo/lex/util"
28
28
"github.com/go-chi/chi/v5"
···
269
269
// this is used to rollback changes made to the PDS
270
270
//
271
271
// it is a no-op if the provided ATURI is empty
272
+
func rollbackRecord(ctx context.Context, aturi string, client *atclient.APIClient) error {
272
-
func rollbackRecord(ctx context.Context, aturi string, client *atpclient.APIClient) error {
273
273
if aturi == "" {
274
274
return nil
275
275
}
+3
-3
appview/oauth/oauth.go
+3
-3
appview/oauth/oauth.go
···
11
11
"time"
12
12
13
13
comatproto "github.com/bluesky-social/indigo/api/atproto"
14
+
"github.com/bluesky-social/indigo/atproto/atclient"
15
+
"github.com/bluesky-social/indigo/atproto/atcrypto"
14
16
"github.com/bluesky-social/indigo/atproto/auth/oauth"
15
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
16
-
atcrypto "github.com/bluesky-social/indigo/atproto/crypto"
17
17
"github.com/bluesky-social/indigo/atproto/syntax"
18
18
xrpc "github.com/bluesky-social/indigo/xrpc"
19
19
"github.com/gorilla/sessions"
···
262
262
return ""
263
263
}
264
264
265
+
func (o *OAuth) AuthorizedClient(r *http.Request) (*atclient.APIClient, error) {
265
-
func (o *OAuth) AuthorizedClient(r *http.Request) (*atpclient.APIClient, error) {
266
266
session, err := o.ResumeSession(r)
267
267
if err != nil {
268
268
return nil, fmt.Errorf("error getting session: %w", err)
appview/ogcard/.gitignore
appview/ogcard/.gitignore
This file has not been changed.
appview/ogcard/bun.lock
appview/ogcard/bun.lock
This file has not been changed.
appview/ogcard/card.go
appview/ogcard/card.go
This file has not been changed.
appview/ogcard/client.go
appview/ogcard/client.go
This file has not been changed.
appview/ogcard/knip.json
appview/ogcard/knip.json
This file has not been changed.
appview/ogcard/package.json
appview/ogcard/package.json
This file has not been changed.
appview/ogcard/packages/runtime/index.ts
appview/ogcard/packages/runtime/index.ts
This file has not been changed.
appview/ogcard/packages/runtime/package.json
appview/ogcard/packages/runtime/package.json
This file has not been changed.
appview/ogcard/packages/runtime/types.ts
appview/ogcard/packages/runtime/types.ts
This file has not been changed.
appview/ogcard/packages/runtime/workerd.ts
appview/ogcard/packages/runtime/workerd.ts
This file has not been changed.
appview/ogcard/src/__tests__/assets/avatar.jpg
appview/ogcard/src/__tests__/assets/avatar.jpg
Failed to calculate interdiff for this file.
appview/ogcard/src/__tests__/fixtures.ts
appview/ogcard/src/__tests__/fixtures.ts
This file has not been changed.
appview/ogcard/src/__tests__/render.test.ts
appview/ogcard/src/__tests__/render.test.ts
This file has not been changed.
appview/ogcard/src/components/cards/issue.tsx
appview/ogcard/src/components/cards/issue.tsx
This file has not been changed.
appview/ogcard/src/components/cards/pull-request.tsx
appview/ogcard/src/components/cards/pull-request.tsx
This file has not been changed.
appview/ogcard/src/components/cards/repository.tsx
appview/ogcard/src/components/cards/repository.tsx
This file has not been changed.
appview/ogcard/src/icons/lucide.tsx
appview/ogcard/src/icons/lucide.tsx
This file has not been changed.
appview/ogcard/src/index.tsx
appview/ogcard/src/index.tsx
This file has not been changed.
appview/ogcard/src/lib/render.ts
appview/ogcard/src/lib/render.ts
This file has not been changed.
appview/ogcard/src/types.d.ts
appview/ogcard/src/types.d.ts
This file has not been changed.
appview/ogcard/src/validation.ts
appview/ogcard/src/validation.ts
This file has not been changed.
appview/ogcard/tsconfig.json
appview/ogcard/tsconfig.json
This file has not been changed.
appview/ogcard/wrangler.jsonc
appview/ogcard/wrangler.jsonc
This file has not been changed.
+7
-1
appview/pages/funcmap.go
+7
-1
appview/pages/funcmap.go
···
195
195
{D: math.MaxInt64, Format: "a long while %s", DivBy: 1},
196
196
})
197
197
},
198
+
"shortTimeFmt": func(t time.Time) string {
199
+
return t.Format("Jan 2, 2006")
200
+
},
198
201
"longTimeFmt": func(t time.Time) string {
199
202
return t.Format("Jan 2, 2006, 3:04 PM MST")
200
203
},
···
209
212
return fmt.Sprintf("P%dD%dH%dM%dS", days, hours, minutes, seconds)
210
213
},
211
214
"durationFmt": func(duration time.Duration) string {
215
+
return durationFmt(duration, [4]string{"d", "h", "m", "s"})
212
-
return durationFmt(duration, [4]string{"d", "hr", "min", "s"})
213
216
},
214
217
"longDurationFmt": func(duration time.Duration) string {
215
218
return durationFmt(duration, [4]string{"days", "hours", "minutes", "seconds"})
···
521
524
}
522
525
523
526
secret := p.avatar.SharedSecret
527
+
if secret == "" {
528
+
return ""
529
+
}
524
530
h := hmac.New(sha256.New, []byte(secret))
525
531
h.Write([]byte(did))
526
532
signature := hex.EncodeToString(h.Sum(nil))
+7
-3
appview/pages/templates/repo/pipelines/workflow.html
+7
-3
appview/pages/templates/repo/pipelines/workflow.html
···
48
48
{{ $lastStatus := $all.Latest }}
49
49
{{ $kind := $lastStatus.Status.String }}
50
50
51
+
<div id="left" class="flex items-center gap-2 flex-1 min-w-0">
52
+
<div class="flex-shrink-0">
53
+
{{ template "repo/pipelines/fragments/workflowSymbol" $all }}
54
+
</div>
55
+
<span class="truncate" title="{{ $name }}">
56
+
{{ $name }}
57
+
</span>
51
-
<div id="left" class="flex items-center gap-2 flex-shrink-0">
52
-
{{ template "repo/pipelines/fragments/workflowSymbol" $all }}
53
-
{{ $name }}
54
58
</div>
55
59
<div id="right" class="flex items-center gap-2 flex-shrink-0">
56
60
<span class="font-bold">{{ $kind }}</span>
+1
appview/pages/templates/repo/pulls/pull.html
+1
appview/pages/templates/repo/pulls/pull.html
appview/pulls/opengraph.go
appview/pulls/opengraph.go
This file has not been changed.
+16
-82
appview/pulls/pulls.go
+16
-82
appview/pulls/pulls.go
···
414
414
return nil
415
415
}
416
416
417
-
scheme := "http"
418
-
if !s.config.Core.Dev {
419
-
scheme = "https"
420
-
}
421
-
host := fmt.Sprintf("%s://%s", scheme, repo.Knot)
422
-
xrpcc := &indigoxrpc.Client{
423
-
Host: host,
424
-
}
425
-
426
-
resp, err := tangled.RepoBranch(r.Context(), xrpcc, branch, fmt.Sprintf("%s/%s", repo.Did, repo.Name))
417
+
xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url}
418
+
resp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, branch, repo.RepoAt().String())
427
419
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
428
420
return nil
429
421
}
···
439
431
return pages.Unknown
440
432
}
441
433
442
-
var knot, ownerDid, repoName string
443
-
434
+
var sourceRepo syntax.ATURI
444
435
if pull.PullSource.RepoAt != nil {
445
436
// fork-based pulls
446
-
sourceRepo, err := db.GetRepoByAtUri(s.db, pull.PullSource.RepoAt.String())
447
-
if err != nil {
448
-
log.Println("failed to get source repo", err)
449
-
return pages.Unknown
450
-
}
451
-
452
-
knot = sourceRepo.Knot
453
-
ownerDid = sourceRepo.Did
454
-
repoName = sourceRepo.Name
437
+
sourceRepo = *pull.PullSource.RepoAt
455
438
} else {
456
439
// pulls within the same repo
457
-
knot = repo.Knot
458
-
ownerDid = repo.Did
459
-
repoName = repo.Name
440
+
sourceRepo = repo.RepoAt()
460
441
}
461
442
462
-
scheme := "http"
463
-
if !s.config.Core.Dev {
464
-
scheme = "https"
465
-
}
466
-
host := fmt.Sprintf("%s://%s", scheme, knot)
467
-
xrpcc := &indigoxrpc.Client{
468
-
Host: host,
469
-
}
470
-
471
-
didSlashName := fmt.Sprintf("%s/%s", ownerDid, repoName)
472
-
branchResp, err := tangled.RepoBranch(r.Context(), xrpcc, pull.PullSource.Branch, didSlashName)
443
+
xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url}
444
+
branchResp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, pull.PullSource.Branch, sourceRepo.String())
473
445
if err != nil {
474
446
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
475
447
log.Println("failed to call XRPC repo.branches", xrpcerr)
···
907
879
908
880
switch r.Method {
909
881
case http.MethodGet:
910
-
scheme := "http"
911
-
if !s.config.Core.Dev {
912
-
scheme = "https"
913
-
}
914
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
915
-
xrpcc := &indigoxrpc.Client{
916
-
Host: host,
917
-
}
882
+
xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url}
918
883
919
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
920
-
xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
884
+
xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
921
885
if err != nil {
922
886
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
923
887
log.Println("failed to call XRPC repo.branches", xrpcerr)
···
1538
1502
return
1539
1503
}
1540
1504
1541
-
scheme := "http"
1542
-
if !s.config.Core.Dev {
1543
-
scheme = "https"
1544
-
}
1545
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
1546
-
xrpcc := &indigoxrpc.Client{
1547
-
Host: host,
1548
-
}
1505
+
xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url}
1549
1506
1550
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
1551
-
xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
1507
+
xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
1552
1508
if err != nil {
1553
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
1554
-
log.Println("failed to call XRPC repo.branches", xrpcerr)
1555
-
s.pages.Error503(w)
1556
-
return
1557
-
}
1558
1509
log.Println("failed to fetch branches", err)
1510
+
s.pages.Error503(w)
1559
1511
return
1560
1512
}
1561
1513
···
1610
1562
return
1611
1563
}
1612
1564
1565
+
xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url}
1566
+
1613
1567
forkVal := r.URL.Query().Get("fork")
1614
1568
repoString := strings.SplitN(forkVal, "/", 2)
1615
1569
forkOwnerDid := repoString[0]
···
1625
1579
return
1626
1580
}
1627
1581
1628
-
sourceScheme := "http"
1629
-
if !s.config.Core.Dev {
1630
-
sourceScheme = "https"
1631
-
}
1632
-
sourceHost := fmt.Sprintf("%s://%s", sourceScheme, repo.Knot)
1633
-
sourceXrpcc := &indigoxrpc.Client{
1634
-
Host: sourceHost,
1635
-
}
1636
-
1637
-
sourceRepo := fmt.Sprintf("%s/%s", forkOwnerDid, repo.Name)
1638
-
sourceXrpcBytes, err := tangled.RepoBranches(r.Context(), sourceXrpcc, "", 0, sourceRepo)
1582
+
sourceXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, repo.RepoAt().String())
1639
1583
if err != nil {
1640
1584
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
1641
1585
log.Println("failed to call XRPC repo.branches for source", xrpcerr)
···
1654
1598
return
1655
1599
}
1656
1600
1657
-
targetScheme := "http"
1658
-
if !s.config.Core.Dev {
1659
-
targetScheme = "https"
1660
-
}
1661
-
targetHost := fmt.Sprintf("%s://%s", targetScheme, f.Knot)
1662
-
targetXrpcc := &indigoxrpc.Client{
1663
-
Host: targetHost,
1664
-
}
1665
-
1666
-
targetRepo := fmt.Sprintf("%s/%s", f.Did, f.Name)
1667
-
targetXrpcBytes, err := tangled.RepoBranches(r.Context(), targetXrpcc, "", 0, targetRepo)
1601
+
targetXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
1668
1602
if err != nil {
1669
1603
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
1670
1604
log.Println("failed to call XRPC repo.branches for target", xrpcerr)
+9
-19
appview/repo/archive.go
+9
-19
appview/repo/archive.go
···
8
8
"strings"
9
9
10
10
"github.com/go-chi/chi/v5"
11
+
"tangled.org/core/api/tangled"
11
12
)
12
13
13
14
func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) {
···
20
21
l.Error("failed to get repo and knot", "err", err)
21
22
return
22
23
}
23
-
scheme := "http"
24
-
if !rp.config.Core.Dev {
25
-
scheme = "https"
26
-
}
27
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
28
-
didSlashRepo := f.DidSlashRepo()
29
24
30
25
// build the xrpc url
31
-
u, err := url.Parse(host)
32
-
if err != nil {
33
-
l.Error("failed to parse host URL", "err", err)
34
-
rp.pages.Error503(w)
35
-
return
36
-
}
37
-
38
-
u.Path = "/xrpc/sh.tangled.repo.archive"
39
26
query := url.Values{}
27
+
query.Set("repo", f.RepoAt().String())
28
+
query.Set("ref", ref)
40
29
query.Set("format", "tar.gz")
41
30
query.Set("prefix", r.URL.Query().Get("prefix"))
31
+
xrpcURL := fmt.Sprintf(
32
+
"%s/xrpc/%s?%s",
33
+
rp.config.KnotMirror.Url,
34
+
tangled.GitTempGetArchiveNSID,
35
+
query.Encode(),
36
+
)
42
-
query.Set("ref", ref)
43
-
query.Set("repo", didSlashRepo)
44
-
u.RawQuery = query.Encode()
45
-
46
-
xrpcURL := u.String()
47
37
48
38
// make the get request
49
39
resp, err := http.Get(xrpcURL)
+2
-10
appview/repo/artifact.go
+2
-10
appview/repo/artifact.go
···
313
313
return nil, err
314
314
}
315
315
316
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
316
-
scheme := "http"
317
-
if !rp.config.Core.Dev {
318
-
scheme = "https"
319
-
}
320
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
321
-
xrpcc := &indigoxrpc.Client{
322
-
Host: host,
323
-
}
324
317
318
+
xrpcBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, f.RepoAt().String())
325
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
326
-
xrpcBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, repo)
327
319
if err != nil {
328
320
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
329
321
l.Error("failed to call XRPC repo.tags", "err", xrpcerr)
+5
-12
appview/repo/branches.go
+5
-12
appview/repo/branches.go
···
21
21
l.Error("failed to get repo and knot", "err", err)
22
22
return
23
23
}
24
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
25
+
26
+
xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
27
+
if err != nil {
28
+
l.Error("failed to call XRPC repo.branches", "err", err)
24
-
scheme := "http"
25
-
if !rp.config.Core.Dev {
26
-
scheme = "https"
27
-
}
28
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
29
-
xrpcc := &indigoxrpc.Client{
30
-
Host: host,
31
-
}
32
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
33
-
xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
34
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
35
-
l.Error("failed to call XRPC repo.branches", "err", xrpcerr)
36
29
rp.pages.Error503(w)
37
30
return
38
31
}
+3
-11
appview/repo/compare.go
+3
-11
appview/repo/compare.go
···
27
27
return
28
28
}
29
29
30
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
30
-
scheme := "http"
31
-
if !rp.config.Core.Dev {
32
-
scheme = "https"
33
-
}
34
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
35
-
xrpcc := &indigoxrpc.Client{
36
-
Host: host,
37
-
}
38
31
32
+
branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
39
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
40
-
branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
41
33
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
42
34
l.Error("failed to call XRPC repo.branches", "err", xrpcerr)
43
35
rp.pages.Error503(w)
···
74
66
head = queryHead
75
67
}
76
68
69
+
tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String())
77
-
tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo)
78
70
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
79
71
l.Error("failed to call XRPC repo.tags", "err", xrpcerr)
80
72
rp.pages.Error503(w)
+34
-56
appview/repo/index.go
+34
-56
appview/repo/index.go
···
22
22
"tangled.org/core/appview/db"
23
23
"tangled.org/core/appview/models"
24
24
"tangled.org/core/appview/pages"
25
-
"tangled.org/core/appview/xrpcclient"
26
25
"tangled.org/core/orm"
27
26
"tangled.org/core/types"
28
27
···
42
41
return
43
42
}
44
43
45
-
scheme := "http"
46
-
if !rp.config.Core.Dev {
47
-
scheme = "https"
48
-
}
49
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
50
-
xrpcc := &indigoxrpc.Client{
51
-
Host: host,
52
-
}
53
-
54
44
user := rp.oauth.GetMultiAccountUser(r)
55
45
56
46
// Build index response from multiple XRPC calls
47
+
result, err := rp.buildIndexResponse(r.Context(), f, ref)
48
+
if err != nil {
49
+
l.Error("failed to build index response", "err", err)
50
+
rp.pages.RepoIndexPage(w, pages.RepoIndexParams{
51
+
LoggedInUser: user,
52
+
KnotUnreachable: true,
53
+
RepoInfo: rp.repoResolver.GetRepoInfo(r, user),
54
+
})
55
+
return
57
-
result, err := rp.buildIndexResponse(r.Context(), xrpcc, f, ref)
58
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
59
-
if errors.Is(xrpcerr, xrpcclient.ErrXrpcUnsupported) {
60
-
l.Error("failed to call XRPC repo.index", "err", err)
61
-
rp.pages.RepoIndexPage(w, pages.RepoIndexParams{
62
-
LoggedInUser: user,
63
-
NeedsKnotUpgrade: true,
64
-
RepoInfo: rp.repoResolver.GetRepoInfo(r, user),
65
-
})
66
-
return
67
-
} else {
68
-
l.Error("failed to build index response", "err", err)
69
-
rp.pages.RepoIndexPage(w, pages.RepoIndexParams{
70
-
LoggedInUser: user,
71
-
KnotUnreachable: true,
72
-
RepoInfo: rp.repoResolver.GetRepoInfo(r, user),
73
-
})
74
-
return
75
-
}
76
56
}
77
57
78
58
tagMap := make(map[string][]string)
···
132
112
l.Error("failed to GetVerifiedObjectCommits", "err", err)
133
113
}
134
114
115
+
var languageInfo []types.RepoLanguageDetails
116
+
if !result.IsEmpty {
117
+
// TODO: a bit dirty
118
+
languageInfo, err = rp.getLanguageInfo(r.Context(), l, f, result.Ref, ref == "")
119
+
if err != nil {
120
+
l.Warn("failed to compute language percentages", "err", err)
121
+
// non-fatal
122
+
}
135
-
// TODO: a bit dirty
136
-
languageInfo, err := rp.getLanguageInfo(r.Context(), l, f, xrpcc, result.Ref, ref == "")
137
-
if err != nil {
138
-
l.Warn("failed to compute language percentages", "err", err)
139
-
// non-fatal
140
123
}
141
124
142
125
var shas []string
···
169
152
ctx context.Context,
170
153
l *slog.Logger,
171
154
repo *models.Repo,
172
-
xrpcc *indigoxrpc.Client,
173
155
currentRef string,
174
156
isDefaultRef bool,
175
157
) ([]types.RepoLanguageDetails, error) {
···
182
164
183
165
if err != nil || langs == nil {
184
166
// non-fatal, fetch langs from ks via XRPC
167
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
168
+
ls, err := tangled.GitTempListLanguages(ctx, xrpcc, currentRef, repo.RepoAt().String())
185
-
didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name)
186
-
ls, err := tangled.RepoLanguages(ctx, xrpcc, currentRef, didSlashRepo)
187
169
if err != nil {
170
+
return nil, fmt.Errorf("calling knotmirror git.listLanguages: %w", err)
188
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
189
-
l.Error("failed to call XRPC repo.languages", "err", xrpcerr)
190
-
return nil, xrpcerr
191
-
}
192
-
return nil, err
193
171
}
194
172
195
173
if ls == nil || ls.Languages == nil {
···
258
236
}
259
237
260
238
// buildIndexResponse creates a RepoIndexResponse by combining multiple xrpc calls in parallel
239
+
func (rp *Repo) buildIndexResponse(ctx context.Context, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) {
240
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
261
-
func (rp *Repo) buildIndexResponse(ctx context.Context, xrpcc *indigoxrpc.Client, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) {
262
-
didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name)
263
241
264
242
// first get branches to determine the ref if not specified
243
+
branchesBytes, err := tangled.GitTempListBranches(ctx, xrpcc, "", 0, repo.RepoAt().String())
265
-
branchesBytes, err := tangled.RepoBranches(ctx, xrpcc, "", 0, didSlashRepo)
266
244
if err != nil {
245
+
return nil, fmt.Errorf("calling knotmirror git.listBranches: %w", err)
267
-
return nil, fmt.Errorf("failed to call repoBranches: %w", err)
268
246
}
269
247
270
248
var branchesResp types.RepoBranchesResponse
···
296
274
297
275
var (
298
276
tagsResp types.RepoTagsResponse
277
+
treeResp *tangled.GitTempGetTree_Output
299
-
treeResp *tangled.RepoTree_Output
300
278
logResp types.RepoLogResponse
301
279
readmeContent string
302
280
readmeFileName string
···
304
282
305
283
// tags
306
284
wg.Go(func() {
285
+
tagsBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, repo.RepoAt().String())
307
-
tagsBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, didSlashRepo)
308
286
if err != nil {
287
+
errs = errors.Join(errs, fmt.Errorf("failed to call git.ListTags: %w", err))
309
-
errs = errors.Join(errs, fmt.Errorf("failed to call repoTags: %w", err))
310
288
return
311
289
}
312
290
313
291
if err := json.Unmarshal(tagsBytes, &tagsResp); err != nil {
292
+
errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListTags: %w", err))
314
-
errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoTags: %w", err))
315
293
}
316
294
})
317
295
318
296
// tree/files
319
297
wg.Go(func() {
298
+
resp, err := tangled.GitTempGetTree(ctx, xrpcc, "", ref, repo.RepoAt().String())
320
-
resp, err := tangled.RepoTree(ctx, xrpcc, "", ref, didSlashRepo)
321
299
if err != nil {
300
+
errs = errors.Join(errs, fmt.Errorf("failed to call git.GetTree: %w", err))
322
-
errs = errors.Join(errs, fmt.Errorf("failed to call repoTree: %w", err))
323
301
return
324
302
}
325
303
treeResp = resp
···
327
305
328
306
// commits
329
307
wg.Go(func() {
308
+
logBytes, err := tangled.GitTempListCommits(ctx, xrpcc, "", 50, ref, repo.RepoAt().String())
330
-
logBytes, err := tangled.RepoLog(ctx, xrpcc, "", 50, "", ref, didSlashRepo)
331
309
if err != nil {
310
+
errs = errors.Join(errs, fmt.Errorf("failed to call git.ListCommits: %w", err))
332
-
errs = errors.Join(errs, fmt.Errorf("failed to call repoLog: %w", err))
333
311
return
334
312
}
335
313
336
314
if err := json.Unmarshal(logBytes, &logResp); err != nil {
315
+
errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListCommits: %w", err))
337
-
errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoLog: %w", err))
338
316
}
339
317
})
340
318
···
376
354
Readme: readmeContent,
377
355
ReadmeFileName: readmeFileName,
378
356
Commits: logResp.Commits,
357
+
Description: "",
379
-
Description: logResp.Description,
380
358
Files: files,
381
359
Branches: branchesResp.Branches,
382
360
Tags: tagsResp.Tags,
+10
-18
appview/repo/log.go
+10
-18
appview/repo/log.go
···
40
40
ref := chi.URLParam(r, "ref")
41
41
ref, _ = url.PathUnescape(ref)
42
42
43
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
43
-
scheme := "http"
44
-
if !rp.config.Core.Dev {
45
-
scheme = "https"
46
-
}
47
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
48
-
xrpcc := &indigoxrpc.Client{
49
-
Host: host,
50
-
}
51
44
52
45
limit := int64(60)
53
46
cursor := ""
···
57
50
cursor = strconv.Itoa(offset)
58
51
}
59
52
53
+
xrpcBytes, err := tangled.GitTempListCommits(r.Context(), xrpcc, cursor, limit, ref, f.RepoAt().String())
54
+
if err != nil {
55
+
l.Error("failed to call XRPC repo.log", "err", err)
60
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
61
-
xrpcBytes, err := tangled.RepoLog(r.Context(), xrpcc, cursor, limit, "", ref, repo)
62
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
63
-
l.Error("failed to call XRPC repo.log", "err", xrpcerr)
64
56
rp.pages.Error503(w)
65
57
return
66
58
}
···
72
64
return
73
65
}
74
66
67
+
tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String())
68
+
if err != nil {
69
+
l.Error("failed to call XRPC repo.tags", "err", err)
75
-
tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo)
76
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
77
-
l.Error("failed to call XRPC repo.tags", "err", xrpcerr)
78
70
rp.pages.Error503(w)
79
71
return
80
72
}
···
93
85
}
94
86
}
95
87
88
+
branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
89
+
if err != nil {
90
+
l.Error("failed to call XRPC repo.branches", "err", err)
96
-
branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
97
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
98
-
l.Error("failed to call XRPC repo.branches", "err", xrpcerr)
99
91
rp.pages.Error503(w)
100
92
return
101
93
}
appview/repo/opengraph.go
appview/repo/opengraph.go
This file has not been changed.
+2
-2
appview/repo/repo.go
+2
-2
appview/repo/repo.go
···
33
33
"tangled.org/core/xrpc/serviceauth"
34
34
35
35
comatproto "github.com/bluesky-social/indigo/api/atproto"
36
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
36
+
"github.com/bluesky-social/indigo/atproto/atclient"
37
37
"github.com/bluesky-social/indigo/atproto/syntax"
38
38
lexutil "github.com/bluesky-social/indigo/lex/util"
39
39
securejoin "github.com/cyphar/filepath-securejoin"
···
1207
1207
// this is used to rollback changes made to the PDS
1208
1208
//
1209
1209
// it is a no-op if the provided ATURI is empty
1210
-
func rollbackRecord(ctx context.Context, aturi string, client *atpclient.APIClient) error {
1210
+
func rollbackRecord(ctx context.Context, aturi string, client *atclient.APIClient) error {
1211
1211
if aturi == "" {
1212
1212
return nil
1213
1213
}
+2
-10
appview/repo/settings.go
+2
-10
appview/repo/settings.go
···
386
386
f, err := rp.repoResolver.Resolve(r)
387
387
user := rp.oauth.GetMultiAccountUser(r)
388
388
389
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
389
-
scheme := "http"
390
-
if !rp.config.Core.Dev {
391
-
scheme = "https"
392
-
}
393
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
394
-
xrpcc := &indigoxrpc.Client{
395
-
Host: host,
396
-
}
397
390
391
+
xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String())
398
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
399
-
xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo)
400
392
var result types.RepoBranchesResponse
401
393
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
402
394
l.Error("failed to call XRPC repo.branches", "err", xrpcerr)
+3
-10
appview/repo/tree.go
+3
-10
appview/repo/tree.go
···
33
33
treePath := chi.URLParam(r, "*")
34
34
treePath, _ = url.PathUnescape(treePath)
35
35
treePath = strings.TrimSuffix(treePath, "/")
36
+
37
+
xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url}
38
+
xrpcResp, err := tangled.GitTempGetTree(r.Context(), xrpcc, treePath, ref, f.RepoAt().String())
36
-
scheme := "http"
37
-
if !rp.config.Core.Dev {
38
-
scheme = "https"
39
-
}
40
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
41
-
xrpcc := &indigoxrpc.Client{
42
-
Host: host,
43
-
}
44
-
repo := fmt.Sprintf("%s/%s", f.Did, f.Name)
45
-
xrpcResp, err := tangled.RepoTree(r.Context(), xrpcc, treePath, ref, repo)
46
39
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
47
40
l.Error("failed to call XRPC repo.tree", "err", xrpcerr)
48
41
rp.pages.Error503(w)
+2
-2
appview/settings/settings.go
+2
-2
appview/settings/settings.go
···
28
28
"tangled.org/core/tid"
29
29
30
30
comatproto "github.com/bluesky-social/indigo/api/atproto"
31
+
"github.com/bluesky-social/indigo/atproto/atclient"
31
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
32
32
"github.com/bluesky-social/indigo/atproto/syntax"
33
33
lexutil "github.com/bluesky-social/indigo/lex/util"
34
34
"github.com/gliderlabs/ssh"
···
816
816
817
817
log.Printf("failed to update handle: %s", err)
818
818
msg := err.Error()
819
+
var apiErr *atclient.APIError
819
-
var apiErr *atpclient.APIError
820
820
if errors.As(err, &apiErr) && apiErr.Message != "" {
821
821
msg = apiErr.Message
822
822
}
+2
-2
appview/state/state.go
+2
-2
appview/state/state.go
···
38
38
"tangled.org/core/tid"
39
39
40
40
comatproto "github.com/bluesky-social/indigo/api/atproto"
41
+
"github.com/bluesky-social/indigo/atproto/atclient"
41
-
atpclient "github.com/bluesky-social/indigo/atproto/client"
42
42
"github.com/bluesky-social/indigo/atproto/syntax"
43
43
lexutil "github.com/bluesky-social/indigo/lex/util"
44
44
"github.com/bluesky-social/indigo/xrpc"
···
588
588
// this is used to rollback changes made to the PDS
589
589
//
590
590
// it is a no-op if the provided ATURI is empty
591
+
func rollbackRecord(ctx context.Context, aturi string, client *atclient.APIClient) error {
591
-
func rollbackRecord(ctx context.Context, aturi string, client *atpclient.APIClient) error {
592
592
if aturi == "" {
593
593
return nil
594
594
}
avatar/src/index.js
avatar/src/index.js
This file has not been changed.
+5
-7
blog/blog.go
+5
-7
blog/blog.go
···
76
76
77
77
rctx := &markup.RenderContext{
78
78
RendererType: markup.RendererTypeDefault,
79
-
Sanitizer: markup.NewSanitizer(),
80
79
}
81
80
var posts []Post
82
81
for _, entry := range entries {
···
100
99
}
101
100
102
101
htmlStr := rctx.RenderMarkdownWith(string(rest), markup.NewMarkdownWith("", textension.Dashes))
103
-
sanitized := rctx.SanitizeDefault(htmlStr)
104
102
105
103
posts = append(posts, Post{
106
104
Meta: meta,
105
+
Body: template.HTML(htmlStr),
107
-
Body: template.HTML(sanitized),
108
106
})
109
107
}
110
108
···
126
124
for _, p := range posts {
127
125
postURL := strings.TrimRight(baseURL, "/") + "/" + p.Meta.Slug
128
126
127
+
var authorName strings.Builder
129
-
var authorName string
130
128
for i, a := range p.Meta.Authors {
131
129
if i > 0 {
130
+
authorName.WriteString(" & ")
132
-
authorName += " & "
133
131
}
132
+
authorName.WriteString(a.Name)
134
-
authorName += a.Name
135
133
}
136
134
137
135
feed.Items = append(feed.Items, &feeds.Item{
138
136
Title: p.Meta.Title,
139
137
Link: &feeds.Link{Href: postURL},
140
138
Description: p.Meta.Subtitle,
139
+
Author: &feeds.Author{Name: authorName.String()},
141
-
Author: &feeds.Author{Name: authorName},
142
140
Created: p.ParsedDate(),
143
141
})
144
142
}
+40
-13
blog/templates/index.html
+40
-13
blog/templates/index.html
···
10
10
11
11
{{ define "topbarLayout" }}
12
12
<header class="max-w-screen-xl mx-auto w-full" style="z-index: 20;">
13
+
<nav class="mx-auto space-x-4 px-6 py-2">
14
+
<div class="flex justify-between p-0 items-center">
15
+
<div id="left-items">
16
+
<a href="/" hx-boost="true" class="text-2xl no-underline hover:no-underline flex items-center gap-2">
17
+
{{ template "fragments/logotypeSmall" }}
18
+
</a>
19
+
</div>
20
+
21
+
<div id="right-items" class="flex items-center gap-4">
22
+
<a href="https://tangled.org/login">login</a>
23
+
<span class="text-gray-500 dark:text-gray-400">or</span>
24
+
<a href="https://tangled.org/signup" class="btn-create py-0 hover:no-underline hover:text-white flex items-center gap-2">
25
+
join now {{ i "arrow-right" "size-4" }}
26
+
</a>
27
+
</div>
28
+
</div>
29
+
</nav>
13
-
{{ template "layouts/fragments/topbar" . }}
14
30
</header>
15
31
{{ end }}
16
32
···
26
42
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-6 mb-14">
27
43
{{ range .Featured }}
28
44
<a href="/{{ .Meta.Slug }}" class="no-underline hover:no-underline group flex flex-col bg-white dark:bg-gray-800 rounded border border-gray-200 dark:border-gray-700 overflow-hidden hover:bg-gray-100/25 hover:dark:bg-gray-700/25 transition-colors">
45
+
<div class="overflow-hidden bg-gray-100 dark:bg-gray-700 md:h-48">
29
-
<div class="aspect-[16/9] overflow-hidden bg-gray-100 dark:bg-gray-700">
30
46
<img src="{{ .Meta.Image }}" alt="{{ .Meta.Title }}" class="w-full h-full object-cover group-hover:scale-[1.02] transition-transform duration-300" />
31
47
</div>
32
48
<div class="flex flex-col flex-1 px-5 py-4">
33
49
<div class="text-xs text-gray-400 dark:text-gray-500 mb-2">
50
+
{{ $date := .ParsedDate }}{{ $date | shortTimeFmt}}
34
-
{{ $date := .ParsedDate }}{{ $date.Format "Jan 2, 2006" }}
35
51
{{ if .Meta.Draft }}<span class="text-red-500">[draft]</span>{{ end }}
36
52
</div>
53
+
<h2 class="font-bold text-gray-900 dark:text-white text-base leading-snug mb-1">{{ .Meta.Title }}</h2>
37
-
<h2 class="font-bold text-gray-900 dark:text-white text-base leading-snug mb-1 group-hover:underline">{{ .Meta.Title }}</h2>
38
54
<p class="text-sm text-gray-500 dark:text-gray-400 line-clamp-2 flex-1">{{ .Meta.Subtitle }}</p>
55
+
<div class="flex items-center mt-4 gap-2">
56
+
{{ $hasAvatar := false }}{{ range .Meta.Authors }}{{ if tinyAvatar .Handle }}{{ $hasAvatar = true }}{{ end }}{{ end }}
57
+
{{ if $hasAvatar }}
39
-
<div class="flex items-center mt-4">
40
58
<div class="inline-flex items-center -space-x-2">
41
59
{{ range .Meta.Authors }}
60
+
{{ $av := tinyAvatar .Handle }}{{ if $av }}<img src="{{ $av }}" class="size-6 rounded-full border border-gray-300 dark:border-gray-700" alt="{{ .Name }}" title="{{ .Name }}" />{{ end }}
61
+
{{ end }}
62
+
</div>
63
+
{{ end }}
64
+
<div class="text-xs">
65
+
{{ $last := sub (len .Meta.Authors) 1 }}
66
+
{{ range $i, $n := .Meta.Authors }}
67
+
{{ $n.Handle }}{{ if ne $i $last }}, {{ end }}
42
-
<img src="{{ tinyAvatar .Handle }}" class="size-6 rounded-full border border-gray-300 dark:border-gray-700" alt="{{ .Name }}" title="{{ .Name }}" />
43
68
{{ end }}
44
69
</div>
45
70
</div>
···
53
78
{{ range .Posts }}
54
79
<a href="/{{ .Meta.Slug }}" class="no-underline hover:no-underline group flex items-center justify-between gap-4 px-6 py-3 hover:bg-gray-100/25 hover:dark:bg-gray-700/25 transition-colors">
55
80
<div class="flex items-center gap-3 min-w-0">
81
+
<span class="font-medium text-gray-900 dark:text-white truncate">
82
+
{{ .Meta.Title }}
83
+
{{ if .Meta.Draft }}<span class="text-red-500 text-xs font-normal ml-1">[draft]</span>{{ end }}
84
+
</span>
85
+
</div>
86
+
<div class="flex items-center gap-2">
56
87
<div class="inline-flex items-center -space-x-2 shrink-0">
57
88
{{ range .Meta.Authors }}
58
89
<img src="{{ tinyAvatar .Handle }}" class="size-5 rounded-full border border-gray-300 dark:border-gray-700" alt="{{ .Name }}" title="{{ .Name }}" />
59
90
{{ end }}
60
91
</div>
92
+
<div class="text-sm text-gray-400 dark:text-gray-500 shrink-0">
93
+
{{ $date := .ParsedDate }}{{ $date | shortTimeFmt }}
94
+
</div>
61
-
<span class="font-medium text-gray-900 dark:text-white group-hover:underline truncate">
62
-
{{ .Meta.Title }}
63
-
{{ if .Meta.Draft }}<span class="text-red-500 text-xs font-normal ml-1">[draft]</span>{{ end }}
64
-
</span>
65
-
</div>
66
-
<div class="text-sm text-gray-400 dark:text-gray-500 shrink-0">
67
-
{{ $date := .ParsedDate }}{{ $date.Format "Jan 02, 2006" }}
68
95
</div>
69
96
</a>
70
97
{{ end }}
+5
-2
blog/templates/post.html
+5
-2
blog/templates/post.html
···
35
35
{{ $authors := .Post.Meta.Authors }}
36
36
<p class="mb-1 text-sm text-gray-600 dark:text-gray-400">
37
37
{{ $date := .Post.ParsedDate }}
38
+
{{ $date | shortTimeFmt }}
38
-
{{ $date.Format "02 Jan, 2006" }}
39
39
</p>
40
40
41
41
<h1 class="mb-0 text-2xl font-bold dark:text-white">
···
45
45
<p class="italic mt-1 mb-3 text-lg text-gray-600 dark:text-gray-400">{{ .Post.Meta.Subtitle }}</p>
46
46
47
47
<div class="flex items-center gap-3 not-prose">
48
+
{{ $hasAvatar := false }}{{ range $authors }}{{ if tinyAvatar .Handle }}{{ $hasAvatar = true }}{{ end }}{{ end }}
49
+
{{ if $hasAvatar }}
48
50
<div class="inline-flex items-center -space-x-2">
49
51
{{ range $authors }}
52
+
{{ $av := tinyAvatar .Handle }}{{ if $av }}<img src="{{ $av }}" class="size-7 rounded-full border border-gray-300 dark:border-gray-700" alt="{{ .Handle }}" title="{{ .Handle }}" />{{ end }}
50
-
<img src="{{ tinyAvatar .Handle }}" class="size-7 rounded-full border border-gray-300 dark:border-gray-700" alt="{{ .Handle }}" title="{{ .Handle }}" />
51
53
{{ end }}
52
54
</div>
55
+
{{ end }}
53
56
<div class="flex items-center gap-1 text-sm text-gray-700 dark:text-gray-300">
54
57
{{ range $i, $a := $authors }}
55
58
{{ if gt $i 0 }}<span class="text-gray-400">&</span>{{ end }}
+1
-1
blog/templates/text.html
+1
-1
blog/templates/text.html
+30
-3
cmd/blog/main.go
+30
-3
cmd/blog/main.go
···
4
4
"context"
5
5
"fmt"
6
6
"io"
7
+
"io/fs"
7
8
"log/slog"
8
9
"net/http"
9
10
"os"
···
84
85
return fmt.Errorf("rendering index: %w", err)
85
86
}
86
87
87
-
// posts — each at build/<slug>/index.html directly (no /blog/ prefix)
88
88
for _, post := range posts {
89
-
post := post
90
89
postDir := filepath.Join(outDir, post.Meta.Slug)
91
90
if err := os.MkdirAll(postDir, 0755); err != nil {
92
91
return err
···
98
97
}
99
98
}
100
99
100
+
// atom feed
101
-
// atom feed — at build/feed.xml
102
101
baseURL := "https://blog.tangled.org"
103
102
atom, err := blog.AtomFeed(posts, baseURL)
104
103
if err != nil {
···
108
107
return fmt.Errorf("writing feed: %w", err)
109
108
}
110
109
110
+
// copy embedded static assets into build/static/ so Cloudflare Pages
111
+
// can serve them from the same origin as the built HTML
112
+
staticSrc, err := fs.Sub(pages.Files, "static")
113
+
if err != nil {
114
+
return fmt.Errorf("accessing embedded static dir: %w", err)
115
+
}
116
+
if err := copyFS(staticSrc, filepath.Join(outDir, "static")); err != nil {
117
+
return fmt.Errorf("copying static assets: %w", err)
118
+
}
119
+
111
120
logger.Info("build complete", "dir", outDir)
112
121
return nil
122
+
}
123
+
124
+
// copyFS copies all files from src into destDir, preserving directory structure.
125
+
func copyFS(src fs.FS, destDir string) error {
126
+
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
127
+
if err != nil {
128
+
return err
129
+
}
130
+
dest := filepath.Join(destDir, path)
131
+
if d.IsDir() {
132
+
return os.MkdirAll(dest, 0755)
133
+
}
134
+
data, err := fs.ReadFile(src, path)
135
+
if err != nil {
136
+
return err
137
+
}
138
+
return os.WriteFile(dest, data, 0644)
139
+
})
113
140
}
114
141
115
142
func runServe(ctx context.Context, logger *slog.Logger, addr string) error {
+58
cmd/knotmirror/main.go
+58
cmd/knotmirror/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
"os"
7
+
"os/signal"
8
+
"syscall"
9
+
10
+
"github.com/carlmjohnson/versioninfo"
11
+
"github.com/urfave/cli/v3"
12
+
"tangled.org/core/knotmirror"
13
+
"tangled.org/core/knotmirror/config"
14
+
"tangled.org/core/log"
15
+
)
16
+
17
+
func main() {
18
+
if err := run(os.Args); err != nil {
19
+
slog.Error("error running knotmirror", "err", err)
20
+
os.Exit(-1)
21
+
}
22
+
}
23
+
24
+
func run(args []string) error {
25
+
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
26
+
defer cancel()
27
+
28
+
logger := log.New("knotmirror")
29
+
slog.SetDefault(logger)
30
+
ctx = log.IntoContext(ctx, logger)
31
+
32
+
app := cli.Command{
33
+
Name: "knotmirror",
34
+
Usage: "knot mirroring service",
35
+
Version: versioninfo.Short(),
36
+
}
37
+
app.Flags = []cli.Flag{}
38
+
app.Commands = []*cli.Command{
39
+
{
40
+
Name: "serve",
41
+
Usage: "run the knotmirror daemon",
42
+
Action: runKnotMirror,
43
+
Flags: []cli.Flag{},
44
+
},
45
+
}
46
+
return app.Run(ctx, args)
47
+
}
48
+
49
+
func runKnotMirror(ctx context.Context, cmd *cli.Command) error {
50
+
logger := log.FromContext(ctx)
51
+
cfg, err := config.Load(ctx)
52
+
if err != nil {
53
+
return err
54
+
}
55
+
56
+
logger.Debug("config loaded:", "config", cfg)
57
+
return knotmirror.Run(ctx, cfg)
58
+
}
+15
-1
flake.nix
+15
-1
flake.nix
···
106
106
knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {};
107
107
knot = self.callPackage ./nix/pkgs/knot.nix {};
108
108
dolly = self.callPackage ./nix/pkgs/dolly.nix {};
109
+
tap = self.callPackage ./nix/pkgs/tap.nix {};
110
+
knotmirror = self.callPackage ./nix/pkgs/knot-mirror.nix {};
109
111
});
110
112
in {
111
113
overlays.default = final: prev: {
114
+
inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly tap knotmirror;
112
-
inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly;
113
115
};
114
116
115
117
packages = forAllSystems (system: let
···
130
132
sqlite-lib
131
133
docs
132
134
dolly
135
+
tap
133
136
;
134
137
135
138
pkgsStatic-appview = staticPackages.appview;
···
204
207
pkgs.coreutils # for those of us who are on systems that use busybox (alpine)
205
208
packages'.lexgen
206
209
packages'.treefmt-wrapper
210
+
packages'.tap
207
211
];
208
212
shellHook = ''
209
213
mkdir -p appview/pages/static
···
348
352
imports = [./nix/modules/appview.nix];
349
353
350
354
services.tangled.appview.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.appview;
355
+
};
356
+
nixosModules.knotmirror = {
357
+
lib,
358
+
pkgs,
359
+
...
360
+
}: {
361
+
imports = [./nix/modules/knotmirror.nix];
362
+
363
+
services.tangled.knotmirror.tap-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.tap;
364
+
services.tangled.knotmirror.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knotmirror;
351
365
};
352
366
nixosModules.knot = {
353
367
lib,
+18
-10
go.mod
+18
-10
go.mod
···
12
12
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
13
13
github.com/blevesearch/bleve/v2 v2.5.3
14
14
github.com/bluekeyes/go-gitdiff v0.8.1
15
+
github.com/bluesky-social/indigo v0.0.0-20260220055544-bf41e2ee75ab
16
+
github.com/bluesky-social/jetstream v0.0.0-20260226214936-e0274250f654
15
-
github.com/bluesky-social/indigo v0.0.0-20251003000214-3259b215110e
16
-
github.com/bluesky-social/jetstream v0.0.0-20241210005130-ea96859b93d1
17
17
github.com/bmatcuk/doublestar/v4 v4.9.1
18
18
github.com/carlmjohnson/versioninfo v0.22.5
19
19
github.com/casbin/casbin/v2 v2.103.0
···
35
35
github.com/hiddeco/sshsig v0.2.0
36
36
github.com/hpcloud/tail v1.0.0
37
37
github.com/ipfs/go-cid v0.5.0
38
+
github.com/jackc/pgx/v5 v5.8.0
38
39
github.com/mattn/go-sqlite3 v1.14.24
39
40
github.com/microcosm-cc/bluemonday v1.0.27
40
41
github.com/openbao/openbao/api/v2 v2.3.0
41
42
github.com/posthog/posthog-go v1.5.5
43
+
github.com/prometheus/client_golang v1.23.2
42
44
github.com/redis/go-redis/v9 v9.7.3
43
45
github.com/resend/resend-go/v2 v2.15.0
44
46
github.com/sethvargo/go-envconfig v1.1.0
45
47
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c
46
48
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
49
+
github.com/stretchr/testify v1.11.1
50
+
github.com/urfave/cli/v3 v3.4.1
47
-
github.com/stretchr/testify v1.10.0
48
-
github.com/urfave/cli/v3 v3.3.3
49
51
github.com/whyrusleeping/cbor-gen v0.3.1
50
52
github.com/yuin/goldmark v1.7.13
51
53
github.com/yuin/goldmark-emoji v1.0.6
52
54
github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc
53
55
gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab
54
56
go.abhg.dev/goldmark/mermaid v0.6.0
57
+
golang.org/x/crypto v0.41.0
55
-
golang.org/x/crypto v0.40.0
56
58
golang.org/x/image v0.31.0
59
+
golang.org/x/net v0.43.0
57
-
golang.org/x/net v0.42.0
58
60
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
59
61
gopkg.in/yaml.v3 v3.0.1
60
62
)
···
116
118
github.com/dlclark/regexp2 v1.11.5 // indirect
117
119
github.com/docker/go-connections v0.5.0 // indirect
118
120
github.com/docker/go-units v0.5.0 // indirect
121
+
github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect
119
122
github.com/emirpasic/gods v1.18.1 // indirect
120
123
github.com/felixge/httpsnoop v1.0.4 // indirect
121
124
github.com/fsnotify/fsnotify v1.6.0 // indirect
···
160
163
github.com/ipfs/go-log v1.0.5 // indirect
161
164
github.com/ipfs/go-log/v2 v2.6.0 // indirect
162
165
github.com/ipfs/go-metrics-interface v0.3.0 // indirect
166
+
github.com/jackc/pgpassfile v1.0.0 // indirect
167
+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
168
+
github.com/jackc/puddle/v2 v2.2.2 // indirect
163
169
github.com/json-iterator/go v1.1.12 // indirect
164
170
github.com/kevinburke/ssh_config v1.2.0 // indirect
165
171
github.com/klauspost/compress v1.18.0 // indirect
···
192
198
github.com/pkg/errors v0.9.1 // indirect
193
199
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
194
200
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
195
-
github.com/prometheus/client_golang v1.22.0 // indirect
196
201
github.com/prometheus/client_model v0.6.2 // indirect
202
+
github.com/prometheus/common v0.66.1 // indirect
197
-
github.com/prometheus/common v0.64.0 // indirect
198
203
github.com/prometheus/procfs v0.16.1 // indirect
199
204
github.com/rivo/uniseg v0.4.7 // indirect
200
205
github.com/ryanuber/go-glob v1.0.0 // indirect
···
221
226
go.uber.org/atomic v1.11.0 // indirect
222
227
go.uber.org/multierr v1.11.0 // indirect
223
228
go.uber.org/zap v1.27.0 // indirect
229
+
go.yaml.in/yaml/v2 v2.4.2 // indirect
224
230
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
225
231
golang.org/x/sync v0.17.0 // indirect
232
+
golang.org/x/sys v0.35.0 // indirect
226
-
golang.org/x/sys v0.34.0 // indirect
227
233
golang.org/x/text v0.29.0 // indirect
228
234
golang.org/x/time v0.12.0 // indirect
229
235
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
230
236
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
231
237
google.golang.org/grpc v1.73.0 // indirect
238
+
google.golang.org/protobuf v1.36.8 // indirect
232
-
google.golang.org/protobuf v1.36.6 // indirect
233
239
gopkg.in/fsnotify.v1 v1.4.7 // indirect
234
240
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
235
241
gopkg.in/warnings.v0 v0.1.2 // indirect
···
245
251
replace github.com/bluekeyes/go-gitdiff => tangled.sh/oppi.li/go-gitdiff v0.8.2
246
252
247
253
replace github.com/alecthomas/chroma/v2 => github.com/oppiliappan/chroma/v2 v2.24.2
254
+
255
+
replace github.com/bluesky-social/indigo => github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2
248
256
249
257
// from bluesky-social/indigo
250
258
replace github.com/gocql/gocql => github.com/scylladb/gocql v1.14.4
+36
-22
go.sum
+36
-22
go.sum
···
94
94
github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
95
95
github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww=
96
96
github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs=
97
+
github.com/bluesky-social/jetstream v0.0.0-20260226214936-e0274250f654 h1:OK76FcHhZp8ohjRB0OMWgti0oYAWFlt3KDQcIkH1pfI=
98
+
github.com/bluesky-social/jetstream v0.0.0-20260226214936-e0274250f654/go.mod h1:vt8kVRKtvrBspt9G38wDD8+BotjIMO8u8IYoVnyE4zY=
97
-
github.com/bluesky-social/indigo v0.0.0-20251003000214-3259b215110e h1:IutKPwmbU0LrYqw03EuwJtMdAe67rDTrL1U8S8dicRU=
98
-
github.com/bluesky-social/indigo v0.0.0-20251003000214-3259b215110e/go.mod h1:n6QE1NDPFoi7PRbMUZmc2y7FibCqiVU4ePpsvhHUBR8=
99
-
github.com/bluesky-social/jetstream v0.0.0-20241210005130-ea96859b93d1 h1:CFvRtYNSnWRAi/98M3O466t9dYuwtesNbu6FVPymRrA=
100
-
github.com/bluesky-social/jetstream v0.0.0-20241210005130-ea96859b93d1/go.mod h1:WiYEeyJSdUwqoaZ71KJSpTblemUCpwJfh5oVXplK6T4=
101
99
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
102
100
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
103
101
github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
104
102
github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
103
+
github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2 h1:63+EsT7kltod8g1eA0eNuvq1q9ANJWRdxlLeJjJDVYY=
104
+
github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2/go.mod h1:VG/LeqLGNI3Ew7lsYixajnZGFfWPv144qbUddh+Oyag=
105
105
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
106
106
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
107
107
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
···
178
178
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
179
179
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
180
180
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
181
+
github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg=
182
+
github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw=
181
183
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
182
184
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
183
185
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
···
348
350
github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8=
349
351
github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU=
350
352
github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY=
353
+
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
354
+
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
355
+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
356
+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
357
+
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
358
+
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
359
+
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
360
+
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
351
361
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
352
362
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
353
363
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
···
369
379
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
370
380
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
371
381
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
382
+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
383
+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
372
384
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
373
385
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
374
386
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
···
470
482
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
471
483
github.com/posthog/posthog-go v1.5.5 h1:2o3j7IrHbTIfxRtj4MPaXKeimuTYg49onNzNBZbwksM=
472
484
github.com/posthog/posthog-go v1.5.5/go.mod h1:3RqUmSnPuwmeVj/GYrS75wNGqcAKdpODiwc83xZWgdE=
485
+
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
486
+
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
473
-
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
474
-
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
475
487
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
476
488
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
489
+
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
490
+
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
477
-
github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4=
478
-
github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
479
491
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
480
492
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
481
493
github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA=
···
521
533
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
522
534
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
523
535
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
536
+
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
537
+
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
524
-
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
525
-
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
526
538
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
527
539
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
528
540
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
···
535
547
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
536
548
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
537
549
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
550
+
github.com/urfave/cli/v3 v3.4.1 h1:1M9UOCy5bLmGnuu1yn3t3CB4rG79Rtoxuv1sPhnm6qM=
551
+
github.com/urfave/cli/v3 v3.4.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
538
-
github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I=
539
-
github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
540
552
github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI=
541
553
github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q=
542
554
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
···
606
618
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
607
619
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
608
620
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
621
+
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
622
+
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
609
623
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
610
624
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
611
625
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
···
613
627
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
614
628
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
615
629
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
630
+
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
631
+
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
616
-
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
617
-
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
618
632
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
619
633
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
620
634
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
···
649
663
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
650
664
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
651
665
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
666
+
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
667
+
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
652
-
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
653
-
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
654
668
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
655
669
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
656
670
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
690
704
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
691
705
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
692
706
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
707
+
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
708
+
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
693
-
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
694
-
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
695
709
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
696
710
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
697
711
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
···
701
715
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
702
716
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
703
717
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
718
+
golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
719
+
golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
704
-
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
705
-
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
706
720
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
707
721
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
708
722
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
···
755
769
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
756
770
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
757
771
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
772
+
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
773
+
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
758
-
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
759
-
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
760
774
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
761
775
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
762
776
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+3
-3
idresolver/resolver.go
+3
-3
idresolver/resolver.go
···
60
60
base := BaseDirectory(plcUrl)
61
61
cached := identity.NewCacheDirectory(base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5)
62
62
return &Resolver{
63
+
directory: cached,
63
-
directory: &cached,
64
64
}
65
65
}
66
66
···
80
80
return nil, err
81
81
}
82
82
83
+
return r.directory.Lookup(ctx, id)
83
-
return r.directory.Lookup(ctx, *id)
84
84
}
85
85
86
86
func (r *Resolver) ResolveIdents(ctx context.Context, idents []string) []*identity.Identity {
···
117
117
return err
118
118
}
119
119
120
+
return r.directory.Purge(ctx, id)
120
-
return r.directory.Purge(ctx, *id)
121
121
}
122
122
123
123
func (r *Resolver) Directory() identity.Directory {
+182
knotmirror/adminpage.go
+182
knotmirror/adminpage.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"database/sql"
5
+
"embed"
6
+
"fmt"
7
+
"html"
8
+
"html/template"
9
+
"log/slog"
10
+
"net/http"
11
+
"strconv"
12
+
"time"
13
+
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/go-chi/chi/v5"
16
+
"tangled.org/core/appview/pagination"
17
+
"tangled.org/core/knotmirror/db"
18
+
"tangled.org/core/knotmirror/models"
19
+
)
20
+
21
+
//go:embed templates/*.html
22
+
var templateFS embed.FS
23
+
24
+
const repoPageSize = 20
25
+
26
+
type AdminServer struct {
27
+
db *sql.DB
28
+
resyncer *Resyncer
29
+
logger *slog.Logger
30
+
}
31
+
32
+
func NewAdminServer(l *slog.Logger, database *sql.DB, resyncer *Resyncer) *AdminServer {
33
+
return &AdminServer{
34
+
db: database,
35
+
resyncer: resyncer,
36
+
logger: l,
37
+
}
38
+
}
39
+
40
+
func (s *AdminServer) Router() http.Handler {
41
+
r := chi.NewRouter()
42
+
r.Get("/repos", s.handleRepos())
43
+
r.Get("/hosts", s.handleHosts())
44
+
45
+
r.Post("/api/triggerRepoResync", s.handleRepoResyncTrigger())
46
+
r.Post("/api/cancelRepoResync", s.handleRepoResyncCancel())
47
+
return r
48
+
}
49
+
50
+
func funcmap() template.FuncMap {
51
+
return template.FuncMap{
52
+
"add": func(a, b int) int { return a + b },
53
+
"sub": func(a, b int) int { return a - b },
54
+
"readt": func(ts int64) string {
55
+
if ts <= 0 {
56
+
return "n/a"
57
+
}
58
+
return time.Unix(ts, 0).Format("2006-01-02 15:04")
59
+
},
60
+
"const": func() map[string]any {
61
+
return map[string]any{
62
+
"AllRepoStates": models.AllRepoStates,
63
+
"AllHostStatuses": models.AllHostStatuses,
64
+
}
65
+
},
66
+
}
67
+
}
68
+
69
+
func (s *AdminServer) handleRepos() http.HandlerFunc {
70
+
tpl := template.Must(template.New("").Funcs(funcmap()).ParseFS(templateFS, "templates/base.html", "templates/repos.html"))
71
+
return func(w http.ResponseWriter, r *http.Request) {
72
+
pageNum, _ := strconv.Atoi(r.URL.Query().Get("page"))
73
+
if pageNum < 1 {
74
+
pageNum = 1
75
+
}
76
+
page := pagination.Page{
77
+
Offset: (pageNum - 1) * repoPageSize,
78
+
Limit: repoPageSize,
79
+
}
80
+
81
+
var (
82
+
did = r.URL.Query().Get("did")
83
+
knot = r.URL.Query().Get("knot")
84
+
state = r.URL.Query().Get("state")
85
+
)
86
+
87
+
repos, err := db.ListRepos(r.Context(), s.db, page, did, knot, state)
88
+
if err != nil {
89
+
http.Error(w, err.Error(), http.StatusInternalServerError)
90
+
}
91
+
counts, err := db.GetRepoCountsByState(r.Context(), s.db)
92
+
if err != nil {
93
+
http.Error(w, err.Error(), http.StatusInternalServerError)
94
+
}
95
+
err = tpl.ExecuteTemplate(w, "base", map[string]any{
96
+
"Repos": repos,
97
+
"RepoCounts": counts,
98
+
"Page": pageNum,
99
+
"FilterByDid": did,
100
+
"FilterByKnot": knot,
101
+
"FilterByState": models.RepoState(state),
102
+
})
103
+
if err != nil {
104
+
slog.Error("failed to render", "err", err)
105
+
}
106
+
}
107
+
}
108
+
109
+
func (s *AdminServer) handleHosts() http.HandlerFunc {
110
+
tpl := template.Must(template.New("").Funcs(funcmap()).ParseFS(templateFS, "templates/base.html", "templates/hosts.html"))
111
+
return func(w http.ResponseWriter, r *http.Request) {
112
+
var status = models.HostStatus(r.URL.Query().Get("status"))
113
+
if status == "" {
114
+
status = models.HostStatusActive
115
+
}
116
+
117
+
hosts, err := db.ListHosts(r.Context(), s.db, status)
118
+
if err != nil {
119
+
http.Error(w, err.Error(), http.StatusInternalServerError)
120
+
}
121
+
err = tpl.ExecuteTemplate(w, "base", map[string]any{
122
+
"Hosts": hosts,
123
+
"FilterByStatus": models.HostStatus(status),
124
+
})
125
+
if err != nil {
126
+
slog.Error("failed to render", "err", err)
127
+
}
128
+
}
129
+
}
130
+
131
+
func (s *AdminServer) handleRepoResyncTrigger() http.HandlerFunc {
132
+
return func(w http.ResponseWriter, r *http.Request) {
133
+
var repoQuery = r.FormValue("repo")
134
+
135
+
repo, err := syntax.ParseATURI(repoQuery)
136
+
if err != nil || repo.RecordKey() == "" {
137
+
writeNotif(w, http.StatusBadRequest, fmt.Sprintf("repo parameter invalid: %s", repoQuery))
138
+
return
139
+
}
140
+
141
+
if err := s.resyncer.TriggerResyncJob(r.Context(), repo); err != nil {
142
+
s.logger.Error("failed to trigger resync job", "err", err)
143
+
writeNotif(w, http.StatusInternalServerError, fmt.Sprintf("repo parameter invalid: %s", repoQuery))
144
+
return
145
+
}
146
+
writeNotif(w, http.StatusOK, "success")
147
+
}
148
+
}
149
+
150
+
func (s *AdminServer) handleRepoResyncCancel() http.HandlerFunc {
151
+
return func(w http.ResponseWriter, r *http.Request) {
152
+
var repoQuery = r.FormValue("repo")
153
+
154
+
repo, err := syntax.ParseATURI(repoQuery)
155
+
if err != nil || repo.RecordKey() == "" {
156
+
writeNotif(w, http.StatusBadRequest, fmt.Sprintf("repo parameter invalid: %s", repoQuery))
157
+
return
158
+
}
159
+
160
+
s.resyncer.CancelResyncJob(repo)
161
+
writeNotif(w, http.StatusOK, "success")
162
+
}
163
+
}
164
+
165
+
func writeNotif(w http.ResponseWriter, status int, msg string) {
166
+
w.Header().Set("Content-Type", "text/html")
167
+
w.WriteHeader(status)
168
+
169
+
class := "info"
170
+
switch {
171
+
case status >= 500:
172
+
class = "error"
173
+
case status >= 400:
174
+
class = "warn"
175
+
}
176
+
177
+
fmt.Fprintf(w,
178
+
`<div hx-swap-oob="beforeend:#notifications"><div class="notif %s">%s</div></div>`,
179
+
class,
180
+
html.EscapeString(msg),
181
+
)
182
+
}
+45
knotmirror/config/config.go
+45
knotmirror/config/config.go
···
1
+
package config
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/sethvargo/go-envconfig"
8
+
)
9
+
10
+
type Config struct {
11
+
PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"`
12
+
TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"`
13
+
DbUrl string `env:"MIRROR_DB_URL, required"`
14
+
KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified
15
+
KnotSSRF bool `env:"MIRROR_KNOT_SSRF, default=false"`
16
+
GitRepoBasePath string `env:"MIRROR_GIT_BASEPATH, default=repos"`
17
+
GitRepoFetchTimeout time.Duration `env:"MIRROR_GIT_FETCH_TIMEOUT, default=600s"`
18
+
ResyncParallelism int `env:"MIRROR_RESYNC_PARALLELISM, default=5"`
19
+
Slurper SlurperConfig `env:",prefix=MIRROR_SLURPER_"`
20
+
UseSSL bool `env:"MIRROR_USE_SSL, default=false"`
21
+
Hostname string `env:"MIRROR_HOSTNAME, required"`
22
+
Listen string `env:"MIRROR_LISTEN, default=:7000"`
23
+
MetricsListen string `env:"MIRROR_METRICS_LISTEN, default=127.0.0.1:7100"`
24
+
AdminListen string `env:"MIRROR_ADMIN_LISTEN, default=127.0.0.1:7200"`
25
+
}
26
+
27
+
func (c *Config) BaseUrl() string {
28
+
if c.UseSSL {
29
+
return "https://" + c.Hostname
30
+
}
31
+
return "http://" + c.Hostname
32
+
}
33
+
34
+
type SlurperConfig struct {
35
+
PersistCursorPeriod time.Duration `env:"PERSIST_CURSOR_PERIOD, default=4s"`
36
+
ConcurrencyPerHost int `env:"CONCURRENCY, default=4"`
37
+
}
38
+
39
+
func Load(ctx context.Context) (*Config, error) {
40
+
var cfg Config
41
+
if err := envconfig.Process(ctx, &cfg); err != nil {
42
+
return nil, err
43
+
}
44
+
return &cfg, nil
45
+
}
+25
knotmirror/crawler.go
+25
knotmirror/crawler.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"log/slog"
7
+
8
+
"tangled.org/core/log"
9
+
)
10
+
11
+
type Crawler struct {
12
+
logger *slog.Logger
13
+
db *sql.DB
14
+
}
15
+
16
+
func NewCrawler(l *slog.Logger, db *sql.DB) *Crawler {
17
+
return &Crawler{
18
+
logger: log.SubLogger(l, "crawler"),
19
+
db: db,
20
+
}
21
+
}
22
+
23
+
func (c *Crawler) Start(ctx context.Context) {
24
+
// TODO: repository crawler
25
+
}
+100
knotmirror/db/db.go
+100
knotmirror/db/db.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"fmt"
7
+
"time"
8
+
9
+
_ "github.com/jackc/pgx/v5/stdlib"
10
+
)
11
+
12
+
func Make(ctx context.Context, dbUrl string, maxConns int) (*sql.DB, error) {
13
+
db, err := sql.Open("pgx", dbUrl)
14
+
if err != nil {
15
+
return nil, fmt.Errorf("opening db: %w", err)
16
+
}
17
+
18
+
db.SetMaxOpenConns(maxConns)
19
+
db.SetMaxIdleConns(maxConns)
20
+
db.SetConnMaxIdleTime(time.Hour)
21
+
22
+
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
23
+
defer cancel()
24
+
if err := db.PingContext(pingCtx); err != nil {
25
+
db.Close()
26
+
return nil, fmt.Errorf("ping db: %w", err)
27
+
}
28
+
29
+
conn, err := db.Conn(ctx)
30
+
if err != nil {
31
+
return nil, err
32
+
}
33
+
defer conn.Close()
34
+
35
+
_, err = conn.ExecContext(ctx, `
36
+
create table if not exists repos (
37
+
did text not null,
38
+
rkey text not null,
39
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.repo' || '/' || rkey) stored,
40
+
cid text,
41
+
42
+
-- record content
43
+
name text not null,
44
+
knot_domain text not null,
45
+
46
+
-- sync info
47
+
git_rev text not null,
48
+
repo_sha text not null,
49
+
state text not null default 'pending',
50
+
error_msg text,
51
+
retry_count integer not null default 0,
52
+
retry_after integer not null default 0,
53
+
db_created_at timestamptz not null default now(),
54
+
db_updated_at timestamptz not null default now(),
55
+
56
+
constraint repos_pkey primary key (did, rkey)
57
+
);
58
+
59
+
-- knot hosts
60
+
create table if not exists hosts (
61
+
hostname text not null,
62
+
no_ssl boolean not null default false,
63
+
status text not null default 'active',
64
+
last_seq bigint not null default -1,
65
+
db_created_at timestamptz not null default now(),
66
+
db_updated_at timestamptz not null default now(),
67
+
68
+
constraint hosts_pkey primary key (hostname)
69
+
);
70
+
71
+
create index if not exists idx_repos_aturi on repos (at_uri);
72
+
create index if not exists idx_repos_db_updated_at on repos (db_updated_at desc);
73
+
create index if not exists idx_hosts_db_updated_at on hosts (db_updated_at desc);
74
+
75
+
create or replace function set_updated_at()
76
+
returns trigger as $$
77
+
begin
78
+
new.db_updated_at = now();
79
+
return new;
80
+
end;
81
+
$$ language plpgsql;
82
+
83
+
drop trigger if exists repos_set_updated_at on repos;
84
+
create trigger repos_set_updated_at
85
+
before update on repos
86
+
for each row
87
+
execute function set_updated_at();
88
+
89
+
drop trigger if exists hosts_set_updated_at on hosts;
90
+
create trigger hosts_set_updated_at
91
+
before update on hosts
92
+
for each row
93
+
execute function set_updated_at();
94
+
`)
95
+
if err != nil {
96
+
return nil, fmt.Errorf("initializing db schema: %w", err)
97
+
}
98
+
99
+
return db, nil
100
+
}
+102
knotmirror/db/hosts.go
+102
knotmirror/db/hosts.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"errors"
7
+
"fmt"
8
+
"log"
9
+
10
+
"tangled.org/core/knotmirror/models"
11
+
)
12
+
13
+
func UpsertHost(ctx context.Context, e *sql.DB, host *models.Host) error {
14
+
if _, err := e.ExecContext(ctx,
15
+
`insert into hosts (hostname, no_ssl, status, last_seq)
16
+
values ($1, $2, $3, $4)
17
+
on conflict(hostname) do update set
18
+
no_ssl = excluded.no_ssl,
19
+
status = excluded.status,
20
+
last_seq = excluded.last_seq
21
+
`,
22
+
host.Hostname,
23
+
host.NoSSL,
24
+
host.Status,
25
+
host.LastSeq,
26
+
); err != nil {
27
+
return fmt.Errorf("upserting host: %w", err)
28
+
}
29
+
return nil
30
+
}
31
+
32
+
func GetHost(ctx context.Context, e *sql.DB, hostname string) (*models.Host, error) {
33
+
var host models.Host
34
+
if err := e.QueryRowContext(ctx,
35
+
`select hostname, no_ssl, status, last_seq
36
+
from hosts where hostname = $1`,
37
+
hostname,
38
+
).Scan(
39
+
&host.Hostname,
40
+
&host.NoSSL,
41
+
&host.Status,
42
+
&host.LastSeq,
43
+
); err != nil {
44
+
if errors.Is(err, sql.ErrNoRows) {
45
+
return nil, nil
46
+
}
47
+
return nil, err
48
+
}
49
+
return &host, nil
50
+
}
51
+
52
+
func StoreCursors(ctx context.Context, e *sql.DB, cursors []models.HostCursor) error {
53
+
tx, err := e.BeginTx(ctx, nil)
54
+
if err != nil {
55
+
return fmt.Errorf("starting transaction: %w", err)
56
+
}
57
+
defer tx.Rollback()
58
+
for _, cur := range cursors {
59
+
if cur.LastSeq <= 0 {
60
+
continue
61
+
}
62
+
if _, err := tx.ExecContext(ctx,
63
+
`update hosts set last_seq = $1 where hostname = $2`,
64
+
cur.LastSeq,
65
+
cur.Hostname,
66
+
); err != nil {
67
+
log.Println("failed to persist host cursor", "host", cur.Hostname, "lastSeq", cur.LastSeq, "err", err)
68
+
}
69
+
}
70
+
return tx.Commit()
71
+
}
72
+
73
+
func ListHosts(ctx context.Context, e *sql.DB, status models.HostStatus) ([]models.Host, error) {
74
+
rows, err := e.QueryContext(ctx,
75
+
`select hostname, no_ssl, status, last_seq
76
+
from hosts
77
+
where status = $1`,
78
+
status,
79
+
)
80
+
if err != nil {
81
+
return nil, fmt.Errorf("querying hosts: %w", err)
82
+
}
83
+
defer rows.Close()
84
+
85
+
var hosts []models.Host
86
+
for rows.Next() {
87
+
var host models.Host
88
+
if err := rows.Scan(
89
+
&host.Hostname,
90
+
&host.NoSSL,
91
+
&host.Status,
92
+
&host.LastSeq,
93
+
); err != nil {
94
+
return nil, fmt.Errorf("scanning row: %w", err)
95
+
}
96
+
hosts = append(hosts, host)
97
+
}
98
+
if err := rows.Err(); err != nil {
99
+
return nil, fmt.Errorf("scanning rows: %w ", err)
100
+
}
101
+
return hosts, nil
102
+
}
+275
knotmirror/db/repos.go
+275
knotmirror/db/repos.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"errors"
7
+
"fmt"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"tangled.org/core/appview/pagination"
11
+
"tangled.org/core/knotmirror/models"
12
+
)
13
+
14
+
func AddRepo(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, cid syntax.CID, name, knot string) error {
15
+
if _, err := e.ExecContext(ctx,
16
+
`insert into repos (did, rkey, cid, name, knot_domain)
17
+
values ($1, $2, $3, $4, $5)`,
18
+
did, rkey, cid, name, knot,
19
+
); err != nil {
20
+
return fmt.Errorf("inserting repo: %w", err)
21
+
}
22
+
return nil
23
+
}
24
+
25
+
func UpsertRepo(ctx context.Context, e *sql.DB, repo *models.Repo) error {
26
+
if _, err := e.ExecContext(ctx,
27
+
`insert into repos (did, rkey, cid, name, knot_domain, git_rev, repo_sha, state, error_msg, retry_count, retry_after)
28
+
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
29
+
on conflict(did, rkey) do update set
30
+
cid = excluded.cid,
31
+
name = excluded.name,
32
+
knot_domain = excluded.knot_domain,
33
+
git_rev = excluded.git_rev,
34
+
repo_sha = excluded.repo_sha,
35
+
state = excluded.state,
36
+
error_msg = excluded.error_msg,
37
+
retry_count = excluded.retry_count,
38
+
retry_after = excluded.retry_after`,
39
+
// where repos.cid != excluded.cid`,
40
+
repo.Did,
41
+
repo.Rkey,
42
+
repo.Cid,
43
+
repo.Name,
44
+
repo.KnotDomain,
45
+
repo.GitRev,
46
+
repo.RepoSha,
47
+
repo.State,
48
+
repo.ErrorMsg,
49
+
repo.RetryCount,
50
+
repo.RetryAfter,
51
+
); err != nil {
52
+
return fmt.Errorf("upserting repo: %w", err)
53
+
}
54
+
return nil
55
+
}
56
+
57
+
func UpdateRepoState(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, state models.RepoState) error {
58
+
if _, err := e.ExecContext(ctx,
59
+
`update repos
60
+
set state = $1
61
+
where did = $2 and rkey = $3`,
62
+
state,
63
+
did, rkey,
64
+
); err != nil {
65
+
return fmt.Errorf("updating repo: %w", err)
66
+
}
67
+
return nil
68
+
}
69
+
70
+
func DeleteRepo(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error {
71
+
if _, err := e.ExecContext(ctx,
72
+
`delete from repos where did = $1 and rkey = $2`,
73
+
did,
74
+
rkey,
75
+
); err != nil {
76
+
return fmt.Errorf("deleting repo: %w", err)
77
+
}
78
+
return nil
79
+
}
80
+
81
+
func GetRepoByName(ctx context.Context, e *sql.DB, did syntax.DID, name string) (*models.Repo, error) {
82
+
var repo models.Repo
83
+
if err := e.QueryRowContext(ctx,
84
+
`select
85
+
did,
86
+
rkey,
87
+
cid,
88
+
name,
89
+
knot_domain,
90
+
git_rev,
91
+
repo_sha,
92
+
state,
93
+
error_msg,
94
+
retry_count,
95
+
retry_after
96
+
from repos
97
+
where did = $1 and name = $2`,
98
+
did,
99
+
name,
100
+
).Scan(
101
+
&repo.Did,
102
+
&repo.Rkey,
103
+
&repo.Cid,
104
+
&repo.Name,
105
+
&repo.KnotDomain,
106
+
&repo.GitRev,
107
+
&repo.RepoSha,
108
+
&repo.State,
109
+
&repo.ErrorMsg,
110
+
&repo.RetryCount,
111
+
&repo.RetryAfter,
112
+
); err != nil {
113
+
if errors.Is(err, sql.ErrNoRows) {
114
+
return nil, nil
115
+
}
116
+
return nil, fmt.Errorf("querying repo: %w", err)
117
+
}
118
+
return &repo, nil
119
+
}
120
+
121
+
func GetRepoByAtUri(ctx context.Context, e *sql.DB, aturi syntax.ATURI) (*models.Repo, error) {
122
+
var repo models.Repo
123
+
if err := e.QueryRowContext(ctx,
124
+
`select
125
+
did,
126
+
rkey,
127
+
cid,
128
+
name,
129
+
knot_domain,
130
+
git_rev,
131
+
repo_sha,
132
+
state,
133
+
error_msg,
134
+
retry_count,
135
+
retry_after
136
+
from repos
137
+
where at_uri = $1`,
138
+
aturi,
139
+
).Scan(
140
+
&repo.Did,
141
+
&repo.Rkey,
142
+
&repo.Cid,
143
+
&repo.Name,
144
+
&repo.KnotDomain,
145
+
&repo.GitRev,
146
+
&repo.RepoSha,
147
+
&repo.State,
148
+
&repo.ErrorMsg,
149
+
&repo.RetryCount,
150
+
&repo.RetryAfter,
151
+
); err != nil {
152
+
if errors.Is(err, sql.ErrNoRows) {
153
+
return nil, nil
154
+
}
155
+
return nil, fmt.Errorf("querying repo: %w", err)
156
+
}
157
+
return &repo, nil
158
+
}
159
+
160
+
func ListRepos(ctx context.Context, e *sql.DB, page pagination.Page, did, knot, state string) ([]models.Repo, error) {
161
+
var conditions []string
162
+
var args []any
163
+
164
+
pageClause := ""
165
+
if page.Limit > 0 {
166
+
pageClause = " limit $1 offset $2 "
167
+
args = append(args, page.Limit, page.Offset)
168
+
}
169
+
170
+
whereClause := ""
171
+
if did != "" {
172
+
conditions = append(conditions, fmt.Sprintf("did = $%d", len(args)+1))
173
+
args = append(args, did)
174
+
}
175
+
if knot != "" {
176
+
conditions = append(conditions, fmt.Sprintf("knot_domain = $%d", len(args)+1))
177
+
args = append(args, knot)
178
+
}
179
+
if state != "" {
180
+
conditions = append(conditions, fmt.Sprintf("state = $%d", len(args)+1))
181
+
args = append(args, state)
182
+
}
183
+
if len(conditions) > 0 {
184
+
whereClause = "WHERE " + conditions[0]
185
+
for _, condition := range conditions[1:] {
186
+
whereClause += " AND " + condition
187
+
}
188
+
}
189
+
190
+
query := `
191
+
select
192
+
did,
193
+
rkey,
194
+
cid,
195
+
name,
196
+
knot_domain,
197
+
git_rev,
198
+
repo_sha,
199
+
state,
200
+
error_msg,
201
+
retry_count,
202
+
retry_after
203
+
from repos
204
+
` + whereClause + pageClause
205
+
rows, err := e.QueryContext(ctx, query, args...)
206
+
if err != nil {
207
+
return nil, err
208
+
}
209
+
defer rows.Close()
210
+
211
+
var repos []models.Repo
212
+
for rows.Next() {
213
+
var repo models.Repo
214
+
if err := rows.Scan(
215
+
&repo.Did,
216
+
&repo.Rkey,
217
+
&repo.Cid,
218
+
&repo.Name,
219
+
&repo.KnotDomain,
220
+
&repo.GitRev,
221
+
&repo.RepoSha,
222
+
&repo.State,
223
+
&repo.ErrorMsg,
224
+
&repo.RetryCount,
225
+
&repo.RetryAfter,
226
+
); err != nil {
227
+
return nil, fmt.Errorf("scanning row: %w", err)
228
+
}
229
+
repos = append(repos, repo)
230
+
}
231
+
if err := rows.Err(); err != nil {
232
+
return nil, fmt.Errorf("scanning rows: %w ", err)
233
+
}
234
+
235
+
return repos, nil
236
+
}
237
+
238
+
func GetRepoCountsByState(ctx context.Context, e *sql.DB) (map[models.RepoState]int64, error) {
239
+
const q = `
240
+
SELECT state, COUNT(*)
241
+
FROM repos
242
+
GROUP BY state
243
+
`
244
+
245
+
rows, err := e.QueryContext(ctx, q)
246
+
if err != nil {
247
+
return nil, err
248
+
}
249
+
defer rows.Close()
250
+
251
+
counts := make(map[models.RepoState]int64)
252
+
253
+
for rows.Next() {
254
+
var state string
255
+
var count int64
256
+
257
+
if err := rows.Scan(&state, &count); err != nil {
258
+
return nil, err
259
+
}
260
+
261
+
counts[models.RepoState(state)] = count
262
+
}
263
+
264
+
if err := rows.Err(); err != nil {
265
+
return nil, err
266
+
}
267
+
268
+
for _, s := range models.AllRepoStates {
269
+
if _, ok := counts[s]; !ok {
270
+
counts[s] = 0
271
+
}
272
+
}
273
+
274
+
return counts, nil
275
+
}
+305
knotmirror/git.go
+305
knotmirror/git.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
"net/url"
8
+
"os"
9
+
"os/exec"
10
+
"path/filepath"
11
+
"regexp"
12
+
"strings"
13
+
14
+
"github.com/go-git/go-git/v5"
15
+
gitconfig "github.com/go-git/go-git/v5/config"
16
+
"github.com/go-git/go-git/v5/plumbing/transport"
17
+
"tangled.org/core/knotmirror/models"
18
+
)
19
+
20
+
type GitMirrorManager interface {
21
+
Exist(repo *models.Repo) (bool, error)
22
+
// RemoteSetUrl updates git repository 'origin' remote
23
+
RemoteSetUrl(ctx context.Context, repo *models.Repo) error
24
+
// Clone clones the repository as a mirror
25
+
Clone(ctx context.Context, repo *models.Repo) error
26
+
// Fetch fetches the repository
27
+
Fetch(ctx context.Context, repo *models.Repo) error
28
+
// Sync mirrors the repository. It will clone the repository if repository doesn't exist.
29
+
Sync(ctx context.Context, repo *models.Repo) error
30
+
}
31
+
32
+
type CliGitMirrorManager struct {
33
+
repoBasePath string
34
+
knotUseSSL bool
35
+
}
36
+
37
+
func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager {
38
+
return &CliGitMirrorManager{
39
+
repoBasePath,
40
+
knotUseSSL,
41
+
}
42
+
}
43
+
44
+
var _ GitMirrorManager = new(CliGitMirrorManager)
45
+
46
+
func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string {
47
+
return filepath.Join(c.repoBasePath, repo.Did.String(), repo.Rkey.String())
48
+
}
49
+
50
+
func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
51
+
return isDir(c.makeRepoPath(repo))
52
+
}
53
+
54
+
func (c *CliGitMirrorManager) RemoteSetUrl(ctx context.Context, repo *models.Repo) error {
55
+
path := c.makeRepoPath(repo)
56
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
57
+
if err != nil {
58
+
return fmt.Errorf("constructing repo remote url: %w", err)
59
+
}
60
+
cmd := exec.CommandContext(ctx, "git", "-C", path, "remote", "set-url", "origin", url)
61
+
if out, err := cmd.CombinedOutput(); err != nil {
62
+
if ctx.Err() != nil {
63
+
return ctx.Err()
64
+
}
65
+
msg := string(out)
66
+
return fmt.Errorf("running 'git remote set-url origin %s': %w\n%s", url, err, msg)
67
+
}
68
+
return nil
69
+
}
70
+
71
+
func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
72
+
path := c.makeRepoPath(repo)
73
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
74
+
if err != nil {
75
+
return fmt.Errorf("constructing repo remote url: %w", err)
76
+
}
77
+
return c.clone(ctx, path, url)
78
+
}
79
+
80
+
func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error {
81
+
cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path)
82
+
if out, err := cmd.CombinedOutput(); err != nil {
83
+
if ctx.Err() != nil {
84
+
return ctx.Err()
85
+
}
86
+
msg := string(out)
87
+
if classification := classifyCliError(msg); classification != nil {
88
+
return classification
89
+
}
90
+
return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg)
91
+
}
92
+
return nil
93
+
}
94
+
95
+
func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
96
+
path := c.makeRepoPath(repo)
97
+
return c.fetch(ctx, path)
98
+
}
99
+
100
+
func (c *CliGitMirrorManager) fetch(ctx context.Context, path string) error {
101
+
// TODO: use `repo.Knot` instead of depending on origin
102
+
cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", "origin")
103
+
if out, err := cmd.CombinedOutput(); err != nil {
104
+
if ctx.Err() != nil {
105
+
return ctx.Err()
106
+
}
107
+
return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out))
108
+
}
109
+
return nil
110
+
}
111
+
112
+
func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
113
+
path := c.makeRepoPath(repo)
114
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
115
+
if err != nil {
116
+
return fmt.Errorf("constructing repo remote url: %w", err)
117
+
}
118
+
119
+
exist, err := isDir(path)
120
+
if err != nil {
121
+
return fmt.Errorf("checking repo path: %w", err)
122
+
}
123
+
if !exist {
124
+
if err := c.clone(ctx, path, url); err != nil {
125
+
return fmt.Errorf("cloning repo: %w", err)
126
+
}
127
+
} else {
128
+
if err := c.fetch(ctx, path); err != nil {
129
+
return fmt.Errorf("fetching repo: %w", err)
130
+
}
131
+
}
132
+
return nil
133
+
}
134
+
135
+
var (
136
+
ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)")
137
+
ErrCertExpired = errors.New("git: knot: certificate has expired")
138
+
ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch")
139
+
ErrTLSHandshake = errors.New("git: knot: tls handshake failure")
140
+
ErrHTTPStatus = errors.New("git: knot: request url returned error")
141
+
ErrUnreachable = errors.New("git: knot: could not connect to server")
142
+
ErrRepoNotFound = errors.New("git: repo: repository not found")
143
+
)
144
+
145
+
var (
146
+
reDNSFailure = regexp.MustCompile(`Could not resolve host:`)
147
+
reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`)
148
+
reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`)
149
+
reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`)
150
+
reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`)
151
+
reUnreachable = regexp.MustCompile(`Could not connect to server`)
152
+
reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`)
153
+
)
154
+
155
+
// classifyCliError classifies git cli error message. It will return nil for unknown error messages
156
+
func classifyCliError(stderr string) error {
157
+
msg := strings.TrimSpace(stderr)
158
+
if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 {
159
+
return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1])
160
+
}
161
+
if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 {
162
+
return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1])
163
+
}
164
+
switch {
165
+
case reDNSFailure.MatchString(msg):
166
+
return ErrDNSFailure
167
+
case reCertExpired.MatchString(msg):
168
+
return ErrCertExpired
169
+
case reCertMismatch.MatchString(msg):
170
+
return ErrCertMismatch
171
+
case reUnreachable.MatchString(msg):
172
+
return ErrUnreachable
173
+
case reRepoNotFound.MatchString(msg):
174
+
return ErrRepoNotFound
175
+
}
176
+
return nil
177
+
}
178
+
179
+
type GoGitMirrorManager struct {
180
+
repoBasePath string
181
+
knotUseSSL bool
182
+
}
183
+
184
+
func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager {
185
+
return &GoGitMirrorManager{
186
+
repoBasePath,
187
+
knotUseSSL,
188
+
}
189
+
}
190
+
191
+
var _ GitMirrorManager = new(GoGitMirrorManager)
192
+
193
+
func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string {
194
+
return filepath.Join(c.repoBasePath, repo.Did.String(), repo.Rkey.String())
195
+
}
196
+
197
+
func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
198
+
return isDir(c.makeRepoPath(repo))
199
+
}
200
+
201
+
func (c *GoGitMirrorManager) RemoteSetUrl(ctx context.Context, repo *models.Repo) error {
202
+
panic("unimplemented")
203
+
}
204
+
205
+
func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
206
+
path := c.makeRepoPath(repo)
207
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
208
+
if err != nil {
209
+
return fmt.Errorf("constructing repo remote url: %w", err)
210
+
}
211
+
return c.clone(ctx, path, url)
212
+
}
213
+
214
+
func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error {
215
+
_, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{
216
+
URL: url,
217
+
Mirror: true,
218
+
})
219
+
if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
220
+
return fmt.Errorf("cloning repo: %w", err)
221
+
}
222
+
return nil
223
+
}
224
+
225
+
func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
226
+
path := c.makeRepoPath(repo)
227
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
228
+
if err != nil {
229
+
return fmt.Errorf("constructing repo remote url: %w", err)
230
+
}
231
+
232
+
return c.fetch(ctx, path, url)
233
+
}
234
+
235
+
func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error {
236
+
gr, err := git.PlainOpen(path)
237
+
if err != nil {
238
+
return fmt.Errorf("opening local repo: %w", err)
239
+
}
240
+
if err := gr.FetchContext(ctx, &git.FetchOptions{
241
+
RemoteURL: url,
242
+
RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")},
243
+
Force: true,
244
+
Prune: true,
245
+
}); err != nil {
246
+
return fmt.Errorf("fetching reppo: %w", err)
247
+
}
248
+
return nil
249
+
}
250
+
251
+
func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
252
+
path := c.makeRepoPath(repo)
253
+
url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL)
254
+
if err != nil {
255
+
return fmt.Errorf("constructing repo remote url: %w", err)
256
+
}
257
+
258
+
exist, err := isDir(path)
259
+
if err != nil {
260
+
return fmt.Errorf("checking repo path: %w", err)
261
+
}
262
+
if !exist {
263
+
if err := c.clone(ctx, path, url); err != nil {
264
+
return fmt.Errorf("cloning repo: %w", err)
265
+
}
266
+
} else {
267
+
if err := c.fetch(ctx, path, url); err != nil {
268
+
return fmt.Errorf("fetching repo: %w", err)
269
+
}
270
+
}
271
+
return nil
272
+
}
273
+
274
+
func makeRepoRemoteUrl(knot, didSlashRepo string, knotUseSSL bool) (string, error) {
275
+
if !strings.Contains(knot, "://") {
276
+
if knotUseSSL {
277
+
knot = "https://" + knot
278
+
} else {
279
+
knot = "http://" + knot
280
+
}
281
+
}
282
+
283
+
u, err := url.Parse(knot)
284
+
if err != nil {
285
+
return "", err
286
+
}
287
+
288
+
if u.Scheme != "http" && u.Scheme != "https" {
289
+
return "", fmt.Errorf("unsupported scheme: %s", u.Scheme)
290
+
}
291
+
292
+
u = u.JoinPath(didSlashRepo)
293
+
return u.String(), nil
294
+
}
295
+
296
+
func isDir(path string) (bool, error) {
297
+
info, err := os.Stat(path)
298
+
if err == nil && info.IsDir() {
299
+
return true, nil
300
+
}
301
+
if os.IsNotExist(err) {
302
+
return false, nil
303
+
}
304
+
return false, err
305
+
}
+138
knotmirror/knotmirror.go
+138
knotmirror/knotmirror.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
_ "net/http/pprof"
8
+
"time"
9
+
10
+
"github.com/go-chi/chi/v5"
11
+
"github.com/prometheus/client_golang/prometheus/promhttp"
12
+
"tangled.org/core/idresolver"
13
+
"tangled.org/core/knotmirror/config"
14
+
"tangled.org/core/knotmirror/db"
15
+
"tangled.org/core/knotmirror/knotstream"
16
+
"tangled.org/core/knotmirror/models"
17
+
"tangled.org/core/knotmirror/xrpc"
18
+
"tangled.org/core/log"
19
+
)
20
+
21
+
func Run(ctx context.Context, cfg *config.Config) error {
22
+
// make sure every services are cleaned up on fast return
23
+
ctx, cancel := context.WithCancel(ctx)
24
+
defer cancel()
25
+
26
+
logger := log.FromContext(ctx)
27
+
28
+
db, err := db.Make(ctx, cfg.DbUrl, 32)
29
+
if err != nil {
30
+
return fmt.Errorf("initializing db: %w", err)
31
+
}
32
+
33
+
resolver := idresolver.DefaultResolver(cfg.PlcUrl)
34
+
35
+
// NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive.
36
+
gitm := NewCliGitMirrorManager(cfg.GitRepoBasePath, cfg.KnotUseSSL)
37
+
38
+
res, err := db.ExecContext(ctx,
39
+
`update repos set state = $1 where state = $2`,
40
+
models.RepoStateDesynchronized,
41
+
models.RepoStateResyncing,
42
+
)
43
+
if err != nil {
44
+
return fmt.Errorf("clearing resyning states: %w", err)
45
+
}
46
+
rows, err := res.RowsAffected()
47
+
if err != nil {
48
+
return fmt.Errorf("getting affected rows: %w", err)
49
+
}
50
+
logger.Info(fmt.Sprintf("clearing resyning states: %d records updated", rows))
51
+
52
+
xrpc := xrpc.New(logger, cfg, db, resolver)
53
+
knotstream := knotstream.NewKnotStream(logger, db, cfg)
54
+
crawler := NewCrawler(logger, db)
55
+
resyncer := NewResyncer(logger, db, gitm, cfg)
56
+
adminpage := NewAdminServer(logger, db, resyncer)
57
+
58
+
// maintain repository list with tap
59
+
// NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events.
60
+
tap := NewTapClient(logger, cfg, db, gitm, knotstream)
61
+
62
+
// start http server
63
+
go func() {
64
+
logger.Info("starting http server", "addr", cfg.Listen)
65
+
66
+
mux := chi.NewRouter()
67
+
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
68
+
w.Write([]byte("Welcome to a knotmirror server.\n"))
69
+
})
70
+
mux.Mount("/xrpc", xrpc.Router())
71
+
72
+
if err := http.ListenAndServe(cfg.Listen, mux); err != nil {
73
+
logger.Error("xrpc server failed", "error", err)
74
+
}
75
+
}()
76
+
77
+
// start metrics endpoint
78
+
go func() {
79
+
metricsAddr := cfg.MetricsListen
80
+
logger.Info("starting metrics server", "addr", metricsAddr)
81
+
http.Handle("/metrics", promhttp.Handler())
82
+
if err := http.ListenAndServe(metricsAddr, nil); err != nil {
83
+
logger.Error("metrics server failed", "error", err)
84
+
}
85
+
}()
86
+
87
+
// start admin page endpoint
88
+
go func() {
89
+
logger.Info("starting admin server", "addr", cfg.AdminListen)
90
+
if err := http.ListenAndServe(cfg.AdminListen, adminpage.Router()); err != nil {
91
+
logger.Error("admin server failed", "error", err)
92
+
}
93
+
}()
94
+
95
+
tap.Start(ctx)
96
+
97
+
resyncer.Start(ctx)
98
+
99
+
// periodically crawl the entire network to mirror the repositories
100
+
crawler.Start(ctx)
101
+
102
+
// listen to knotstream (currently we don't have relay for knots, so subscribe every known knots)
103
+
knotstream.Start(ctx)
104
+
105
+
svcErr := make(chan error, 1)
106
+
if err := knotstream.ResubscribeAllHosts(ctx); err != nil {
107
+
svcErr <- fmt.Errorf("resubscribing known hosts: %w", err)
108
+
}
109
+
110
+
logger.Info("startup complete")
111
+
select {
112
+
case <-ctx.Done():
113
+
logger.Info("received shutdown signal", "reason", ctx.Err())
114
+
case err := <-svcErr:
115
+
if err != nil {
116
+
logger.Error("service error", "error", err)
117
+
}
118
+
cancel()
119
+
}
120
+
121
+
logger.Info("shutting down knotmirror")
122
+
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
123
+
defer shutdownCancel()
124
+
125
+
var errs []error
126
+
if err := knotstream.Shutdown(shutdownCtx); err != nil {
127
+
errs = append(errs, err)
128
+
}
129
+
if err := db.Close(); err != nil {
130
+
errs = append(errs, err)
131
+
}
132
+
for _, err := range errs {
133
+
logger.Error("error during shutdown", "err", err)
134
+
}
135
+
136
+
logger.Info("shutdown complete")
137
+
return nil
138
+
}
+88
knotmirror/knotstream/knotstream.go
+88
knotmirror/knotstream/knotstream.go
···
1
+
package knotstream
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"fmt"
7
+
"log/slog"
8
+
"time"
9
+
10
+
"tangled.org/core/knotmirror/config"
11
+
"tangled.org/core/knotmirror/db"
12
+
"tangled.org/core/knotmirror/models"
13
+
"tangled.org/core/log"
14
+
)
15
+
16
+
type KnotStream struct {
17
+
logger *slog.Logger
18
+
db *sql.DB
19
+
slurper *KnotSlurper
20
+
}
21
+
22
+
func NewKnotStream(l *slog.Logger, db *sql.DB, cfg *config.Config) *KnotStream {
23
+
l = log.SubLogger(l, "knotstream")
24
+
return &KnotStream{
25
+
logger: l,
26
+
db: db,
27
+
slurper: NewKnotSlurper(l, db, cfg.Slurper),
28
+
}
29
+
}
30
+
31
+
func (s *KnotStream) Start(ctx context.Context) {
32
+
go s.slurper.Run(ctx)
33
+
}
34
+
35
+
func (s *KnotStream) Shutdown(ctx context.Context) error {
36
+
return s.slurper.Shutdown(ctx)
37
+
}
38
+
39
+
func (s *KnotStream) CheckIfSubscribed(hostname string) bool {
40
+
return s.slurper.CheckIfSubscribed(hostname)
41
+
}
42
+
43
+
func (s *KnotStream) SubscribeHost(ctx context.Context, hostname string, noSSL bool) error {
44
+
l := s.logger.With("hostname", hostname, "nossl", noSSL)
45
+
l.Debug("subscribe")
46
+
host, err := db.GetHost(ctx, s.db, hostname)
47
+
if err != nil {
48
+
return fmt.Errorf("loading host from db: %w", err)
49
+
}
50
+
51
+
if host == nil {
52
+
host = &models.Host{
53
+
Hostname: hostname,
54
+
NoSSL: noSSL,
55
+
Status: models.HostStatusActive,
56
+
LastSeq: 0,
57
+
}
58
+
59
+
if err := db.UpsertHost(ctx, s.db, host); err != nil {
60
+
return fmt.Errorf("adding host to db: %w", err)
61
+
}
62
+
63
+
l.Info("adding new host subscription")
64
+
}
65
+
66
+
if host.Status == models.HostStatusBanned {
67
+
return fmt.Errorf("cannot subscribe to banned knot")
68
+
}
69
+
return s.slurper.Subscribe(ctx, *host)
70
+
}
71
+
72
+
func (s *KnotStream) ResubscribeAllHosts(ctx context.Context) error {
73
+
hosts, err := db.ListHosts(ctx, s.db, models.HostStatusActive)
74
+
if err != nil {
75
+
return fmt.Errorf("listing hosts: %w", err)
76
+
}
77
+
78
+
for _, host := range hosts {
79
+
l := s.logger.With("hostname", host.Hostname)
80
+
l.Info("re-subscribing to active host")
81
+
if err := s.slurper.Subscribe(ctx, host); err != nil {
82
+
l.Warn("failed to re-subscribe to host", "err", err)
83
+
}
84
+
// sleep for a very short period, so we don't open tons of sockets at the same time
85
+
time.Sleep(1 * time.Millisecond)
86
+
}
87
+
return nil
88
+
}
+28
knotmirror/knotstream/metrics.go
+28
knotmirror/knotstream/metrics.go
···
1
+
package knotstream
2
+
3
+
import (
4
+
"github.com/prometheus/client_golang/prometheus"
5
+
"github.com/prometheus/client_golang/prometheus/promauto"
6
+
)
7
+
8
+
// KnotStream metrics
9
+
var (
10
+
knotstreamEventsReceived = promauto.NewCounter(prometheus.CounterOpts{
11
+
Name: "knotmirror_knotstream_events_received_total",
12
+
Help: "Total number of events received from knotstream",
13
+
})
14
+
knotstreamEventsProcessed = promauto.NewCounter(prometheus.CounterOpts{
15
+
Name: "knotmirror_knotstream_events_processed_total",
16
+
Help: "Total number of events successfully processed",
17
+
})
18
+
knotstreamEventsSkipped = promauto.NewCounter(prometheus.CounterOpts{
19
+
Name: "knotmirror_knotstream_events_skipped_total",
20
+
Help: "Total number of events skipped (not tracked)",
21
+
})
22
+
)
23
+
24
+
// slurper metrics
25
+
var connectedInbound = promauto.NewGauge(prometheus.GaugeOpts{
26
+
Name: "knotmirror_connected_inbound",
27
+
Help: "Number of inbound knotstream we are consuming",
28
+
})
+102
knotmirror/knotstream/scheduler.go
+102
knotmirror/knotstream/scheduler.go
···
1
+
package knotstream
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
"sync"
7
+
"sync/atomic"
8
+
"time"
9
+
10
+
"tangled.org/core/log"
11
+
)
12
+
13
+
type ParallelScheduler struct {
14
+
concurrency int
15
+
16
+
do func(ctx context.Context, task *Task) error
17
+
18
+
feeder chan *Task
19
+
lk sync.Mutex
20
+
scheduled map[string][]*Task
21
+
lastSeq atomic.Int64
22
+
23
+
logger *slog.Logger
24
+
}
25
+
26
+
type Task struct {
27
+
key string
28
+
message []byte
29
+
}
30
+
31
+
func NewParallelScheduler(maxC int, ident string, do func(context.Context, *Task) error) *ParallelScheduler {
32
+
return &ParallelScheduler{
33
+
concurrency: maxC,
34
+
do: do,
35
+
feeder: make(chan *Task),
36
+
scheduled: make(map[string][]*Task),
37
+
logger: log.New("parallel-scheduler"),
38
+
}
39
+
}
40
+
41
+
func (s *ParallelScheduler) Start(ctx context.Context) {
42
+
for range s.concurrency {
43
+
go s.ForEach(ctx, s.do)
44
+
}
45
+
}
46
+
47
+
func (s *ParallelScheduler) AddTask(ctx context.Context, task *Task) {
48
+
s.lk.Lock()
49
+
if st, ok := s.scheduled[task.key]; ok {
50
+
// schedule task
51
+
s.scheduled[task.key] = append(st, task)
52
+
s.lk.Unlock()
53
+
return
54
+
}
55
+
s.scheduled[task.key] = []*Task{}
56
+
s.lk.Unlock()
57
+
58
+
select {
59
+
case <-ctx.Done():
60
+
return
61
+
case s.feeder <- task:
62
+
return
63
+
}
64
+
}
65
+
66
+
func (s *ParallelScheduler) ForEach(ctx context.Context, fn func(context.Context, *Task) error) {
67
+
for task := range s.feeder {
68
+
for task != nil {
69
+
select {
70
+
case <-ctx.Done():
71
+
return
72
+
default:
73
+
}
74
+
if err := fn(ctx, task); err != nil {
75
+
s.logger.Error("event handler failed", "err", err)
76
+
}
77
+
78
+
s.lk.Lock()
79
+
func() {
80
+
rem, ok := s.scheduled[task.key]
81
+
if !ok {
82
+
s.logger.Error("should always have an 'active' entry if a worker is processing a job")
83
+
}
84
+
if len(rem) == 0 {
85
+
delete(s.scheduled, task.key)
86
+
task = nil
87
+
} else {
88
+
task = rem[0]
89
+
s.scheduled[task.key] = rem[1:]
90
+
}
91
+
92
+
// TODO: update seq from received message
93
+
s.lastSeq.Store(time.Now().UnixNano())
94
+
}()
95
+
s.lk.Unlock()
96
+
}
97
+
}
98
+
}
99
+
100
+
func (s *ParallelScheduler) LastSeq() int64 {
101
+
return s.lastSeq.Load()
102
+
}
+334
knotmirror/knotstream/slurper.go
+334
knotmirror/knotstream/slurper.go
···
1
+
package knotstream
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"encoding/json"
7
+
"fmt"
8
+
"log/slog"
9
+
"math/rand"
10
+
"net/http"
11
+
"sync"
12
+
"time"
13
+
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/bluesky-social/indigo/util/ssrf"
16
+
"github.com/carlmjohnson/versioninfo"
17
+
"github.com/gorilla/websocket"
18
+
"tangled.org/core/api/tangled"
19
+
"tangled.org/core/knotmirror/config"
20
+
"tangled.org/core/knotmirror/db"
21
+
"tangled.org/core/knotmirror/models"
22
+
"tangled.org/core/log"
23
+
)
24
+
25
+
type KnotSlurper struct {
26
+
logger *slog.Logger
27
+
db *sql.DB
28
+
cfg config.SlurperConfig
29
+
30
+
subsLk sync.Mutex
31
+
subs map[string]*subscription
32
+
}
33
+
34
+
func NewKnotSlurper(l *slog.Logger, db *sql.DB, cfg config.SlurperConfig) *KnotSlurper {
35
+
return &KnotSlurper{
36
+
logger: log.SubLogger(l, "slurper"),
37
+
db: db,
38
+
cfg: cfg,
39
+
subs: make(map[string]*subscription),
40
+
}
41
+
}
42
+
43
+
func (s *KnotSlurper) Run(ctx context.Context) {
44
+
for {
45
+
select {
46
+
case <-ctx.Done():
47
+
return
48
+
case <-time.After(s.cfg.PersistCursorPeriod):
49
+
if err := s.persistCursors(ctx); err != nil {
50
+
s.logger.Error("failed to flush cursors", "err", err)
51
+
}
52
+
}
53
+
}
54
+
}
55
+
56
+
func (s *KnotSlurper) CheckIfSubscribed(hostname string) bool {
57
+
s.subsLk.Lock()
58
+
defer s.subsLk.Unlock()
59
+
60
+
_, ok := s.subs[hostname]
61
+
return ok
62
+
}
63
+
64
+
func (s *KnotSlurper) Shutdown(ctx context.Context) error {
65
+
s.logger.Info("starting shutdown host cursor flush")
66
+
err := s.persistCursors(ctx)
67
+
if err != nil {
68
+
s.logger.Error("shutdown error", "err", err)
69
+
}
70
+
s.logger.Info("slurper shutdown complete")
71
+
return err
72
+
}
73
+
74
+
func (s *KnotSlurper) persistCursors(ctx context.Context) error {
75
+
// // gather cursor list from subscriptions and store them to DB
76
+
// start := time.Now()
77
+
78
+
s.subsLk.Lock()
79
+
cursors := make([]models.HostCursor, len(s.subs))
80
+
i := 0
81
+
for _, sub := range s.subs {
82
+
cursors[i] = sub.HostCursor()
83
+
i++
84
+
}
85
+
s.subsLk.Unlock()
86
+
87
+
err := db.StoreCursors(ctx, s.db, cursors)
88
+
// s.logger.Info("finished persisting cursors", "count", len(cursors), "duration", time.Since(start).String(), "err", err)
89
+
return err
90
+
}
91
+
92
+
func (s *KnotSlurper) Subscribe(ctx context.Context, host models.Host) error {
93
+
s.subsLk.Lock()
94
+
defer s.subsLk.Unlock()
95
+
96
+
_, ok := s.subs[host.Hostname]
97
+
if ok {
98
+
return fmt.Errorf("already subscribed: %s", host.Hostname)
99
+
}
100
+
101
+
// TODO: include `cancel` function to kill subscription by hostname
102
+
sub := &subscription{
103
+
hostname: host.Hostname,
104
+
scheduler: NewParallelScheduler(
105
+
s.cfg.ConcurrencyPerHost,
106
+
host.Hostname,
107
+
s.ProcessEvent,
108
+
),
109
+
}
110
+
s.subs[host.Hostname] = sub
111
+
112
+
sub.scheduler.Start(ctx)
113
+
go s.subscribeWithRedialer(ctx, host, sub)
114
+
return nil
115
+
}
116
+
117
+
func (s *KnotSlurper) subscribeWithRedialer(ctx context.Context, host models.Host, sub *subscription) {
118
+
l := s.logger.With("host", host.Hostname)
119
+
120
+
dialer := websocket.Dialer{
121
+
HandshakeTimeout: time.Second * 5,
122
+
}
123
+
124
+
// if this isn't a localhost / private connection, then we should enable SSRF protections
125
+
if !host.NoSSL {
126
+
netDialer := ssrf.PublicOnlyDialer()
127
+
dialer.NetDialContext = netDialer.DialContext
128
+
}
129
+
130
+
cursor := host.LastSeq
131
+
132
+
connectedInbound.Inc()
133
+
defer connectedInbound.Dec()
134
+
135
+
var backoff int
136
+
for {
137
+
select {
138
+
case <-ctx.Done():
139
+
return
140
+
default:
141
+
}
142
+
u := host.LegacyEventsURL(cursor)
143
+
l.Debug("made url with cursor", "cursor", cursor, "url", u)
144
+
145
+
// NOTE: manual backoff retry implementation to explicitly handle fails
146
+
hdr := make(http.Header)
147
+
hdr.Add("User-Agent", userAgent())
148
+
conn, resp, err := dialer.DialContext(ctx, u, hdr)
149
+
if err != nil {
150
+
l.Warn("dialing failed", "err", err, "backoff", backoff)
151
+
time.Sleep(sleepForBackoff(backoff))
152
+
backoff++
153
+
if backoff > 30 {
154
+
l.Warn("host does not appear to be online, disabling for now")
155
+
host.Status = models.HostStatusOffline
156
+
if err := db.UpsertHost(ctx, s.db, &host); err != nil {
157
+
l.Error("failed to update host status", "err", err)
158
+
}
159
+
return
160
+
}
161
+
continue
162
+
}
163
+
164
+
l.Debug("knot event subscription response", "code", resp.StatusCode, "url", u)
165
+
166
+
if err := s.handleConnection(ctx, conn, sub); err != nil {
167
+
// TODO: measure the last N connection error times and if they're coming too fast reconnect slower or don't reconnect and wait for requestCrawl
168
+
l.Warn("host connection failed", "err", err, "backoff", backoff)
169
+
}
170
+
171
+
updatedCursor := sub.LastSeq()
172
+
didProgress := updatedCursor > cursor
173
+
l.Debug("cursor compare", "cursor", cursor, "updatedCursor", updatedCursor, "didProgress", didProgress)
174
+
if cursor == 0 || didProgress {
175
+
cursor = updatedCursor
176
+
backoff = 0
177
+
178
+
batch := []models.HostCursor{sub.HostCursor()}
179
+
if err := db.StoreCursors(ctx, s.db, batch); err != nil {
180
+
l.Error("failed to store cursors", "err", err)
181
+
}
182
+
}
183
+
}
184
+
}
185
+
186
+
// handleConnection handles websocket connection.
187
+
// Schedules task from received event and return when connection is closed
188
+
func (s *KnotSlurper) handleConnection(ctx context.Context, conn *websocket.Conn, sub *subscription) error {
189
+
// ping on every 30s
190
+
ctx, cancel := context.WithCancel(ctx)
191
+
defer cancel() // close the background ping job on connection close
192
+
go func() {
193
+
t := time.NewTicker(30 * time.Second)
194
+
defer t.Stop()
195
+
failcount := 0
196
+
197
+
for {
198
+
select {
199
+
case <-t.C:
200
+
if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second*10)); err != nil {
201
+
s.logger.Warn("failed to ping", "err", err)
202
+
failcount++
203
+
if failcount >= 4 {
204
+
s.logger.Error("too many ping fails", "count", failcount)
205
+
_ = conn.Close()
206
+
return
207
+
}
208
+
} else {
209
+
failcount = 0 // ok ping
210
+
}
211
+
case <-ctx.Done():
212
+
_ = conn.Close()
213
+
return
214
+
}
215
+
}
216
+
}()
217
+
218
+
conn.SetPingHandler(func(message string) error {
219
+
err := conn.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Minute))
220
+
if err == websocket.ErrCloseSent {
221
+
return nil
222
+
}
223
+
return err
224
+
})
225
+
conn.SetPongHandler(func(_ string) error {
226
+
if err := conn.SetReadDeadline(time.Now().Add(time.Minute)); err != nil {
227
+
s.logger.Error("failed to set read deadline", "err", err)
228
+
}
229
+
return nil
230
+
})
231
+
232
+
for {
233
+
select {
234
+
case <-ctx.Done():
235
+
return ctx.Err()
236
+
default:
237
+
}
238
+
msgType, msg, err := conn.ReadMessage()
239
+
if err != nil {
240
+
return err
241
+
}
242
+
243
+
if msgType != websocket.TextMessage {
244
+
continue
245
+
}
246
+
247
+
sub.scheduler.AddTask(ctx, &Task{
248
+
key: sub.hostname, // TODO: replace to repository AT-URI for better concurrency
249
+
message: msg,
250
+
})
251
+
}
252
+
}
253
+
254
+
type LegacyGitEvent struct {
255
+
Rkey string
256
+
Nsid string
257
+
Event tangled.GitRefUpdate
258
+
}
259
+
260
+
func (s *KnotSlurper) ProcessEvent(ctx context.Context, task *Task) error {
261
+
var legacyMessage LegacyGitEvent
262
+
if err := json.Unmarshal(task.message, &legacyMessage); err != nil {
263
+
return fmt.Errorf("unmarshaling message: %w", err)
264
+
}
265
+
266
+
if err := s.ProcessLegacyGitRefUpdate(ctx, &legacyMessage); err != nil {
267
+
return fmt.Errorf("processing gitRefUpdate: %w", err)
268
+
}
269
+
return nil
270
+
}
271
+
272
+
func (s *KnotSlurper) ProcessLegacyGitRefUpdate(ctx context.Context, evt *LegacyGitEvent) error {
273
+
knotstreamEventsReceived.Inc()
274
+
275
+
curr, err := db.GetRepoByName(ctx, s.db, syntax.DID(evt.Event.RepoDid), evt.Event.RepoName)
276
+
if err != nil {
277
+
return fmt.Errorf("failed to get repo '%s': %w", evt.Event.RepoDid+"/"+evt.Event.RepoName, err)
278
+
}
279
+
if curr == nil {
280
+
// if repo doesn't exist in DB, just ignore the event. That repo is unknown.
281
+
//
282
+
// Normally did+name is already enough to perform git-fetch as that's
283
+
// what needed to fetch the repository.
284
+
// But we want to store that in did/rkey in knot-mirror.
285
+
// Therefore, we should ignore when the repository is unknown.
286
+
// Hopefully crawler will sync it later.
287
+
s.logger.Warn("skipping event from unknown repo", "did/repo", evt.Event.RepoDid+"/"+evt.Event.RepoName)
288
+
knotstreamEventsSkipped.Inc()
289
+
return nil
290
+
}
291
+
l := s.logger.With("repoAt", curr.AtUri())
292
+
293
+
// TODO: should plan resync to resyncBuffer on RepoStateResyncing
294
+
if curr.State != models.RepoStateActive {
295
+
l.Debug("skipping non-active repo")
296
+
knotstreamEventsSkipped.Inc()
297
+
return nil
298
+
}
299
+
300
+
if curr.GitRev != "" && evt.Rkey <= curr.GitRev.String() {
301
+
l.Debug("skipping replayed event", "event.Rkey", evt.Rkey, "currentRev", curr.GitRev)
302
+
knotstreamEventsSkipped.Inc()
303
+
return nil
304
+
}
305
+
306
+
// if curr.State == models.RepoStateResyncing {
307
+
// firehoseEventsSkipped.Inc()
308
+
// return fp.events.addToResyncBuffer(ctx, commit)
309
+
// }
310
+
311
+
// can't skip anything, update repo state
312
+
if err := db.UpdateRepoState(ctx, s.db, curr.Did, curr.Rkey, models.RepoStateDesynchronized); err != nil {
313
+
return err
314
+
}
315
+
316
+
l.Info("event processed", "eventRev", evt.Rkey)
317
+
318
+
knotstreamEventsProcessed.Inc()
319
+
return nil
320
+
}
321
+
322
+
func userAgent() string {
323
+
return fmt.Sprintf("knotmirror/%s", versioninfo.Short())
324
+
}
325
+
326
+
func sleepForBackoff(b int) time.Duration {
327
+
if b == 0 {
328
+
return 0
329
+
}
330
+
if b < 10 {
331
+
return time.Millisecond * time.Duration((50*b)+rand.Intn(500))
332
+
}
333
+
return time.Second * 30
334
+
}
+22
knotmirror/knotstream/subscription.go
+22
knotmirror/knotstream/subscription.go
···
1
+
package knotstream
2
+
3
+
import "tangled.org/core/knotmirror/models"
4
+
5
+
// subscription represents websocket connection with that host
6
+
type subscription struct {
7
+
hostname string
8
+
9
+
// embedded parallel job scheduler
10
+
scheduler *ParallelScheduler
11
+
}
12
+
13
+
func (s *subscription) LastSeq() int64 {
14
+
return s.scheduler.LastSeq()
15
+
}
16
+
17
+
func (s *subscription) HostCursor() models.HostCursor {
18
+
return models.HostCursor{
19
+
Hostname: s.hostname,
20
+
LastSeq: s.LastSeq(),
21
+
}
22
+
}
+29
knotmirror/metrics.go
+29
knotmirror/metrics.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"github.com/prometheus/client_golang/prometheus"
5
+
"github.com/prometheus/client_golang/prometheus/promauto"
6
+
)
7
+
8
+
// Resync metrics
9
+
var (
10
+
// TODO:
11
+
// - working / waiting resycner counts
12
+
resyncsStarted = promauto.NewCounter(prometheus.CounterOpts{
13
+
Name: "knotmirror_resyncs_started_total",
14
+
Help: "Total number of repo resyncs started",
15
+
})
16
+
resyncsCompleted = promauto.NewCounter(prometheus.CounterOpts{
17
+
Name: "knotmirror_resyncs_completed_total",
18
+
Help: "Total number of repo resyncs completed",
19
+
})
20
+
resyncsFailed = promauto.NewCounter(prometheus.CounterOpts{
21
+
Name: "knotmirror_resyncs_failed_total",
22
+
Help: "Total number of repo resyncs failed",
23
+
})
24
+
resyncDuration = promauto.NewHistogram(prometheus.HistogramOpts{
25
+
Name: "knotmirror_resync_duration_seconds",
26
+
Help: "Duration of repo resync operations",
27
+
Buckets: prometheus.ExponentialBuckets(0.1, 2, 12),
28
+
})
29
+
)
+110
knotmirror/models/models.go
+110
knotmirror/models/models.go
···
1
+
package models
2
+
3
+
import (
4
+
"fmt"
5
+
6
+
"github.com/bluesky-social/indigo/atproto/syntax"
7
+
"tangled.org/core/api/tangled"
8
+
)
9
+
10
+
type Repo struct {
11
+
Did syntax.DID
12
+
Rkey syntax.RecordKey
13
+
Cid *syntax.CID
14
+
// content of tangled.Repo
15
+
Name string
16
+
KnotDomain string
17
+
18
+
GitRev syntax.TID // last processed git.refUpdate revision
19
+
RepoSha string // sha256 sum of git refs (to avoid no-op git fetch)
20
+
State RepoState
21
+
ErrorMsg string
22
+
RetryCount int
23
+
RetryAfter int64 // Unix timestamp (seconds)
24
+
}
25
+
26
+
func (r *Repo) AtUri() syntax.ATURI {
27
+
return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, tangled.RepoNSID, r.Rkey))
28
+
}
29
+
30
+
func (r *Repo) DidSlashRepo() string {
31
+
return fmt.Sprintf("%s/%s", r.Did, r.Name)
32
+
}
33
+
34
+
type RepoState string
35
+
36
+
const (
37
+
RepoStatePending RepoState = "pending"
38
+
RepoStateDesynchronized RepoState = "desynchronized"
39
+
RepoStateResyncing RepoState = "resyncing"
40
+
RepoStateActive RepoState = "active"
41
+
RepoStateSuspended RepoState = "suspended"
42
+
RepoStateError RepoState = "error"
43
+
)
44
+
45
+
var AllRepoStates = []RepoState{
46
+
RepoStatePending,
47
+
RepoStateDesynchronized,
48
+
RepoStateResyncing,
49
+
RepoStateActive,
50
+
RepoStateSuspended,
51
+
RepoStateError,
52
+
}
53
+
54
+
func (s RepoState) IsResyncing() bool {
55
+
return s == RepoStateResyncing
56
+
}
57
+
58
+
type HostCursor struct {
59
+
Hostname string
60
+
LastSeq int64
61
+
}
62
+
63
+
type Host struct {
64
+
Hostname string
65
+
NoSSL bool
66
+
Status HostStatus
67
+
LastSeq int64
68
+
}
69
+
70
+
type HostStatus string
71
+
72
+
const (
73
+
HostStatusActive HostStatus = "active"
74
+
HostStatusIdle HostStatus = "idle"
75
+
HostStatusOffline HostStatus = "offline"
76
+
HostStatusThrottled HostStatus = "throttled"
77
+
HostStatusBanned HostStatus = "banned"
78
+
)
79
+
80
+
var AllHostStatuses = []HostStatus{
81
+
HostStatusActive,
82
+
HostStatusIdle,
83
+
HostStatusOffline,
84
+
HostStatusThrottled,
85
+
HostStatusBanned,
86
+
}
87
+
88
+
// func (h *Host) SubscribeGitRefsURL(cursor int64) string {
89
+
// scheme := "wss"
90
+
// if h.NoSSL {
91
+
// scheme = "ws"
92
+
// }
93
+
// u := fmt.Sprintf("%s://%s/xrpc/%s", scheme, h.Hostname, tangled.SubscribeGitRefsNSID)
94
+
// if cursor > 0 {
95
+
// u = fmt.Sprintf("%s?cursor=%d", u, h.LastSeq)
96
+
// }
97
+
// return u
98
+
// }
99
+
100
+
func (h *Host) LegacyEventsURL(cursor int64) string {
101
+
scheme := "wss"
102
+
if h.NoSSL {
103
+
scheme = "ws"
104
+
}
105
+
u := fmt.Sprintf("%s://%s/events", scheme, h.Hostname)
106
+
if cursor > 0 {
107
+
u = fmt.Sprintf("%s?cursor=%d", u, cursor)
108
+
}
109
+
return u
110
+
}
+8
knotmirror/readme.md
+8
knotmirror/readme.md
···
1
+
# KnotMirror
2
+
3
+
KnotMirror is a git mirror service for all known repos. Heavily inspired by [indigo/relay] and [indigo/tap].
4
+
5
+
KnotMirror syncs repo list using tap and subscribe to all known knots as KnotStream.
6
+
7
+
[indigo/relay]: https://github.com/bluesky-social/indigo/tree/main/cmd/relay
8
+
[indigo/tap]: https://github.com/bluesky-social/indigo/tree/main/cmd/tap
+356
knotmirror/resyncer.go
+356
knotmirror/resyncer.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"errors"
7
+
"fmt"
8
+
"log/slog"
9
+
"math/rand"
10
+
"net/http"
11
+
"net/url"
12
+
"strings"
13
+
"sync"
14
+
"time"
15
+
16
+
"github.com/bluesky-social/indigo/atproto/syntax"
17
+
"tangled.org/core/knotmirror/config"
18
+
"tangled.org/core/knotmirror/db"
19
+
"tangled.org/core/knotmirror/models"
20
+
"tangled.org/core/log"
21
+
)
22
+
23
+
type Resyncer struct {
24
+
logger *slog.Logger
25
+
db *sql.DB
26
+
gitm GitMirrorManager
27
+
28
+
claimJobMu sync.Mutex
29
+
30
+
runningJobs map[syntax.ATURI]context.CancelFunc
31
+
runningJobsMu sync.Mutex
32
+
33
+
repoFetchTimeout time.Duration
34
+
manualResyncTimeout time.Duration
35
+
parallelism int
36
+
37
+
knotBackoff map[string]time.Time
38
+
knotBackoffMu sync.RWMutex
39
+
}
40
+
41
+
func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config.Config) *Resyncer {
42
+
return &Resyncer{
43
+
logger: log.SubLogger(l, "resyncer"),
44
+
db: db,
45
+
gitm: gitm,
46
+
47
+
runningJobs: make(map[syntax.ATURI]context.CancelFunc),
48
+
49
+
repoFetchTimeout: cfg.GitRepoFetchTimeout,
50
+
manualResyncTimeout: 30 * time.Minute,
51
+
parallelism: cfg.ResyncParallelism,
52
+
53
+
knotBackoff: make(map[string]time.Time),
54
+
}
55
+
}
56
+
57
+
func (r *Resyncer) Start(ctx context.Context) {
58
+
for i := 0; i < r.parallelism; i++ {
59
+
go r.runResyncWorker(ctx, i)
60
+
}
61
+
}
62
+
63
+
func (r *Resyncer) runResyncWorker(ctx context.Context, workerID int) {
64
+
l := r.logger.With("worker", workerID)
65
+
for {
66
+
select {
67
+
case <-ctx.Done():
68
+
l.Info("resync worker shutting down", "error", ctx.Err())
69
+
return
70
+
default:
71
+
}
72
+
repoAt, found, err := r.claimResyncJob(ctx)
73
+
if err != nil {
74
+
l.Error("failed to claim resync job", "error", err)
75
+
time.Sleep(time.Second)
76
+
continue
77
+
}
78
+
if !found {
79
+
time.Sleep(time.Second)
80
+
continue
81
+
}
82
+
l.Info("processing resync", "aturi", repoAt)
83
+
if err := r.resyncRepo(ctx, repoAt); err != nil {
84
+
l.Error("resync failed", "aturi", repoAt, "error", err)
85
+
}
86
+
}
87
+
}
88
+
89
+
func (r *Resyncer) registerRunning(repo syntax.ATURI, cancel context.CancelFunc) {
90
+
r.runningJobsMu.Lock()
91
+
defer r.runningJobsMu.Unlock()
92
+
93
+
if _, exists := r.runningJobs[repo]; exists {
94
+
return
95
+
}
96
+
r.runningJobs[repo] = cancel
97
+
}
98
+
99
+
func (r *Resyncer) unregisterRunning(repo syntax.ATURI) {
100
+
r.runningJobsMu.Lock()
101
+
defer r.runningJobsMu.Unlock()
102
+
103
+
delete(r.runningJobs, repo)
104
+
}
105
+
106
+
func (r *Resyncer) CancelResyncJob(repo syntax.ATURI) {
107
+
r.runningJobsMu.Lock()
108
+
defer r.runningJobsMu.Unlock()
109
+
110
+
cancel, ok := r.runningJobs[repo]
111
+
if !ok {
112
+
return
113
+
}
114
+
delete(r.runningJobs, repo)
115
+
cancel()
116
+
}
117
+
118
+
// TriggerResyncJob manually triggers the resync job
119
+
func (r *Resyncer) TriggerResyncJob(ctx context.Context, repoAt syntax.ATURI) error {
120
+
repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt)
121
+
if err != nil {
122
+
return fmt.Errorf("failed to get repo: %w", err)
123
+
}
124
+
if repo == nil {
125
+
return fmt.Errorf("repo not found: %s", repoAt)
126
+
}
127
+
128
+
if repo.State == models.RepoStateResyncing {
129
+
return fmt.Errorf("repo already resyncing")
130
+
}
131
+
132
+
repo.State = models.RepoStatePending
133
+
repo.RetryAfter = -1 // resyncer will prioritize this
134
+
135
+
if err := db.UpsertRepo(ctx, r.db, repo); err != nil {
136
+
return fmt.Errorf("updating repo state to pending %w", err)
137
+
}
138
+
return nil
139
+
}
140
+
141
+
func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, bool, error) {
142
+
// use mutex to prevent duplicated jobs
143
+
r.claimJobMu.Lock()
144
+
defer r.claimJobMu.Unlock()
145
+
146
+
var repoAt syntax.ATURI
147
+
now := time.Now().Unix()
148
+
if err := r.db.QueryRowContext(ctx,
149
+
`update repos
150
+
set state = $1
151
+
where at_uri = (
152
+
select at_uri from repos
153
+
where state in ($2, $3, $4)
154
+
and (retry_after = -1 or retry_after = 0 or retry_after < $5)
155
+
order by
156
+
(retry_after = -1) desc,
157
+
(retry_after = 0) desc,
158
+
retry_after
159
+
limit 1
160
+
)
161
+
returning at_uri
162
+
`,
163
+
models.RepoStateResyncing,
164
+
models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError,
165
+
now,
166
+
).Scan(&repoAt); err != nil {
167
+
if errors.Is(err, sql.ErrNoRows) {
168
+
return "", false, nil
169
+
}
170
+
return "", false, err
171
+
}
172
+
173
+
return repoAt, true, nil
174
+
}
175
+
176
+
func (r *Resyncer) resyncRepo(ctx context.Context, repoAt syntax.ATURI) error {
177
+
// ctx, span := tracer.Start(ctx, "resyncRepo")
178
+
// span.SetAttributes(attribute.String("aturi", repoAt))
179
+
// defer span.End()
180
+
181
+
resyncsStarted.Inc()
182
+
startTime := time.Now()
183
+
184
+
jobCtx, cancel := context.WithCancel(ctx)
185
+
r.registerRunning(repoAt, cancel)
186
+
defer r.unregisterRunning(repoAt)
187
+
188
+
success, err := r.doResync(jobCtx, repoAt)
189
+
if !success {
190
+
resyncsFailed.Inc()
191
+
resyncDuration.Observe(time.Since(startTime).Seconds())
192
+
return r.handleResyncFailure(ctx, repoAt, err)
193
+
}
194
+
195
+
resyncsCompleted.Inc()
196
+
resyncDuration.Observe(time.Since(startTime).Seconds())
197
+
return nil
198
+
}
199
+
200
+
func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, error) {
201
+
// ctx, span := tracer.Start(ctx, "doResync")
202
+
// span.SetAttributes(attribute.String("aturi", repoAt))
203
+
// defer span.End()
204
+
205
+
repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt)
206
+
if err != nil {
207
+
return false, fmt.Errorf("failed to get repo: %w", err)
208
+
}
209
+
if repo == nil { // untracked repo, skip
210
+
return false, nil
211
+
}
212
+
213
+
r.knotBackoffMu.RLock()
214
+
backoffUntil, inBackoff := r.knotBackoff[repo.KnotDomain]
215
+
r.knotBackoffMu.RUnlock()
216
+
if inBackoff && time.Now().Before(backoffUntil) {
217
+
return false, nil
218
+
}
219
+
220
+
// HACK: check knot reachability with short timeout before running actual fetch.
221
+
// This is crucial as git-cli doesn't support http connection timeout.
222
+
// `http.lowSpeedTime` is only applied _after_ the connection.
223
+
if err := r.checkKnotReachability(ctx, repo); err != nil {
224
+
if isRateLimitError(err) {
225
+
r.knotBackoffMu.Lock()
226
+
r.knotBackoff[repo.KnotDomain] = time.Now().Add(10 * time.Second)
227
+
r.knotBackoffMu.Unlock()
228
+
return false, nil
229
+
}
230
+
// TODO: suspend repo on 404. KnotStream updates will change the repo state back online
231
+
return false, fmt.Errorf("knot unreachable: %w", err)
232
+
}
233
+
234
+
timeout := r.repoFetchTimeout
235
+
if repo.RetryAfter == -1 {
236
+
timeout = r.manualResyncTimeout
237
+
}
238
+
fetchCtx, cancel := context.WithTimeout(ctx, timeout)
239
+
defer cancel()
240
+
241
+
if err := r.gitm.Sync(fetchCtx, repo); err != nil {
242
+
return false, err
243
+
}
244
+
245
+
// repo.GitRev = <processed git.refUpdate revision>
246
+
// repo.RepoSha = <sha256 sum of git refs>
247
+
repo.State = models.RepoStateActive
248
+
repo.ErrorMsg = ""
249
+
repo.RetryCount = 0
250
+
repo.RetryAfter = 0
251
+
if err := db.UpsertRepo(ctx, r.db, repo); err != nil {
252
+
return false, fmt.Errorf("updating repo state to active %w", err)
253
+
}
254
+
return true, nil
255
+
}
256
+
257
+
type knotStatusError struct {
258
+
StatusCode int
259
+
}
260
+
261
+
func (ke *knotStatusError) Error() string {
262
+
return fmt.Sprintf("request failed with status code (HTTP %d)", ke.StatusCode)
263
+
}
264
+
265
+
func isRateLimitError(err error) bool {
266
+
var knotErr *knotStatusError
267
+
if errors.As(err, &knotErr) {
268
+
return knotErr.StatusCode == http.StatusTooManyRequests
269
+
}
270
+
return false
271
+
}
272
+
273
+
// checkKnotReachability checks if Knot is reachable and is valid git remote server
274
+
func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) error {
275
+
repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), true)
276
+
if err != nil {
277
+
return err
278
+
}
279
+
280
+
repoUrl += "/info/refs?service=git-upload-pack"
281
+
282
+
client := http.Client{
283
+
Timeout: 30 * time.Second,
284
+
}
285
+
req, err := http.NewRequestWithContext(ctx, "GET", repoUrl, nil)
286
+
if err != nil {
287
+
return err
288
+
}
289
+
req.Header.Set("User-Agent", "git/2.x")
290
+
req.Header.Set("Accept", "*/*")
291
+
292
+
resp, err := client.Do(req)
293
+
if err != nil {
294
+
var uerr *url.Error
295
+
if errors.As(err, &uerr) {
296
+
return fmt.Errorf("request failed: %w", uerr.Unwrap())
297
+
}
298
+
return fmt.Errorf("request failed: %w", err)
299
+
}
300
+
defer resp.Body.Close()
301
+
302
+
if resp.StatusCode != http.StatusOK {
303
+
return &knotStatusError{resp.StatusCode}
304
+
}
305
+
306
+
// check if target is git server
307
+
ct := resp.Header.Get("Content-Type")
308
+
if !strings.Contains(ct, "application/x-git-upload-pack-advertisement") {
309
+
return fmt.Errorf("unexpected content-type: %s", ct)
310
+
}
311
+
312
+
return nil
313
+
}
314
+
315
+
func (r *Resyncer) handleResyncFailure(ctx context.Context, repoAt syntax.ATURI, err error) error {
316
+
r.logger.Debug("handleResyncFailure", "at_uri", repoAt, "err", err)
317
+
var state models.RepoState
318
+
var errMsg string
319
+
if err == nil {
320
+
state = models.RepoStateDesynchronized
321
+
errMsg = ""
322
+
} else {
323
+
state = models.RepoStateError
324
+
errMsg = err.Error()
325
+
}
326
+
327
+
repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt)
328
+
if err != nil {
329
+
return fmt.Errorf("failed to get repo: %w", err)
330
+
}
331
+
if repo == nil {
332
+
return fmt.Errorf("failed to get repo. repo '%s' doesn't exist in db", repoAt)
333
+
}
334
+
335
+
// start a 1 min & go up to 1 hr between retries
336
+
var retryCount = repo.RetryCount + 1
337
+
var retryAfter = time.Now().Add(backoff(retryCount, 60) * 60).Unix()
338
+
339
+
// remove null bytes
340
+
errMsg = strings.ReplaceAll(errMsg, "\x00", "")
341
+
342
+
repo.State = state
343
+
repo.ErrorMsg = errMsg
344
+
repo.RetryCount = retryCount
345
+
repo.RetryAfter = retryAfter
346
+
if err := db.UpsertRepo(ctx, r.db, repo); err != nil {
347
+
return fmt.Errorf("failed to update repo state: %w", err)
348
+
}
349
+
return nil
350
+
}
351
+
352
+
func backoff(retries int, max int) time.Duration {
353
+
dur := min(1<<retries, max)
354
+
jitter := time.Millisecond * time.Duration(rand.Intn(1000))
355
+
return time.Second*time.Duration(dur) + jitter
356
+
}
+152
knotmirror/tapclient.go
+152
knotmirror/tapclient.go
···
1
+
package knotmirror
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"encoding/json"
7
+
"fmt"
8
+
"log/slog"
9
+
"net/netip"
10
+
"net/url"
11
+
"time"
12
+
13
+
"tangled.org/core/api/tangled"
14
+
"tangled.org/core/knotmirror/config"
15
+
"tangled.org/core/knotmirror/db"
16
+
"tangled.org/core/knotmirror/knotstream"
17
+
"tangled.org/core/knotmirror/models"
18
+
"tangled.org/core/log"
19
+
"tangled.org/core/tapc"
20
+
)
21
+
22
+
type Tap struct {
23
+
logger *slog.Logger
24
+
cfg *config.Config
25
+
tap tapc.Client
26
+
db *sql.DB
27
+
gitm GitMirrorManager
28
+
ks *knotstream.KnotStream
29
+
}
30
+
31
+
func NewTapClient(l *slog.Logger, cfg *config.Config, db *sql.DB, gitm GitMirrorManager, ks *knotstream.KnotStream) *Tap {
32
+
return &Tap{
33
+
logger: log.SubLogger(l, "tapclient"),
34
+
cfg: cfg,
35
+
tap: tapc.NewClient(cfg.TapUrl, ""),
36
+
db: db,
37
+
gitm: gitm,
38
+
ks: ks,
39
+
}
40
+
}
41
+
42
+
func (t *Tap) Start(ctx context.Context) {
43
+
// TODO: better reconnect logic
44
+
go func() {
45
+
for {
46
+
t.tap.Connect(ctx, &tapc.SimpleIndexer{
47
+
EventHandler: t.processEvent,
48
+
})
49
+
time.Sleep(time.Second)
50
+
}
51
+
}()
52
+
}
53
+
54
+
func (t *Tap) processEvent(ctx context.Context, evt tapc.Event) error {
55
+
l := t.logger.With("component", "tapIndexer")
56
+
57
+
var err error
58
+
switch evt.Type {
59
+
case tapc.EvtRecord:
60
+
switch evt.Record.Collection.String() {
61
+
case tangled.RepoNSID:
62
+
err = t.processRepo(ctx, evt.Record)
63
+
}
64
+
}
65
+
66
+
if err != nil {
67
+
l.Error("failed to process message. will retry later", "event.ID", evt.ID, "err", err)
68
+
return err
69
+
}
70
+
return nil
71
+
}
72
+
73
+
func (t *Tap) processRepo(ctx context.Context, evt *tapc.RecordEventData) error {
74
+
switch evt.Action {
75
+
case tapc.RecordCreateAction, tapc.RecordUpdateAction:
76
+
record := tangled.Repo{}
77
+
if err := json.Unmarshal(evt.Record, &record); err != nil {
78
+
return fmt.Errorf("parsing record: %w", err)
79
+
}
80
+
81
+
status := models.RepoStatePending
82
+
errMsg := ""
83
+
u, err := url.Parse("http://" + record.Knot) // parsing with fake scheme
84
+
if err != nil {
85
+
status = models.RepoStateSuspended
86
+
errMsg = "failed to parse knot url"
87
+
} else if t.cfg.KnotSSRF && isPrivate(u.Hostname()) {
88
+
status = models.RepoStateSuspended
89
+
errMsg = "suspending non-public knot"
90
+
}
91
+
92
+
repo := &models.Repo{
93
+
Did: evt.Did,
94
+
Rkey: evt.Rkey,
95
+
Cid: evt.CID,
96
+
Name: record.Name,
97
+
KnotDomain: record.Knot,
98
+
State: status,
99
+
ErrorMsg: errMsg,
100
+
RetryAfter: 0, // clear retry info
101
+
RetryCount: 0,
102
+
}
103
+
104
+
if evt.Action == tapc.RecordUpdateAction {
105
+
exist, err := t.gitm.Exist(repo)
106
+
if err != nil {
107
+
return fmt.Errorf("checking git repo existance: %w", err)
108
+
}
109
+
if exist {
110
+
// update git repo remote url
111
+
if err := t.gitm.RemoteSetUrl(ctx, repo); err != nil {
112
+
return fmt.Errorf("updating git repo remote url: %w", err)
113
+
}
114
+
}
115
+
}
116
+
117
+
if err := db.UpsertRepo(ctx, t.db, repo); err != nil {
118
+
return fmt.Errorf("upserting repo to db: %w", err)
119
+
}
120
+
121
+
if !t.ks.CheckIfSubscribed(record.Knot) {
122
+
if err := t.ks.SubscribeHost(ctx, record.Knot, !t.cfg.KnotUseSSL); err != nil {
123
+
return fmt.Errorf("subscribing to knot: %w", err)
124
+
}
125
+
}
126
+
127
+
case tapc.RecordDeleteAction:
128
+
if err := db.DeleteRepo(ctx, t.db, evt.Did, evt.Rkey); err != nil {
129
+
return fmt.Errorf("deleting repo from db: %w", err)
130
+
}
131
+
}
132
+
return nil
133
+
}
134
+
135
+
// isPrivate checks if host is private network. It doesn't perform DNS resolution
136
+
func isPrivate(host string) bool {
137
+
if host == "localhost" {
138
+
return true
139
+
}
140
+
addr, err := netip.ParseAddr(host)
141
+
if err != nil {
142
+
return false
143
+
}
144
+
return isPrivateAddr(addr)
145
+
}
146
+
147
+
func isPrivateAddr(addr netip.Addr) bool {
148
+
return addr.IsLoopback() ||
149
+
addr.IsPrivate() ||
150
+
addr.IsLinkLocalUnicast() ||
151
+
addr.IsLinkLocalMulticast()
152
+
}
+55
knotmirror/templates/base.html
+55
knotmirror/templates/base.html
···
1
+
{{define "base"}}
2
+
<!DOCTYPE html>
3
+
<html>
4
+
<head>
5
+
<title>KnotMirror Admin</title>
6
+
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.8/dist/htmx.min.js" integrity="sha384-/TgkGk7p307TH7EXJDuUlgG3Ce1UVolAOFopFekQkkXihi5u/6OCvVKyz1W+idaz" crossorigin="anonymous"></script>
7
+
<style>
8
+
nav { margin-bottom: 20px; border-bottom: 1px solid #ccc; padding: 10px 0; }
9
+
nav a { margin-right: 15px; }
10
+
table { width: 100%; border-collapse: collapse; }
11
+
th, td { text-align: left; padding: 8px; border: 1px solid #ddd; }
12
+
.pagination { margin-top: 20px; }
13
+
.filters { background: #f4f4f4; padding: 15px; margin-bottom: 20px; }
14
+
#notifications {
15
+
position: fixed;
16
+
bottom: 8px;
17
+
right: 8px;
18
+
z-index: 1000;
19
+
pointer-events: none;
20
+
}
21
+
.notif {
22
+
pointer-events: auto;
23
+
background: #333;
24
+
color: #fff;
25
+
padding: 2px 4px;
26
+
margin: 4px 0;
27
+
opacity: 0.95;
28
+
}
29
+
.notif.warn { background: #ed6c02 }
30
+
.notif.error { background: #d32f2f }
31
+
</style>
32
+
</head>
33
+
<body>
34
+
<nav>
35
+
<a href="/repos">Repositories</a>
36
+
<a href="/hosts">Knot Hosts</a>
37
+
</nav>
38
+
<main id="main">
39
+
{{template "content" .}}
40
+
</main>
41
+
<div id="notifications"></div>
42
+
<script>
43
+
document.body.addEventListener("htmx:oobBeforeSwap", (evt) => {
44
+
evt.detail.fragment.querySelectorAll(".notif").forEach((el) => {
45
+
console.debug("set timeout to notif element", el)
46
+
setTimeout(() => {
47
+
console.debug("clearing notif element", el);
48
+
el.remove();
49
+
}, 10 * 1000);
50
+
});
51
+
});
52
+
</script>
53
+
</body>
54
+
</html>
55
+
{{end}}
+44
knotmirror/templates/hosts.html
+44
knotmirror/templates/hosts.html
···
1
+
{{template "base" .}}
2
+
{{define "content"}}
3
+
<h2>Knot Hosts</h2>
4
+
5
+
<div class="filters">
6
+
<form
7
+
hx-get=""
8
+
hx-target="#table"
9
+
hx-select="#table"
10
+
hx-swap="outerHTML"
11
+
hx-trigger="every 10s"
12
+
>
13
+
<select name="status">
14
+
{{ range const.AllHostStatuses }}
15
+
<option value="{{.}}" {{ if eq $.FilterByStatus . }}selected{{end}}>{{.}}</option>
16
+
{{ end }}
17
+
</select>
18
+
<button type="submit">Filter</button>
19
+
</form>
20
+
</div>
21
+
22
+
<table id="table">
23
+
<thead>
24
+
<tr>
25
+
<th>Hostname</th>
26
+
<th>SSL</th>
27
+
<th>Status</th>
28
+
<th>Last Seq</th>
29
+
</tr>
30
+
</thead>
31
+
<tbody>
32
+
{{range .Hosts}}
33
+
<tr>
34
+
<td>{{.Hostname}}</td>
35
+
<td>{{if .NoSSL}}False{{else}}True{{end}}</td>
36
+
<td>{{.Status}}</td>
37
+
<td>{{.LastSeq}}</td>
38
+
</tr>
39
+
{{else}}
40
+
<tr><td colspan="4">No hosts registered.</td></tr>
41
+
{{end}}
42
+
</tbody>
43
+
</table>
44
+
{{end}}
+86
knotmirror/templates/repos.html
+86
knotmirror/templates/repos.html
···
1
+
{{template "base" .}}
2
+
{{define "content"}}
3
+
<h2>Repositories</h2>
4
+
5
+
<div class="filters">
6
+
<form
7
+
hx-get=""
8
+
hx-target="#table"
9
+
hx-select="#table"
10
+
hx-swap="outerHTML"
11
+
hx-trigger="every 10s"
12
+
>
13
+
<input type="text" name="did" placeholder="DID" value="{{.FilterByDid}}">
14
+
<input type="text" name="knot" placeholder="Knot Domain" value="{{.FilterByKnot}}">
15
+
<select name="state">
16
+
<option value="">-- State --</option>
17
+
{{ range const.AllRepoStates }}
18
+
<option value="{{.}}" {{ if eq $.FilterByState . }}selected{{end}}>{{.}}</option>
19
+
{{ end }}
20
+
</select>
21
+
<button type="submit">Filter</button>
22
+
<a href="/repos">Clear</a>
23
+
</form>
24
+
</div>
25
+
26
+
<div id="table">
27
+
<div class="repo-state-indicators">
28
+
{{range const.AllRepoStates}}
29
+
<span class="state-pill state-{{.}}">
30
+
{{.}}: {{index $.RepoCounts .}}
31
+
</span>
32
+
{{end}}
33
+
</div>
34
+
<table>
35
+
<thead>
36
+
<tr>
37
+
<th>DID</th>
38
+
<th>Name</th>
39
+
<th>Knot</th>
40
+
<th>State</th>
41
+
<th>Retry</th>
42
+
<th>Retry After</th>
43
+
<th>Error Message</th>
44
+
<th>Action</th>
45
+
</tr>
46
+
</thead>
47
+
<tbody>
48
+
{{range .Repos}}
49
+
<tr>
50
+
<td><code>{{.Did}}</code></td>
51
+
<td>{{.Name}}</td>
52
+
<td>{{.KnotDomain}}</td>
53
+
<td><strong>{{.State}}</strong></td>
54
+
<td>{{.RetryCount}}</td>
55
+
<td>{{readt .RetryAfter}}</td>
56
+
<td>{{.ErrorMsg}}</td>
57
+
<td>
58
+
<form
59
+
{{ if .State.IsResyncing -}}
60
+
hx-post="/api/cancelRepoResync"
61
+
{{- else -}}
62
+
hx-post="/api/triggerRepoResync"
63
+
{{- end }}
64
+
hx-swap="none"
65
+
hx-disabled-elt="find button"
66
+
>
67
+
<input type="hidden" name="repo" value="{{.AtUri}}">
68
+
<button type="submit">{{ if .State.IsResyncing }}cancel{{ else }}resync{{ end }}</button>
69
+
</form>
70
+
</td>
71
+
</tr>
72
+
{{else}}
73
+
<tr><td colspan="99">No repositories found.</td></tr>
74
+
{{end}}
75
+
</tbody>
76
+
</table>
77
+
</div>
78
+
79
+
<div class="pagination">
80
+
{{if gt .Page 1}}
81
+
<a href="?page={{sub .Page 1}}&did={{.FilterByDid}}&knot={{.FilterByKnot}}&state={{.FilterByState}}">« Previous</a>
82
+
{{end}}
83
+
<span>Page {{.Page}}</span>
84
+
<a href="?page={{add .Page 1}}&did={{.FilterByDid}}&knot={{.FilterByKnot}}&state={{.FilterByState}}">Next »</a>
85
+
</div>
86
+
{{end}}
+106
knotmirror/xrpc/git_getArchive.go
+106
knotmirror/xrpc/git_getArchive.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"compress/gzip"
5
+
"fmt"
6
+
"net/http"
7
+
"net/url"
8
+
"strings"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/go-git/go-git/v5/plumbing"
13
+
"tangled.org/core/api/tangled"
14
+
"tangled.org/core/knotmirror/db"
15
+
"tangled.org/core/knotserver/git"
16
+
)
17
+
18
+
func (x *Xrpc) GetArchive(w http.ResponseWriter, r *http.Request) {
19
+
var (
20
+
repoQuery = r.URL.Query().Get("repo")
21
+
ref = r.URL.Query().Get("ref")
22
+
format = r.URL.Query().Get("format")
23
+
prefix = r.URL.Query().Get("prefix")
24
+
)
25
+
26
+
repo, err := syntax.ParseATURI(repoQuery)
27
+
if err != nil || repo.RecordKey() == "" {
28
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
29
+
return
30
+
}
31
+
32
+
if format != "tar.gz" {
33
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "only tar.gz format is supported"})
34
+
return
35
+
}
36
+
if format == "" {
37
+
format = "tar.gz"
38
+
}
39
+
40
+
l := x.logger.With("repo", repo, "ref", ref, "format", format, "prefix", prefix)
41
+
ctx := r.Context()
42
+
43
+
repoPath, err := x.makeRepoPath(ctx, repo)
44
+
if err != nil {
45
+
l.Error("failed to resolve repo at-uri", "err", err)
46
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to resolve repo"})
47
+
return
48
+
}
49
+
50
+
gr, err := git.Open(repoPath, ref)
51
+
if err != nil {
52
+
l.Error("failed to open git repo", "err", err)
53
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to open git repo"})
54
+
return
55
+
}
56
+
57
+
repoName, err := func() (string, error) {
58
+
r, err := db.GetRepoByAtUri(ctx, x.db, repo)
59
+
if err != nil {
60
+
return "", err
61
+
}
62
+
if r == nil {
63
+
return "", fmt.Errorf("repo not found: %s", repo)
64
+
}
65
+
return r.Name, nil
66
+
}()
67
+
if err != nil {
68
+
l.Error("failed to get repo name", "err", err)
69
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to retrieve repo name"})
70
+
return
71
+
}
72
+
73
+
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
74
+
immutableLink := func() string {
75
+
params := url.Values{}
76
+
params.Set("repo", repo.String())
77
+
params.Set("ref", gr.Hash().String())
78
+
params.Set("format", format)
79
+
params.Set("prefix", prefix)
80
+
return fmt.Sprintf("%s/xrpc/%s?%s", x.cfg.BaseUrl(), tangled.GitTempGetArchiveNSID, params.Encode())
81
+
}()
82
+
83
+
filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename)
84
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
85
+
w.Header().Set("Content-Type", "application/gzip")
86
+
w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink))
87
+
88
+
gw := gzip.NewWriter(w)
89
+
defer gw.Close()
90
+
91
+
if err := gr.WriteTar(gw, prefix); err != nil {
92
+
// once we start writing to the body we can't report error anymore
93
+
// so we are only left with logging the error
94
+
l.Error("writing tar file", "err", err.Error())
95
+
w.WriteHeader(http.StatusInternalServerError)
96
+
return
97
+
}
98
+
99
+
if err := gw.Flush(); err != nil {
100
+
// once we start writing to the body we can't report error anymore
101
+
// so we are only left with logging the error
102
+
l.Error("flushing", "err", err.Error())
103
+
w.WriteHeader(http.StatusInternalServerError)
104
+
return
105
+
}
106
+
}
+86
knotmirror/xrpc/git_getBlob.go
+86
knotmirror/xrpc/git_getBlob.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"io"
7
+
"net/http"
8
+
"slices"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/go-git/go-git/v5/plumbing/object"
13
+
"tangled.org/core/knotserver/git"
14
+
)
15
+
16
+
func (x *Xrpc) GetBlob(w http.ResponseWriter, r *http.Request) {
17
+
var (
18
+
repoQuery = r.URL.Query().Get("repo")
19
+
ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this)
20
+
path = r.URL.Query().Get("path")
21
+
)
22
+
23
+
repo, err := syntax.ParseATURI(repoQuery)
24
+
if err != nil || repo.RecordKey() == "" {
25
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
26
+
return
27
+
}
28
+
29
+
l := x.logger.With("repo", repo, "ref", ref, "path", path)
30
+
31
+
if path == "" {
32
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing path parameter"})
33
+
return
34
+
}
35
+
36
+
file, err := x.getFile(r.Context(), repo, ref, path)
37
+
if err != nil {
38
+
// TODO: better error return
39
+
l.Error("failed to get blob", "err", err)
40
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get blob"})
41
+
return
42
+
}
43
+
44
+
reader, err := file.Reader()
45
+
if err != nil {
46
+
l.Error("failed to read blob", "err", err)
47
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"})
48
+
return
49
+
}
50
+
defer reader.Close()
51
+
52
+
w.Header().Set("Content-Type", "application/octet-stream")
53
+
if _, err := io.Copy(w, reader); err != nil {
54
+
l.Error("failed to serve the blob", "err", err)
55
+
}
56
+
}
57
+
58
+
func (x *Xrpc) getFile(ctx context.Context, repo syntax.ATURI, ref, path string) (*object.File, error) {
59
+
repoPath, err := x.makeRepoPath(ctx, repo)
60
+
if err != nil {
61
+
return nil, fmt.Errorf("resolving repo at-uri: %w", err)
62
+
}
63
+
64
+
gr, err := git.Open(repoPath, ref)
65
+
if err != nil {
66
+
return nil, fmt.Errorf("opening git repo: %w", err)
67
+
}
68
+
69
+
return gr.File(path)
70
+
}
71
+
72
+
var textualMimeTypes = []string{
73
+
"application/json",
74
+
"application/xml",
75
+
"application/yaml",
76
+
"application/x-yaml",
77
+
"application/toml",
78
+
"application/javascript",
79
+
"application/ecmascript",
80
+
}
81
+
82
+
// isTextualMimeType returns true if the MIME type represents textual content
83
+
// that should be served as text/plain for security reasons
84
+
func isTextualMimeType(mimeType string) bool {
85
+
return slices.Contains(textualMimeTypes, mimeType)
86
+
}
+85
knotmirror/xrpc/git_getBranch.go
+85
knotmirror/xrpc/git_getBranch.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
"net/url"
8
+
"time"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"tangled.org/core/api/tangled"
13
+
"tangled.org/core/knotserver/git"
14
+
)
15
+
16
+
// TODO: maybe rename to `sh.tangled.repo.temp.getCommit`?
17
+
// then, we should ensure the given `ref` is valid
18
+
func (x *Xrpc) GetBranch(w http.ResponseWriter, r *http.Request) {
19
+
var (
20
+
repoQuery = r.URL.Query().Get("repo")
21
+
nameQuery = r.URL.Query().Get("name")
22
+
)
23
+
24
+
repo, err := syntax.ParseATURI(repoQuery)
25
+
if err != nil || repo.RecordKey() == "" {
26
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
27
+
return
28
+
}
29
+
30
+
if nameQuery == "" {
31
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing name parameter"})
32
+
return
33
+
}
34
+
branchName, _ := url.PathUnescape(nameQuery)
35
+
36
+
l := x.logger.With("repo", repo, "branch", branchName)
37
+
38
+
out, err := x.getBranch(r.Context(), repo, branchName)
39
+
if err != nil {
40
+
// TODO: better error return
41
+
l.Error("failed to get branch", "err", err)
42
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get branch"})
43
+
return
44
+
}
45
+
writeJson(w, http.StatusOK, out)
46
+
}
47
+
48
+
func (x *Xrpc) getBranch(ctx context.Context, repo syntax.ATURI, branchName string) (*tangled.GitTempGetBranch_Output, error) {
49
+
repoPath, err := x.makeRepoPath(ctx, repo)
50
+
if err != nil {
51
+
return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err)
52
+
}
53
+
54
+
gr, err := git.PlainOpen(repoPath)
55
+
if err != nil {
56
+
return nil, fmt.Errorf("failed to open git repo: %w", err)
57
+
}
58
+
59
+
ref, err := gr.Branch(branchName)
60
+
if err != nil {
61
+
return nil, fmt.Errorf("getting branch '%s': %w", branchName, err)
62
+
}
63
+
64
+
commit, err := gr.Commit(ref.Hash())
65
+
if err != nil {
66
+
return nil, fmt.Errorf("getting commit '%s': %w", ref.Hash(), err)
67
+
}
68
+
69
+
out := tangled.GitTempGetBranch_Output{
70
+
Name: ref.Name().Short(),
71
+
Hash: ref.Hash().String(),
72
+
When: commit.Author.When.Format(time.RFC3339),
73
+
Author: &tangled.GitTempDefs_Signature{
74
+
Name: commit.Author.Name,
75
+
Email: commit.Author.Email,
76
+
When: commit.Author.When.Format(time.RFC3339),
77
+
},
78
+
}
79
+
80
+
if commit.Message != "" {
81
+
out.Message = &commit.Message
82
+
}
83
+
84
+
return &out, nil
85
+
}
+92
knotmirror/xrpc/git_getTag.go
+92
knotmirror/xrpc/git_getTag.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
8
+
"github.com/bluesky-social/indigo/atproto/atclient"
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"github.com/go-git/go-git/v5/plumbing"
11
+
"github.com/go-git/go-git/v5/plumbing/object"
12
+
"tangled.org/core/knotserver/git"
13
+
"tangled.org/core/types"
14
+
)
15
+
16
+
func (x *Xrpc) GetTag(w http.ResponseWriter, r *http.Request) {
17
+
var (
18
+
repoQuery = r.URL.Query().Get("repo")
19
+
tagName = r.URL.Query().Get("tag")
20
+
)
21
+
22
+
repo, err := syntax.ParseATURI(repoQuery)
23
+
if err != nil || repo.RecordKey() == "" {
24
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
25
+
return
26
+
}
27
+
28
+
if tagName == "" {
29
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing 'tag' parameter"})
30
+
return
31
+
}
32
+
33
+
l := x.logger.With("repo", repo, "tag", tagName)
34
+
35
+
out, err := x.getTag(r.Context(), repo, tagName)
36
+
if err != nil {
37
+
// TODO: better error return
38
+
l.Error("failed to get tag", "err", err)
39
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tag"})
40
+
return
41
+
}
42
+
writeJson(w, http.StatusOK, out)
43
+
}
44
+
45
+
func (x *Xrpc) getTag(ctx context.Context, repo syntax.ATURI, tagName string) (*types.RepoTagResponse, error) {
46
+
repoPath, err := x.makeRepoPath(ctx, repo)
47
+
if err != nil {
48
+
return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err)
49
+
}
50
+
51
+
gr, err := git.PlainOpen(repoPath)
52
+
if err != nil {
53
+
return nil, fmt.Errorf("failed to open git repo: %w", err)
54
+
}
55
+
56
+
// if this is not already formatted as refs/tags/v0.1.0, then format it
57
+
if !plumbing.ReferenceName(tagName).IsTag() {
58
+
tagName = plumbing.NewTagReferenceName(tagName).String()
59
+
}
60
+
61
+
tag, err := func() (object.Tag, error) {
62
+
tags, err := gr.Tags(&git.TagsOptions{
63
+
Pattern: tagName,
64
+
})
65
+
if err != nil {
66
+
return object.Tag{}, err
67
+
}
68
+
if len(tags) != 1 {
69
+
return object.Tag{}, fmt.Errorf("expected 1 tag to be returned, got %d tags", len(tags))
70
+
}
71
+
return tags[0], nil
72
+
}()
73
+
if err != nil {
74
+
return nil, fmt.Errorf("getting tag: %w", err)
75
+
}
76
+
77
+
var target *object.Tag
78
+
if tag.Target != plumbing.ZeroHash {
79
+
target = &tag
80
+
}
81
+
82
+
return &types.RepoTagResponse{
83
+
Tag: &types.TagReference{
84
+
Tag: target,
85
+
Reference: types.Reference{
86
+
Name: tag.Name,
87
+
Hash: tag.Hash.String(),
88
+
},
89
+
Message: tag.Message,
90
+
},
91
+
}, nil
92
+
}
+118
knotmirror/xrpc/git_getTree.go
+118
knotmirror/xrpc/git_getTree.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
"path/filepath"
8
+
"time"
9
+
"unicode/utf8"
10
+
11
+
"github.com/bluesky-social/indigo/atproto/atclient"
12
+
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"tangled.org/core/api/tangled"
14
+
"tangled.org/core/appview/pages/markup"
15
+
"tangled.org/core/knotserver/git"
16
+
)
17
+
18
+
func (x *Xrpc) GetTree(w http.ResponseWriter, r *http.Request) {
19
+
var (
20
+
repoQuery = r.URL.Query().Get("repo")
21
+
ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this)
22
+
path = r.URL.Query().Get("path") // path can be empty (defaults to root)
23
+
)
24
+
25
+
repo, err := syntax.ParseATURI(repoQuery)
26
+
if err != nil || repo.RecordKey() == "" {
27
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
28
+
return
29
+
}
30
+
31
+
l := x.logger.With("repo", repo, "ref", ref, "path", path)
32
+
33
+
out, err := x.getTree(r.Context(), repo, ref, path)
34
+
if err != nil {
35
+
// TODO: better error return
36
+
l.Error("failed to get tree", "err", err)
37
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tree"})
38
+
return
39
+
}
40
+
writeJson(w, http.StatusOK, out)
41
+
}
42
+
43
+
func (x *Xrpc) getTree(ctx context.Context, repo syntax.ATURI, ref, path string) (*tangled.GitTempGetTree_Output, error) {
44
+
repoPath, err := x.makeRepoPath(ctx, repo)
45
+
if err != nil {
46
+
return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err)
47
+
}
48
+
49
+
gr, err := git.Open(repoPath, ref)
50
+
if err != nil {
51
+
return nil, fmt.Errorf("opening git repo: %w", err)
52
+
}
53
+
54
+
files, err := gr.FileTree(ctx, path)
55
+
if err != nil {
56
+
return nil, fmt.Errorf("reading file tree: %w", err)
57
+
}
58
+
59
+
// if any of these files are a readme candidate, pass along its blob contents too
60
+
var readmeFileName string
61
+
var readmeContents string
62
+
for _, file := range files {
63
+
if markup.IsReadmeFile(file.Name) {
64
+
contents, err := gr.RawContent(filepath.Join(path, file.Name))
65
+
if err != nil {
66
+
x.logger.Error("failed to read contents of file", "path", path, "file", file.Name)
67
+
}
68
+
69
+
if utf8.Valid(contents) {
70
+
readmeFileName = file.Name
71
+
readmeContents = string(contents)
72
+
break
73
+
}
74
+
}
75
+
}
76
+
77
+
// convert NiceTree -> tangled.RepoTempGetTree_TreeEntry
78
+
treeEntries := make([]*tangled.GitTempGetTree_TreeEntry, len(files))
79
+
for i, file := range files {
80
+
entry := &tangled.GitTempGetTree_TreeEntry{
81
+
Name: file.Name,
82
+
Mode: file.Mode,
83
+
Size: file.Size,
84
+
}
85
+
if file.LastCommit != nil {
86
+
entry.Last_commit = &tangled.GitTempGetTree_LastCommit{
87
+
Hash: file.LastCommit.Hash.String(),
88
+
Message: file.LastCommit.Message,
89
+
When: file.LastCommit.When.Format(time.RFC3339),
90
+
}
91
+
}
92
+
treeEntries[i] = entry
93
+
}
94
+
95
+
var parentPtr *string
96
+
if path != "" {
97
+
parentPtr = &path
98
+
}
99
+
100
+
var dotdotPtr *string
101
+
if path != "" {
102
+
dotdot := filepath.Dir(path)
103
+
if dotdot != "." {
104
+
dotdotPtr = &dotdot
105
+
}
106
+
}
107
+
108
+
return &tangled.GitTempGetTree_Output{
109
+
Ref: ref,
110
+
Parent: parentPtr,
111
+
Dotdot: dotdotPtr,
112
+
Files: treeEntries,
113
+
Readme: &tangled.GitTempGetTree_Readme{
114
+
Filename: readmeFileName,
115
+
Contents: readmeContents,
116
+
},
117
+
}, nil
118
+
}
+95
knotmirror/xrpc/git_listBranches.go
+95
knotmirror/xrpc/git_listBranches.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
"path/filepath"
8
+
"strconv"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"tangled.org/core/knotserver/git"
13
+
"tangled.org/core/types"
14
+
)
15
+
16
+
func (x *Xrpc) ListBranches(w http.ResponseWriter, r *http.Request) {
17
+
var (
18
+
repoQuery = r.URL.Query().Get("repo")
19
+
limitQuery = r.URL.Query().Get("limit")
20
+
cursorQuery = r.URL.Query().Get("cursor")
21
+
)
22
+
23
+
repo, err := syntax.ParseATURI(repoQuery)
24
+
if err != nil || repo.RecordKey() == "" {
25
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
26
+
return
27
+
}
28
+
29
+
limit := 50
30
+
if limitQuery != "" {
31
+
limit, err = strconv.Atoi(limitQuery)
32
+
if err != nil || limit < 1 || limit > 1000 {
33
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)})
34
+
return
35
+
}
36
+
}
37
+
38
+
var cursor int64
39
+
if cursorQuery != "" {
40
+
cursor, err = strconv.ParseInt(cursorQuery, 10, 64)
41
+
if err != nil || cursor < 0 {
42
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)})
43
+
return
44
+
}
45
+
}
46
+
47
+
l := x.logger.With("repo", repoQuery, "limit", limit, "cursor", cursor)
48
+
49
+
out, err := x.listBranches(r.Context(), repo, limit, cursor)
50
+
if err != nil {
51
+
// TODO: better error return
52
+
l.Error("failed to list branches", "err", err)
53
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list branches"})
54
+
return
55
+
}
56
+
writeJson(w, http.StatusOK, out)
57
+
}
58
+
59
+
func (x *Xrpc) listBranches(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoBranchesResponse, error) {
60
+
repoPath, err := x.makeRepoPath(ctx, repo)
61
+
if err != nil {
62
+
return nil, fmt.Errorf("resolving repo at-uri: %w", err)
63
+
}
64
+
65
+
gr, err := git.PlainOpen(repoPath)
66
+
if err != nil {
67
+
return nil, fmt.Errorf("opening git repo: %w", err)
68
+
}
69
+
70
+
branches, err := gr.Branches(&git.BranchesOptions{
71
+
Limit: limit,
72
+
Offset: int(cursor),
73
+
})
74
+
if err != nil {
75
+
return nil, fmt.Errorf("listing git branches: %w", err)
76
+
}
77
+
78
+
return &types.RepoBranchesResponse{
79
+
// TODO: include default branch and cursor
80
+
Branches: branches,
81
+
}, nil
82
+
}
83
+
84
+
func (x *Xrpc) makeRepoPath(ctx context.Context, repo syntax.ATURI) (string, error) {
85
+
id, err := x.resolver.ResolveIdent(ctx, repo.Authority().String())
86
+
if err != nil {
87
+
return "", err
88
+
}
89
+
90
+
return filepath.Join(
91
+
x.cfg.GitRepoBasePath,
92
+
id.DID.String(),
93
+
repo.RecordKey().String(),
94
+
), nil
95
+
}
+95
knotmirror/xrpc/git_listCommits.go
+95
knotmirror/xrpc/git_listCommits.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
"strconv"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/atclient"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
"tangled.org/core/knotserver/git"
12
+
"tangled.org/core/types"
13
+
)
14
+
15
+
func (x *Xrpc) ListCommits(w http.ResponseWriter, r *http.Request) {
16
+
var (
17
+
repoQuery = r.URL.Query().Get("repo")
18
+
ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this)
19
+
limitQuery = r.URL.Query().Get("limit")
20
+
cursorQuery = r.URL.Query().Get("cursor")
21
+
)
22
+
23
+
repo, err := syntax.ParseATURI(repoQuery)
24
+
if err != nil || repo.RecordKey() == "" {
25
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
26
+
return
27
+
}
28
+
29
+
limit := 50
30
+
if limitQuery != "" {
31
+
limit, err = strconv.Atoi(limitQuery)
32
+
if err != nil || limit < 1 || limit > 1000 {
33
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)})
34
+
return
35
+
}
36
+
}
37
+
38
+
var cursor int64
39
+
if cursorQuery != "" {
40
+
cursor, err = strconv.ParseInt(cursorQuery, 10, 64)
41
+
if err != nil || cursor < 0 {
42
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)})
43
+
return
44
+
}
45
+
}
46
+
47
+
l := x.logger.With("repo", repo, "ref", ref)
48
+
49
+
out, err := x.listCommits(r.Context(), repo, ref, limit, cursor)
50
+
if err != nil {
51
+
// TODO: better error return
52
+
l.Error("failed to list commits", "err", err)
53
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list commits"})
54
+
return
55
+
}
56
+
writeJson(w, http.StatusOK, out)
57
+
}
58
+
59
+
func (x *Xrpc) listCommits(ctx context.Context, repo syntax.ATURI, ref string, limit int, cursor int64) (*types.RepoLogResponse, error) {
60
+
repoPath, err := x.makeRepoPath(ctx, repo)
61
+
if err != nil {
62
+
return nil, fmt.Errorf("resolving repo at-uri: %w", err)
63
+
}
64
+
65
+
gr, err := git.Open(repoPath, ref)
66
+
if err != nil {
67
+
return nil, fmt.Errorf("opening git repo: %w", err)
68
+
}
69
+
70
+
offset := int(cursor)
71
+
72
+
commits, err := gr.Commits(offset, limit)
73
+
if err != nil {
74
+
return nil, fmt.Errorf("listing git commits: %w", err)
75
+
}
76
+
77
+
tcommits := make([]types.Commit, len(commits))
78
+
for i, c := range commits {
79
+
tcommits[i].FromGoGitCommit(c)
80
+
}
81
+
82
+
total, err := gr.TotalCommits()
83
+
if err != nil {
84
+
return nil, fmt.Errorf("counting total commits: %w", err)
85
+
}
86
+
87
+
return &types.RepoLogResponse{
88
+
Commits: tcommits,
89
+
Ref: ref,
90
+
Page: (offset / limit) + 1,
91
+
PerPage: limit,
92
+
Total: total,
93
+
Log: true,
94
+
}, nil
95
+
}
+86
knotmirror/xrpc/git_listLanguages.go
+86
knotmirror/xrpc/git_listLanguages.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"math"
7
+
"net/http"
8
+
"time"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"tangled.org/core/api/tangled"
13
+
"tangled.org/core/knotserver/git"
14
+
)
15
+
16
+
func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) {
17
+
var (
18
+
repoQuery = r.URL.Query().Get("repo")
19
+
ref = r.URL.Query().Get("ref")
20
+
)
21
+
l := x.logger.With("repo", repoQuery, "ref", ref)
22
+
23
+
repo, err := syntax.ParseATURI(repoQuery)
24
+
if err != nil || repo.RecordKey() == "" {
25
+
l.Error("invalid repo at-uri", "err", err)
26
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
27
+
return
28
+
}
29
+
30
+
out, err := x.listLanguages(r.Context(), repo, ref)
31
+
if err != nil {
32
+
l.Error("failed to list languages", "err", err)
33
+
writeErr(w, err)
34
+
return
35
+
}
36
+
37
+
writeJson(w, http.StatusOK, out)
38
+
}
39
+
40
+
func (x *Xrpc) listLanguages(ctx context.Context, repo syntax.ATURI, ref string) (*tangled.GitTempListLanguages_Output, error) {
41
+
repoPath, err := x.makeRepoPath(ctx, repo)
42
+
if err != nil {
43
+
return nil, fmt.Errorf("resolving repo at-uri: %w", err)
44
+
}
45
+
46
+
gr, err := git.Open(repoPath, ref)
47
+
if err != nil {
48
+
return nil, &atclient.APIError{StatusCode: http.StatusNotFound, Name: "RepoNotFound", Message: "failed to find git repo"}
49
+
}
50
+
51
+
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
52
+
defer cancel()
53
+
54
+
sizes, err := gr.AnalyzeLanguages(ctx)
55
+
if err != nil {
56
+
return nil, fmt.Errorf("analyzing languages: %w", err)
57
+
}
58
+
59
+
return &tangled.GitTempListLanguages_Output{
60
+
Ref: ref,
61
+
Languages: sizesToLanguages(sizes),
62
+
}, nil
63
+
}
64
+
65
+
func sizesToLanguages(sizes git.LangBreakdown) []*tangled.GitTempListLanguages_Language {
66
+
var apiLanguages []*tangled.GitTempListLanguages_Language
67
+
var totalSize int64
68
+
for _, size := range sizes {
69
+
totalSize += size
70
+
}
71
+
72
+
for name, size := range sizes {
73
+
percentagef64 := float64(size) / float64(totalSize) * 100
74
+
percentage := math.Round(percentagef64)
75
+
76
+
lang := &tangled.GitTempListLanguages_Language{
77
+
Name: name,
78
+
Size: size,
79
+
Percentage: int64(percentage),
80
+
}
81
+
82
+
apiLanguages = append(apiLanguages, lang)
83
+
}
84
+
85
+
return apiLanguages
86
+
}
+98
knotmirror/xrpc/git_listTags.go
+98
knotmirror/xrpc/git_listTags.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net/http"
7
+
"strconv"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/atclient"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
"github.com/go-git/go-git/v5/plumbing"
12
+
"github.com/go-git/go-git/v5/plumbing/object"
13
+
"tangled.org/core/knotserver/git"
14
+
"tangled.org/core/types"
15
+
)
16
+
17
+
func (x *Xrpc) ListTags(w http.ResponseWriter, r *http.Request) {
18
+
var (
19
+
repoQuery = r.URL.Query().Get("repo")
20
+
limitQuery = r.URL.Query().Get("limit")
21
+
cursorQuery = r.URL.Query().Get("cursor")
22
+
)
23
+
24
+
repo, err := syntax.ParseATURI(repoQuery)
25
+
if err != nil || repo.RecordKey() == "" {
26
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
27
+
return
28
+
}
29
+
30
+
limit := 50
31
+
if limitQuery != "" {
32
+
limit, err = strconv.Atoi(limitQuery)
33
+
if err != nil || limit < 1 || limit > 1000 {
34
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)})
35
+
return
36
+
}
37
+
}
38
+
39
+
var cursor int64
40
+
if cursorQuery != "" {
41
+
cursor, err = strconv.ParseInt(cursorQuery, 10, 64)
42
+
if err != nil || cursor < 0 {
43
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)})
44
+
return
45
+
}
46
+
}
47
+
48
+
l := x.logger.With("repo", repo, "limit", limit, "cursor", cursor)
49
+
50
+
out, err := x.listTags(r.Context(), repo, limit, cursor)
51
+
if err != nil {
52
+
// TODO: better error return
53
+
l.Error("failed to list tags", "err", err)
54
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list tags"})
55
+
return
56
+
}
57
+
writeJson(w, http.StatusOK, out)
58
+
}
59
+
60
+
func (x *Xrpc) listTags(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoTagsResponse, error) {
61
+
repoPath, err := x.makeRepoPath(ctx, repo)
62
+
if err != nil {
63
+
return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err)
64
+
}
65
+
66
+
gr, err := git.PlainOpen(repoPath)
67
+
if err != nil {
68
+
return nil, fmt.Errorf("failed to open git repo: %w", err)
69
+
}
70
+
71
+
tags, err := gr.Tags(&git.TagsOptions{
72
+
Limit: limit,
73
+
Offset: int(cursor),
74
+
})
75
+
if err != nil {
76
+
return nil, fmt.Errorf("failed to get git tags: %w", err)
77
+
}
78
+
79
+
rtags := make([]*types.TagReference, len(tags))
80
+
for i, tag := range tags {
81
+
var target *object.Tag
82
+
if tag.Target != plumbing.ZeroHash {
83
+
target = &tag
84
+
}
85
+
rtags[i] = &types.TagReference{
86
+
Reference: types.Reference{
87
+
Name: tag.Name,
88
+
Hash: tag.Hash.String(),
89
+
},
90
+
Tag: target,
91
+
Message: tag.Message,
92
+
}
93
+
}
94
+
95
+
return &types.RepoTagsResponse{
96
+
Tags: rtags,
97
+
}, nil
98
+
}
+69
knotmirror/xrpc/xrpc.go
+69
knotmirror/xrpc/xrpc.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"database/sql"
5
+
"encoding/json"
6
+
"errors"
7
+
"log/slog"
8
+
"net/http"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/go-chi/chi/v5"
12
+
"tangled.org/core/api/tangled"
13
+
"tangled.org/core/idresolver"
14
+
"tangled.org/core/knotmirror/config"
15
+
"tangled.org/core/log"
16
+
)
17
+
18
+
type Xrpc struct {
19
+
cfg *config.Config
20
+
db *sql.DB
21
+
resolver *idresolver.Resolver
22
+
logger *slog.Logger
23
+
}
24
+
25
+
func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver) *Xrpc {
26
+
return &Xrpc{
27
+
cfg,
28
+
db,
29
+
resolver,
30
+
log.SubLogger(logger, "xrpc"),
31
+
}
32
+
}
33
+
34
+
func (x *Xrpc) Router() http.Handler {
35
+
r := chi.NewRouter()
36
+
37
+
r.Get("/"+tangled.GitTempGetArchiveNSID, x.GetArchive)
38
+
r.Get("/"+tangled.GitTempGetBlobNSID, x.GetBlob)
39
+
r.Get("/"+tangled.GitTempGetBranchNSID, x.GetBranch)
40
+
// r.Get("/"+tangled.GitTempGetCommitNSID, x.GetCommit) // todo
41
+
// r.Get("/"+tangled.GitTempGetDiffNSID, x.GetDiff) // todo
42
+
// r.Get("/"+tangled.GitTempGetEntityNSID, x.GetEntity) // todo
43
+
// r.Get("/"+tangled.GitTempGetHeadNSID, x.GetHead) // todo
44
+
r.Get("/"+tangled.GitTempGetTagNSID, x.GetTag) // using types.Response
45
+
r.Get("/"+tangled.GitTempGetTreeNSID, x.GetTree)
46
+
r.Get("/"+tangled.GitTempListBranchesNSID, x.ListBranches) // wip, unknown output
47
+
r.Get("/"+tangled.GitTempListCommitsNSID, x.ListCommits)
48
+
r.Get("/"+tangled.GitTempListLanguagesNSID, x.ListLanguages)
49
+
r.Get("/"+tangled.GitTempListTagsNSID, x.ListTags)
50
+
51
+
return r
52
+
}
53
+
54
+
func writeJson(w http.ResponseWriter, status int, response any) error {
55
+
w.Header().Set("Content-Type", "application/json")
56
+
w.WriteHeader(status)
57
+
if err := json.NewEncoder(w).Encode(response); err != nil {
58
+
return err
59
+
}
60
+
return nil
61
+
}
62
+
63
+
func writeErr(w http.ResponseWriter, err error) error {
64
+
var apiErr *atclient.APIError
65
+
if errors.As(err, &apiErr) {
66
+
return writeJson(w, apiErr.StatusCode, atclient.ErrorBody{Name: apiErr.Name, Message: apiErr.Message})
67
+
}
68
+
return writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "internal server error"})
69
+
}
+14
knotserver/git/git.go
+14
knotserver/git/git.go
···
199
199
return io.ReadAll(reader)
200
200
}
201
201
202
+
func (g *GitRepo) File(path string) (*object.File, error) {
203
+
c, err := g.r.CommitObject(g.h)
204
+
if err != nil {
205
+
return nil, fmt.Errorf("commit object: %w", err)
206
+
}
207
+
208
+
tree, err := c.Tree()
209
+
if err != nil {
210
+
return nil, fmt.Errorf("file tree: %w", err)
211
+
}
212
+
213
+
return tree.File(path)
214
+
}
215
+
202
216
// read and parse .gitmodules
203
217
func (g *GitRepo) Submodules() (*config.Modules, error) {
204
218
c, err := g.r.CommitObject(g.h)
+3
-2
knotserver/router.go
+3
-2
knotserver/router.go
···
83
83
84
84
r.Route("/{did}", func(r chi.Router) {
85
85
r.Use(h.resolveDidRedirect)
86
-
r.Use(h.resolveRepo)
87
86
r.Route("/{name}", func(r chi.Router) {
87
+
r.Use(h.resolveRepo)
88
+
88
89
// routes for git operations
89
90
r.Get("/info/refs", h.InfoRefs)
90
91
r.Post("/git-upload-archive", h.UploadArchive)
···
176
177
return
177
178
}
178
179
180
+
ctx := context.WithValue(r.Context(), ctxRepoPathKey{}, repoPath)
179
-
ctx := context.WithValue(r.Context(), "repoPath", repoPath)
180
181
next.ServeHTTP(w, r.WithContext(ctx))
181
182
})
182
183
}
+64
lexicons/git/temp/analyzeMerge.json
+64
lexicons/git/temp/analyzeMerge.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.analyzeMerge",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "Check if a merge is possible between two branches",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": ["repo", "patch", "branch"],
11
+
"properties": {
12
+
"repo": {
13
+
"type": "string",
14
+
"format": "at-uri",
15
+
"description": "AT-URI of the repository"
16
+
},
17
+
"patch": {
18
+
"type": "string",
19
+
"description": "Patch or pull request to check for merge conflicts"
20
+
},
21
+
"branch": {
22
+
"type": "string",
23
+
"description": "Target branch to merge into"
24
+
}
25
+
}
26
+
},
27
+
"output": {
28
+
"encoding": "application/json",
29
+
"schema": {
30
+
"type": "object",
31
+
"required": ["is_conflicted"],
32
+
"properties": {
33
+
"is_conflicted": {
34
+
"type": "boolean",
35
+
"description": "Whether the merge has conflicts"
36
+
},
37
+
"conflicts": {
38
+
"type": "array",
39
+
"description": "List of files with merge conflicts",
40
+
"items": {
41
+
"type": "ref",
42
+
"ref": "#conflictInfo"
43
+
}
44
+
}
45
+
}
46
+
}
47
+
}
48
+
},
49
+
"conflictInfo": {
50
+
"type": "object",
51
+
"required": ["filename", "reason"],
52
+
"properties": {
53
+
"filename": {
54
+
"type": "string",
55
+
"description": "Name of the conflicted file"
56
+
},
57
+
"reason": {
58
+
"type": "string",
59
+
"description": "Reason for the conflict"
60
+
}
61
+
}
62
+
}
63
+
}
64
+
}
+112
lexicons/git/temp/defs.json
+112
lexicons/git/temp/defs.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.defs",
4
+
"defs": {
5
+
"blob": {
6
+
"type": "object",
7
+
"description": "blob metadata. This object doesn't include the blob content",
8
+
"required": ["name", "mode", "size", "lastCommit"],
9
+
"properties": {
10
+
"name": {
11
+
"type": "string",
12
+
"description": "The file name"
13
+
},
14
+
"mode": {
15
+
"type": "string"
16
+
},
17
+
"size": {
18
+
"type": "integer",
19
+
"description": "File size in bytes"
20
+
},
21
+
"lastCommit": {
22
+
"type": "ref",
23
+
"ref": "#commit"
24
+
},
25
+
"submodule": {
26
+
"type": "ref",
27
+
"ref": "#submodule",
28
+
"description": "Submodule information if path is a submodule"
29
+
}
30
+
}
31
+
},
32
+
"branch": {
33
+
"type": "object",
34
+
"required": ["name", "commit"],
35
+
"properties": {
36
+
"name": {
37
+
"type": "string",
38
+
"description": "branch name"
39
+
},
40
+
"commit": {
41
+
"type": "ref",
42
+
"ref": "#commit",
43
+
"description": "hydrated commit object"
44
+
}
45
+
}
46
+
},
47
+
"tag": {
48
+
"type": "object",
49
+
"required": ["name", "tagger", "target"],
50
+
"properties": {
51
+
"name": {
52
+
"type": "string",
53
+
"description": "tag name"
54
+
},
55
+
"tagger": { "type": "ref", "ref": "#signature" },
56
+
"message": { "type": "string" },
57
+
"target": { "type": "unknown" }
58
+
}
59
+
},
60
+
"commit": {
61
+
"type": "object",
62
+
"required": ["hash", "author", "committer", "message", "tree"],
63
+
"properties": {
64
+
"hash": { "type": "ref", "ref": "#hash" },
65
+
"author": { "type": "ref", "ref": "#signature" },
66
+
"committer": { "type": "ref", "ref": "#signature" },
67
+
"message": { "type": "string" },
68
+
"tree": { "type": "ref", "ref": "#hash" }
69
+
}
70
+
},
71
+
"hash": {
72
+
"type": "string"
73
+
},
74
+
"signature": {
75
+
"type": "object",
76
+
"required": ["name", "email", "when"],
77
+
"properties": {
78
+
"name": {
79
+
"type": "string",
80
+
"description": "Person name"
81
+
},
82
+
"email": {
83
+
"type": "string",
84
+
"description": "Person email"
85
+
},
86
+
"when": {
87
+
"type": "string",
88
+
"format": "datetime",
89
+
"description": "Timestamp of the signature"
90
+
}
91
+
}
92
+
},
93
+
"submodule": {
94
+
"type": "object",
95
+
"required": ["name", "url"],
96
+
"properties": {
97
+
"name": {
98
+
"type": "string",
99
+
"description": "Submodule name"
100
+
},
101
+
"url": {
102
+
"type": "string",
103
+
"description": "Submodule repository URL"
104
+
},
105
+
"branch": {
106
+
"type": "string",
107
+
"description": "Branch to track in the submodule"
108
+
}
109
+
}
110
+
}
111
+
}
112
+
}
+56
lexicons/git/temp/getArchive.json
+56
lexicons/git/temp/getArchive.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getArchive",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo", "ref"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"ref": {
17
+
"type": "string",
18
+
"description": "Git reference (branch, tag, or commit SHA)"
19
+
},
20
+
"format": {
21
+
"type": "string",
22
+
"description": "Archive format",
23
+
"enum": ["tar", "zip", "tar.gz", "tar.bz2", "tar.xz"],
24
+
"default": "tar.gz"
25
+
},
26
+
"prefix": {
27
+
"type": "string",
28
+
"description": "Prefix for files in the archive"
29
+
}
30
+
}
31
+
},
32
+
"output": {
33
+
"encoding": "*/*",
34
+
"description": "Binary archive data"
35
+
},
36
+
"errors": [
37
+
{
38
+
"name": "RepoNotFound",
39
+
"description": "Repository not found or access denied"
40
+
},
41
+
{
42
+
"name": "RefNotFound",
43
+
"description": "Git reference not found"
44
+
},
45
+
{
46
+
"name": "InvalidRequest",
47
+
"description": "Invalid request parameters"
48
+
},
49
+
{
50
+
"name": "ArchiveError",
51
+
"description": "Failed to create archive"
52
+
}
53
+
]
54
+
}
55
+
}
56
+
}
+47
lexicons/git/temp/getBlob.json
+47
lexicons/git/temp/getBlob.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getBlob",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo", "path"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"ref": {
17
+
"type": "string",
18
+
"description": "Git reference (branch, tag, or commit SHA)",
19
+
"default": "HEAD"
20
+
},
21
+
"path": {
22
+
"type": "string",
23
+
"description": "Path within the repository tree"
24
+
}
25
+
}
26
+
},
27
+
"output": {
28
+
"encoding": "*/*",
29
+
"description": "raw blob served in octet-stream"
30
+
},
31
+
"errors": [
32
+
{
33
+
"name": "RepoNotFound",
34
+
"description": "Repository not found or access denied"
35
+
},
36
+
{
37
+
"name": "BlobNotFound",
38
+
"description": "Blob not found"
39
+
},
40
+
{
41
+
"name": "InvalidRequest",
42
+
"description": "Invalid request parameters"
43
+
}
44
+
]
45
+
}
46
+
}
47
+
}
+68
lexicons/git/temp/getBranch.json
+68
lexicons/git/temp/getBranch.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getBranch",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo", "name"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"name": {
17
+
"type": "string",
18
+
"description": "Branch name to get information for"
19
+
}
20
+
}
21
+
},
22
+
"output": {
23
+
"encoding": "application/json",
24
+
"schema": {
25
+
"type": "object",
26
+
"required": ["name", "hash", "when"],
27
+
"properties": {
28
+
"name": {
29
+
"type": "string",
30
+
"description": "Branch name"
31
+
},
32
+
"hash": {
33
+
"type": "string",
34
+
"description": "Latest commit hash on this branch"
35
+
},
36
+
"when": {
37
+
"type": "string",
38
+
"format": "datetime",
39
+
"description": "Timestamp of latest commit"
40
+
},
41
+
"message": {
42
+
"type": "string",
43
+
"description": "Latest commit message"
44
+
},
45
+
"author": {
46
+
"type": "ref",
47
+
"ref": "sh.tangled.git.temp.defs#signature"
48
+
}
49
+
}
50
+
}
51
+
},
52
+
"errors": [
53
+
{
54
+
"name": "RepoNotFound",
55
+
"description": "Repository not found or access denied"
56
+
},
57
+
{
58
+
"name": "BranchNotFound",
59
+
"description": "Branch not found"
60
+
},
61
+
{
62
+
"name": "InvalidRequest",
63
+
"description": "Invalid request parameters"
64
+
}
65
+
]
66
+
}
67
+
}
68
+
}
+46
lexicons/git/temp/getCommit.json
+46
lexicons/git/temp/getCommit.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getCommit",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "resolve commit from given ref",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": ["repo", "ref"],
11
+
"properties": {
12
+
"repo": {
13
+
"type": "string",
14
+
"format": "at-uri",
15
+
"description": "AT-URI of the repository"
16
+
},
17
+
"ref": {
18
+
"type": "string",
19
+
"description": "reference name to resolve"
20
+
}
21
+
}
22
+
},
23
+
"output": {
24
+
"encoding": "application/json",
25
+
"schema": {
26
+
"type": "ref",
27
+
"ref": "sh.tangled.git.temp.defs#commit"
28
+
}
29
+
},
30
+
"errors": [
31
+
{
32
+
"name": "RepoNotFound",
33
+
"description": "Repository not found or access denied"
34
+
},
35
+
{
36
+
"name": "CommitNotFound",
37
+
"description": "Commit not found"
38
+
},
39
+
{
40
+
"name": "InvalidRequest",
41
+
"description": "Invalid request parameters"
42
+
}
43
+
]
44
+
}
45
+
}
46
+
}
+50
lexicons/git/temp/getDiff.json
+50
lexicons/git/temp/getDiff.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getDiff",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo", "rev1", "rev2"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"rev1": {
17
+
"type": "string",
18
+
"description": "First revision (commit, branch, or tag)"
19
+
},
20
+
"rev2": {
21
+
"type": "string",
22
+
"description": "Second revision (commit, branch, or tag)"
23
+
}
24
+
}
25
+
},
26
+
"output": {
27
+
"encoding": "*/*",
28
+
"description": "Compare output in application/json"
29
+
},
30
+
"errors": [
31
+
{
32
+
"name": "RepoNotFound",
33
+
"description": "Repository not found or access denied"
34
+
},
35
+
{
36
+
"name": "RevisionNotFound",
37
+
"description": "One or both revisions not found"
38
+
},
39
+
{
40
+
"name": "InvalidRequest",
41
+
"description": "Invalid request parameters"
42
+
},
43
+
{
44
+
"name": "CompareError",
45
+
"description": "Failed to compare revisions"
46
+
}
47
+
]
48
+
}
49
+
}
50
+
}
+51
lexicons/git/temp/getEntity.json
+51
lexicons/git/temp/getEntity.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getEntity",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "get metadata of blob by ref and path",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": ["repo", "path"],
11
+
"properties": {
12
+
"repo": {
13
+
"type": "string",
14
+
"format": "at-uri",
15
+
"description": "AT-URI of the repository"
16
+
},
17
+
"ref": {
18
+
"type": "string",
19
+
"description": "Git reference (branch, tag, or commit SHA)",
20
+
"default": "HEAD"
21
+
},
22
+
"path": {
23
+
"type": "string",
24
+
"description": "path of the entity"
25
+
}
26
+
}
27
+
},
28
+
"output": {
29
+
"encoding": "application/json",
30
+
"schema": {
31
+
"type": "ref",
32
+
"ref": "sh.tangled.git.temp.defs#blob"
33
+
}
34
+
},
35
+
"errors": [
36
+
{
37
+
"name": "RepoNotFound",
38
+
"description": "Repository not found or access denied"
39
+
},
40
+
{
41
+
"name": "BlobNotFound",
42
+
"description": "Blob not found"
43
+
},
44
+
{
45
+
"name": "InvalidRequest",
46
+
"description": "Invalid request parameters"
47
+
}
48
+
]
49
+
}
50
+
}
51
+
}
+37
lexicons/git/temp/getHead.json
+37
lexicons/git/temp/getHead.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getHead",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
}
16
+
}
17
+
},
18
+
"output": {
19
+
"encoding": "application/json",
20
+
"schema": {
21
+
"type": "ref",
22
+
"ref": "sh.tangled.git.temp.defs#branch"
23
+
}
24
+
},
25
+
"errors": [
26
+
{
27
+
"name": "RepoNotFound",
28
+
"description": "Repository not found or access denied"
29
+
},
30
+
{
31
+
"name": "InvalidRequest",
32
+
"description": "Invalid request parameters"
33
+
}
34
+
]
35
+
}
36
+
}
37
+
}
+44
lexicons/git/temp/getTag.json
+44
lexicons/git/temp/getTag.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getTag",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": [
10
+
"repo",
11
+
"tag"
12
+
],
13
+
"properties": {
14
+
"repo": {
15
+
"type": "string",
16
+
"format": "at-uri",
17
+
"description": "AT-URI of the repository"
18
+
},
19
+
"tag": {
20
+
"type": "string",
21
+
"description": "Name of tag, such as v1.3.0"
22
+
}
23
+
}
24
+
},
25
+
"output": {
26
+
"encoding": "*/*"
27
+
},
28
+
"errors": [
29
+
{
30
+
"name": "RepoNotFound",
31
+
"description": "Repository not found or access denied"
32
+
},
33
+
{
34
+
"name": "TagNotFound",
35
+
"description": "Tag not found"
36
+
},
37
+
{
38
+
"name": "InvalidRequest",
39
+
"description": "Invalid request parameters"
40
+
}
41
+
]
42
+
}
43
+
}
44
+
}
+183
lexicons/git/temp/getTree.json
+183
lexicons/git/temp/getTree.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.getTree",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": [
10
+
"repo",
11
+
"ref"
12
+
],
13
+
"properties": {
14
+
"repo": {
15
+
"type": "string",
16
+
"format": "at-uri",
17
+
"description": "AT-URI of the repository"
18
+
},
19
+
"ref": {
20
+
"type": "string",
21
+
"description": "Git reference (branch, tag, or commit SHA)"
22
+
},
23
+
"path": {
24
+
"type": "string",
25
+
"description": "Path within the repository tree",
26
+
"default": ""
27
+
}
28
+
}
29
+
},
30
+
"output": {
31
+
"encoding": "application/json",
32
+
"schema": {
33
+
"type": "object",
34
+
"required": [
35
+
"ref",
36
+
"files"
37
+
],
38
+
"properties": {
39
+
"ref": {
40
+
"type": "string",
41
+
"description": "The git reference used"
42
+
},
43
+
"parent": {
44
+
"type": "string",
45
+
"description": "The parent path in the tree"
46
+
},
47
+
"dotdot": {
48
+
"type": "string",
49
+
"description": "Parent directory path"
50
+
},
51
+
"readme": {
52
+
"type": "ref",
53
+
"ref": "#readme",
54
+
"description": "Readme for this file tree"
55
+
},
56
+
"lastCommit": {
57
+
"type": "ref",
58
+
"ref": "#lastCommit"
59
+
},
60
+
"files": {
61
+
"type": "array",
62
+
"items": {
63
+
"type": "ref",
64
+
"ref": "#treeEntry"
65
+
}
66
+
}
67
+
}
68
+
}
69
+
},
70
+
"errors": [
71
+
{
72
+
"name": "RepoNotFound",
73
+
"description": "Repository not found or access denied"
74
+
},
75
+
{
76
+
"name": "RefNotFound",
77
+
"description": "Git reference not found"
78
+
},
79
+
{
80
+
"name": "PathNotFound",
81
+
"description": "Path not found in repository tree"
82
+
},
83
+
{
84
+
"name": "InvalidRequest",
85
+
"description": "Invalid request parameters"
86
+
}
87
+
]
88
+
},
89
+
"readme": {
90
+
"type": "object",
91
+
"required": [
92
+
"filename",
93
+
"contents"
94
+
],
95
+
"properties": {
96
+
"filename": {
97
+
"type": "string",
98
+
"description": "Name of the readme file"
99
+
},
100
+
"contents": {
101
+
"type": "string",
102
+
"description": "Contents of the readme file"
103
+
}
104
+
}
105
+
},
106
+
"treeEntry": {
107
+
"type": "object",
108
+
"required": [
109
+
"name",
110
+
"mode",
111
+
"size"
112
+
],
113
+
"properties": {
114
+
"name": {
115
+
"type": "string",
116
+
"description": "Relative file or directory name"
117
+
},
118
+
"mode": {
119
+
"type": "string",
120
+
"description": "File mode"
121
+
},
122
+
"size": {
123
+
"type": "integer",
124
+
"description": "File size in bytes"
125
+
},
126
+
"last_commit": {
127
+
"type": "ref",
128
+
"ref": "#lastCommit"
129
+
}
130
+
}
131
+
},
132
+
"lastCommit": {
133
+
"type": "object",
134
+
"required": [
135
+
"hash",
136
+
"message",
137
+
"when"
138
+
],
139
+
"properties": {
140
+
"hash": {
141
+
"type": "string",
142
+
"description": "Commit hash"
143
+
},
144
+
"message": {
145
+
"type": "string",
146
+
"description": "Commit message"
147
+
},
148
+
"author": {
149
+
"type": "ref",
150
+
"ref": "#signature"
151
+
},
152
+
"when": {
153
+
"type": "string",
154
+
"format": "datetime",
155
+
"description": "Commit timestamp"
156
+
}
157
+
}
158
+
},
159
+
"signature": {
160
+
"type": "object",
161
+
"required": [
162
+
"name",
163
+
"email",
164
+
"when"
165
+
],
166
+
"properties": {
167
+
"name": {
168
+
"type": "string",
169
+
"description": "Author name"
170
+
},
171
+
"email": {
172
+
"type": "string",
173
+
"description": "Author email"
174
+
},
175
+
"when": {
176
+
"type": "string",
177
+
"format": "datetime",
178
+
"description": "Author timestamp"
179
+
}
180
+
}
181
+
}
182
+
}
183
+
}
+44
lexicons/git/temp/listBranches.json
+44
lexicons/git/temp/listBranches.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.listBranches",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"limit": {
17
+
"type": "integer",
18
+
"description": "Maximum number of branches to return",
19
+
"minimum": 1,
20
+
"maximum": 100,
21
+
"default": 50
22
+
},
23
+
"cursor": {
24
+
"type": "string",
25
+
"description": "Pagination cursor"
26
+
}
27
+
}
28
+
},
29
+
"output": {
30
+
"encoding": "*/*"
31
+
},
32
+
"errors": [
33
+
{
34
+
"name": "RepoNotFound",
35
+
"description": "Repository not found or access denied"
36
+
},
37
+
{
38
+
"name": "InvalidRequest",
39
+
"description": "Invalid request parameters"
40
+
}
41
+
]
42
+
}
43
+
}
44
+
}
+56
lexicons/git/temp/listCommits.json
+56
lexicons/git/temp/listCommits.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.listCommits",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"ref": {
17
+
"type": "string",
18
+
"description": "Git reference (branch, tag, or commit SHA)"
19
+
},
20
+
"limit": {
21
+
"type": "integer",
22
+
"description": "Maximum number of commits to return",
23
+
"minimum": 1,
24
+
"maximum": 100,
25
+
"default": 50
26
+
},
27
+
"cursor": {
28
+
"type": "string",
29
+
"description": "Pagination cursor (commit SHA)"
30
+
}
31
+
}
32
+
},
33
+
"output": {
34
+
"encoding": "*/*"
35
+
},
36
+
"errors": [
37
+
{
38
+
"name": "RepoNotFound",
39
+
"description": "Repository not found or access denied"
40
+
},
41
+
{
42
+
"name": "RefNotFound",
43
+
"description": "Git reference not found"
44
+
},
45
+
{
46
+
"name": "PathNotFound",
47
+
"description": "Path not found in repository"
48
+
},
49
+
{
50
+
"name": "InvalidRequest",
51
+
"description": "Invalid request parameters"
52
+
}
53
+
]
54
+
}
55
+
}
56
+
}
+100
lexicons/git/temp/listLanguages.json
+100
lexicons/git/temp/listLanguages.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.listLanguages",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"ref": {
17
+
"type": "string",
18
+
"description": "Git reference (branch, tag, or commit SHA)",
19
+
"default": "HEAD"
20
+
}
21
+
}
22
+
},
23
+
"output": {
24
+
"encoding": "application/json",
25
+
"schema": {
26
+
"type": "object",
27
+
"required": ["ref", "languages"],
28
+
"properties": {
29
+
"ref": {
30
+
"type": "string",
31
+
"description": "The git reference used"
32
+
},
33
+
"languages": {
34
+
"type": "array",
35
+
"items": {
36
+
"type": "ref",
37
+
"ref": "#language"
38
+
}
39
+
},
40
+
"totalSize": {
41
+
"type": "integer",
42
+
"description": "Total size of all analyzed files in bytes"
43
+
},
44
+
"totalFiles": {
45
+
"type": "integer",
46
+
"description": "Total number of files analyzed"
47
+
}
48
+
}
49
+
}
50
+
},
51
+
"errors": [
52
+
{
53
+
"name": "RepoNotFound",
54
+
"description": "Repository not found or access denied"
55
+
},
56
+
{
57
+
"name": "RefNotFound",
58
+
"description": "Git reference not found"
59
+
},
60
+
{
61
+
"name": "InvalidRequest",
62
+
"description": "Invalid request parameters"
63
+
}
64
+
]
65
+
},
66
+
"language": {
67
+
"type": "object",
68
+
"required": ["name", "size", "percentage"],
69
+
"properties": {
70
+
"name": {
71
+
"type": "string",
72
+
"description": "Programming language name"
73
+
},
74
+
"size": {
75
+
"type": "integer",
76
+
"description": "Total size of files in this language (bytes)"
77
+
},
78
+
"percentage": {
79
+
"type": "integer",
80
+
"description": "Percentage of total codebase (0-100)"
81
+
},
82
+
"fileCount": {
83
+
"type": "integer",
84
+
"description": "Number of files in this language"
85
+
},
86
+
"color": {
87
+
"type": "string",
88
+
"description": "Hex color code for this language"
89
+
},
90
+
"extensions": {
91
+
"type": "array",
92
+
"items": {
93
+
"type": "string"
94
+
},
95
+
"description": "File extensions associated with this language"
96
+
}
97
+
}
98
+
}
99
+
}
100
+
}
+44
lexicons/git/temp/listTags.json
+44
lexicons/git/temp/listTags.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "sh.tangled.git.temp.listTags",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"parameters": {
8
+
"type": "params",
9
+
"required": ["repo"],
10
+
"properties": {
11
+
"repo": {
12
+
"type": "string",
13
+
"format": "at-uri",
14
+
"description": "AT-URI of the repository"
15
+
},
16
+
"limit": {
17
+
"type": "integer",
18
+
"description": "Maximum number of tags to return",
19
+
"minimum": 1,
20
+
"maximum": 100,
21
+
"default": 50
22
+
},
23
+
"cursor": {
24
+
"type": "string",
25
+
"description": "Pagination cursor"
26
+
}
27
+
}
28
+
},
29
+
"output": {
30
+
"encoding": "*/*"
31
+
},
32
+
"errors": [
33
+
{
34
+
"name": "RepoNotFound",
35
+
"description": "Repository not found or access denied"
36
+
},
37
+
{
38
+
"name": "InvalidRequest",
39
+
"description": "Invalid request parameters"
40
+
}
41
+
]
42
+
}
43
+
}
44
+
}
+39
-20
nix/gomod2nix.toml
+39
-20
nix/gomod2nix.toml
···
139
139
hash = "sha256-GWm5i1ukuBukV0GMF1rffpbOSSXZdfg6/0pABMiGzLQ="
140
140
replaced = "tangled.sh/oppi.li/go-gitdiff"
141
141
[mod."github.com/bluesky-social/indigo"]
142
+
version = "v0.0.0-20260315101958-fb1dfa36fed2"
143
+
hash = "sha256-R5Dmcsi1a5LquA/a30YyjLAh7Mjg17EuTNVCDxyw4JE="
144
+
replaced = "github.com/boltlessengineer/indigo"
142
-
version = "v0.0.0-20251003000214-3259b215110e"
143
-
hash = "sha256-qi/GrquJznbLnnHVpd7IqoryCESbi6xE4X1SiEM2qlo="
144
145
[mod."github.com/bluesky-social/jetstream"]
146
+
version = "v0.0.0-20260226214936-e0274250f654"
147
+
hash = "sha256-VE93NvI3PreteLHnlv7WT6GgH2vSjtoFjMygCmrznfg="
145
-
version = "v0.0.0-20241210005130-ea96859b93d1"
146
-
hash = "sha256-AiapbrkjXboIKc5QNiWH0KyNs0zKnn6UlGwWFlkUfm0="
147
148
[mod."github.com/bmatcuk/doublestar/v4"]
148
149
version = "v4.9.1"
149
150
hash = "sha256-0iyHjyTAsfhgYSsE+NKxSNGBuM3Id615VWeQhssTShE="
···
225
226
[mod."github.com/dustin/go-humanize"]
226
227
version = "v1.0.1"
227
228
hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc="
229
+
[mod."github.com/earthboundkid/versioninfo/v2"]
230
+
version = "v2.24.1"
231
+
hash = "sha256-nbRdiX9WN2y1aiw1CR/DQ6AYqztow8FazndwY3kByHM="
228
232
[mod."github.com/emirpasic/gods"]
229
233
version = "v1.18.1"
230
234
hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="
···
394
398
[mod."github.com/ipfs/go-metrics-interface"]
395
399
version = "v0.3.0"
396
400
hash = "sha256-b3tp3jxecLmJEGx2kW7MiKGlAKPEWg/LJ7hXylSC8jQ="
401
+
[mod."github.com/jackc/pgpassfile"]
402
+
version = "v1.0.0"
403
+
hash = "sha256-H0nFbC34/3pZUFnuiQk9W7yvAMh6qJDrqvHp+akBPLM="
404
+
[mod."github.com/jackc/pgservicefile"]
405
+
version = "v0.0.0-20240606120523-5a60cdf6a761"
406
+
hash = "sha256-ETpGsLAA2wcm5xJBayr/mZrCE1YsWbnkbSSX3ptrFn0="
407
+
[mod."github.com/jackc/pgx/v5"]
408
+
version = "v5.8.0"
409
+
hash = "sha256-Mq5/A/Obcceu6kKxUv30DPC2ZaVvD8Iq/YtmLm1BVec="
410
+
[mod."github.com/jackc/puddle/v2"]
411
+
version = "v2.2.2"
412
+
hash = "sha256-IUxdu4JYfsCh/qlz2SiUWu7EVPHhyooiVA4oaS2Z6yk="
397
413
[mod."github.com/json-iterator/go"]
398
414
version = "v1.1.12"
399
415
hash = "sha256-To8A0h+lbfZ/6zM+2PpRpY3+L6725OPC66lffq6fUoM="
···
503
519
version = "v1.5.5"
504
520
hash = "sha256-ouhfDUCXsfpcgaCLfJE9oYprAQHuV61OJzb/aEhT0j8="
505
521
[mod."github.com/prometheus/client_golang"]
522
+
version = "v1.23.2"
523
+
hash = "sha256-3GD4fBFa1tJu8MS4TNP6r2re2eViUE+kWUaieIOQXCg="
506
-
version = "v1.22.0"
507
-
hash = "sha256-OJ/9rlWG1DIPQJAZUTzjykkX0o+f+4IKLvW8YityaMQ="
508
524
[mod."github.com/prometheus/client_model"]
509
525
version = "v0.6.2"
510
526
hash = "sha256-q6Fh6v8iNJN9ypD47LjWmx66YITa3FyRjZMRsuRTFeQ="
511
527
[mod."github.com/prometheus/common"]
528
+
version = "v0.66.1"
529
+
hash = "sha256-bqHPaV9IV70itx63wqwgy2PtxMN0sn5ThVxDmiD7+Tk="
512
-
version = "v0.64.0"
513
-
hash = "sha256-uy3KO60F2Cvhamz3fWQALGSsy13JiTk3NfpXgRuwqtI="
514
530
[mod."github.com/prometheus/procfs"]
515
531
version = "v0.16.1"
516
532
hash = "sha256-OBCvKlLW2obct35p0L9Q+1ZrxZjpTmbgHMP2rng9hpo="
···
543
559
version = "v0.0.0-20220730225603-2ab79fcdd4ef"
544
560
hash = "sha256-/XmSE/J+f6FLWXGvljh6uBK71uoCAK3h82XQEQ1Ki68="
545
561
[mod."github.com/stretchr/testify"]
562
+
version = "v1.11.1"
563
+
hash = "sha256-sWfjkuKJyDllDEtnM8sb/pdLzPQmUYWYtmeWz/5suUc="
546
-
version = "v1.10.0"
547
-
hash = "sha256-fJ4gnPr0vnrOhjQYQwJ3ARDKPsOtA7d4olQmQWR+wpI="
548
564
[mod."github.com/tidwall/gjson"]
549
565
version = "v1.18.0"
550
566
hash = "sha256-CO6hqDu8Y58Po6A01e5iTpwiUBQ5khUZsw7czaJHw0I="
···
558
574
version = "v1.2.5"
559
575
hash = "sha256-OYGNolkmL7E1Qs2qrQ3IVpQp5gkcHNU/AB/z2O+Myps="
560
576
[mod."github.com/urfave/cli/v3"]
577
+
version = "v3.4.1"
578
+
hash = "sha256-cDMaQrIVMthUhdyI1mKXzDC5/wIK151073lzRl92RnA="
561
-
version = "v3.3.3"
562
-
hash = "sha256-FdPiu7koY1qBinkfca4A05zCrX+Vu4eRz8wlRDZJyGg="
563
579
[mod."github.com/vmihailenco/go-tinylfu"]
564
580
version = "v0.2.2"
565
581
hash = "sha256-ZHr4g7DJAV6rLcfrEWZwo9wJSeZcXB9KSP38UIOFfaM="
···
629
645
[mod."go.uber.org/zap"]
630
646
version = "v1.27.0"
631
647
hash = "sha256-8655KDrulc4Das3VRduO9MjCn8ZYD5WkULjCvruaYsU="
648
+
[mod."go.yaml.in/yaml/v2"]
649
+
version = "v2.4.2"
650
+
hash = "sha256-oC8RWdf1zbMYCtmR0ATy/kCkhIwPR9UqFZSMOKLVF/A="
632
651
[mod."golang.org/x/crypto"]
652
+
version = "v0.41.0"
653
+
hash = "sha256-o5Di0lsFmYnXl7a5MBTqmN9vXMCRpE9ay71C1Ar8jEY="
633
-
version = "v0.40.0"
634
-
hash = "sha256-I6p2fqvz63P9MwAuoQrljI7IUbfZQvCem0ii4Q2zZng="
635
654
[mod."golang.org/x/exp"]
636
655
version = "v0.0.0-20250620022241-b7579e27df2b"
637
656
hash = "sha256-IsDTeuWLj4UkPO4NhWTvFeZ22WNtlxjoWiyAJh6zdig="
···
639
658
version = "v0.31.0"
640
659
hash = "sha256-ZFTlu9+4QToPPLA8C5UcG2eq/lQylq81RoG/WtYo9rg="
641
660
[mod."golang.org/x/net"]
661
+
version = "v0.43.0"
662
+
hash = "sha256-bf3iQFrsC8BoarVaS0uSspEFAcr1zHp1uziTtBpwV34="
642
-
version = "v0.42.0"
643
-
hash = "sha256-YxileisIIez+kcAI+21kY5yk0iRuEqti2YdmS8jvP2s="
644
663
[mod."golang.org/x/sync"]
645
664
version = "v0.17.0"
646
665
hash = "sha256-M85lz4hK3/fzmcUViAp/CowHSxnr3BHSO7pjHp1O6i0="
647
666
[mod."golang.org/x/sys"]
667
+
version = "v0.35.0"
668
+
hash = "sha256-ZKM8pesQE6NAFZeKQ84oPn5JMhGr8g4TSwLYAsHMGSI="
648
-
version = "v0.34.0"
649
-
hash = "sha256-5rZ7p8IaGli5X1sJbfIKOcOEwY4c0yQhinJPh2EtK50="
650
669
[mod."golang.org/x/text"]
651
670
version = "v0.29.0"
652
671
hash = "sha256-2cWBtJje+Yc+AnSgCANqBlIwnOMZEGkpQ2cFI45VfLI="
···
666
685
version = "v1.73.0"
667
686
hash = "sha256-LfVlwip++q2DX70RU6CxoXglx1+r5l48DwlFD05G11c="
668
687
[mod."google.golang.org/protobuf"]
688
+
version = "v1.36.8"
689
+
hash = "sha256-yZN8ZON0b5HjUNUSubHst7zbvnMsOzd81tDPYQRtPgM="
669
-
version = "v1.36.6"
670
-
hash = "sha256-lT5qnefI5FDJnowz9PEkAGylH3+fE+A3DJDkAyy9RMc="
671
690
[mod."gopkg.in/fsnotify.v1"]
672
691
version = "v1.4.7"
673
692
hash = "sha256-j/Ts92oXa3k1MFU7Yd8/AqafRTsFn7V2pDKCyDJLah8="
+143
nix/modules/knotmirror.nix
+143
nix/modules/knotmirror.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
lib,
5
+
...
6
+
}: let
7
+
cfg = config.services.tangled.knotmirror;
8
+
in
9
+
with lib; {
10
+
options.services.tangled.knotmirror = {
11
+
enable = mkOption {
12
+
type = types.bool;
13
+
default = false;
14
+
description = "Enable a tangled knot";
15
+
};
16
+
17
+
package = mkOption {
18
+
type = types.package;
19
+
description = "Package to use for the knotmirror";
20
+
};
21
+
22
+
tap-package = mkOption {
23
+
type = types.package;
24
+
description = "tap package to use for the knotmirror";
25
+
};
26
+
27
+
listenAddr = mkOption {
28
+
type = types.str;
29
+
default = "0.0.0.0:7000";
30
+
description = "Address to listen on";
31
+
};
32
+
33
+
adminListenAddr = mkOption {
34
+
type = types.str;
35
+
default = "127.0.0.1:7200";
36
+
description = "Address to listen on";
37
+
};
38
+
39
+
hostname = mkOption {
40
+
type = types.str;
41
+
example = "my.knotmirror.com";
42
+
description = "Hostname for the server (required)";
43
+
};
44
+
45
+
dbUrl = mkOption {
46
+
type = types.str;
47
+
example = "postgresql://...";
48
+
description = "Database URL. postgresql expected (required)";
49
+
};
50
+
51
+
atpPlcUrl = mkOption {
52
+
type = types.str;
53
+
default = "https://plc.directory";
54
+
description = "atproto PLC directory";
55
+
};
56
+
57
+
atpRelayUrl = mkOption {
58
+
type = types.str;
59
+
default = "https://relay1.us-east.bsky.network";
60
+
description = "atproto relay";
61
+
};
62
+
63
+
fullNetwork = mkOption {
64
+
type = types.bool;
65
+
default = false;
66
+
description = "Whether to automatically mirror from entire network";
67
+
};
68
+
69
+
tap = {
70
+
port = mkOption {
71
+
type = types.port;
72
+
default = 7480;
73
+
description = "Internal port to run the knotmirror tap";
74
+
};
75
+
76
+
dbUrl = mkOption {
77
+
type = types.str;
78
+
default = "sqlite:///var/lib/knotmirror-tap/tap.db";
79
+
description = "database connection string (sqlite://path or postgres://...)";
80
+
};
81
+
};
82
+
};
83
+
config = mkIf cfg.enable {
84
+
environment.systemPackages = [
85
+
pkgs.git
86
+
cfg.package
87
+
];
88
+
89
+
systemd.services.tap-knotmirror = {
90
+
description = "knotmirror tap service";
91
+
after = ["network.target"];
92
+
wantedBy = ["multi-user.target"];
93
+
serviceConfig = {
94
+
LogsDirectory = "knotmirror-tap";
95
+
StateDirectory = "knotmirror-tap";
96
+
Environment = [
97
+
"TAP_BIND=:${toString cfg.tap.port}"
98
+
"TAP_PLC_URL=${cfg.atpPlcUrl}"
99
+
"TAP_RELAY_URL=${cfg.atpRelayUrl}"
100
+
"TAP_RESYNC_PARALLELISM=10"
101
+
"TAP_DATABASE_URL=${cfg.tap.dbUrl}"
102
+
"TAP_RETRY_TIMEOUT=60s"
103
+
"TAP_COLLECTION_FILTERS=sh.tangled.repo"
104
+
(
105
+
if cfg.fullNetwork
106
+
then "TAP_SIGNAL_COLLECTION=sh.tangled.repo"
107
+
else "TAP_FULL_NETWORK=false"
108
+
)
109
+
];
110
+
ExecStart = "${getExe cfg.tap-package} run";
111
+
};
112
+
};
113
+
114
+
systemd.services.knotmirror = {
115
+
description = "knotmirror service";
116
+
after = ["network.target" "tap-knotmirror.service"];
117
+
wantedBy = ["multi-user.target"];
118
+
path = [
119
+
pkgs.git
120
+
];
121
+
serviceConfig = {
122
+
LogsDirectory = "knotmirror";
123
+
StateDirectory = "knotmirror";
124
+
Environment = [
125
+
# TODO: add environment variables
126
+
"MIRROR_LISTEN=${cfg.listenAddr}"
127
+
"MIRROR_HOSTNAME=${cfg.hostname}"
128
+
"MIRROR_TAP_URL=http://localhost:${toString cfg.tap.port}"
129
+
"MIRROR_DB_URL=${cfg.dbUrl}"
130
+
"MIRROR_GIT_BASEPATH=/var/lib/knotmirror/repos"
131
+
"MIRROR_KNOT_USE_SSL=true"
132
+
"MIRROR_KNOT_SSRF=true"
133
+
"MIRROR_RESYNC_PARALLELISM=12"
134
+
"MIRROR_METRICS_LISTEN=127.0.0.1:7100"
135
+
"MIRROR_ADMIN_LISTEN=${cfg.adminListenAddr}"
136
+
"MIRROR_SLURPER_CONCURRENCY=4"
137
+
];
138
+
ExecStart = "${getExe cfg.package} serve";
139
+
Restart = "always";
140
+
};
141
+
};
142
+
};
143
+
}
+18
nix/pkgs/knot-mirror.nix
+18
nix/pkgs/knot-mirror.nix
+20
nix/pkgs/tap.nix
+20
nix/pkgs/tap.nix
···
1
+
{
2
+
buildGoModule,
3
+
fetchFromGitHub,
4
+
}:
5
+
buildGoModule {
6
+
pname = "tap";
7
+
version = "0.1.0";
8
+
src = fetchFromGitHub {
9
+
owner = "bluesky-social";
10
+
repo = "indigo";
11
+
rev = "498ecb9693e8ae050f73234c86f340f51ad896a9";
12
+
sha256 = "sha256-KASCdwkg/hlKBt7RTW3e3R5J3hqJkphoarFbaMgtN1k=";
13
+
};
14
+
subPackages = ["cmd/tap"];
15
+
vendorHash = "sha256-UOedwNYnM8Jx6B7Y9tFcZX8IeUBESAFAPTRYk7n0yo8=";
16
+
doCheck = false;
17
+
meta = {
18
+
mainProgram = "tap";
19
+
};
20
+
}
+3
tapc/readme.md
+3
tapc/readme.md
+24
tapc/simpleIndexer.go
+24
tapc/simpleIndexer.go
···
1
+
package tapc
2
+
3
+
import "context"
4
+
5
+
type SimpleIndexer struct {
6
+
EventHandler func(ctx context.Context, evt Event) error
7
+
ErrorHandler func(ctx context.Context, err error)
8
+
}
9
+
10
+
var _ Handler = (*SimpleIndexer)(nil)
11
+
12
+
func (i *SimpleIndexer) OnEvent(ctx context.Context, evt Event) error {
13
+
if i.EventHandler == nil {
14
+
return nil
15
+
}
16
+
return i.EventHandler(ctx, evt)
17
+
}
18
+
19
+
func (i *SimpleIndexer) OnError(ctx context.Context, err error) {
20
+
if i.ErrorHandler == nil {
21
+
return
22
+
}
23
+
i.ErrorHandler(ctx, err)
24
+
}
+170
tapc/tap.go
+170
tapc/tap.go
···
1
+
/// heavily inspired by <https://github.com/bluesky-social/atproto/blob/c7f5a868837d3e9b3289f988fee2267789327b06/packages/tap/README.md>
2
+
3
+
package tapc
4
+
5
+
import (
6
+
"bytes"
7
+
"context"
8
+
"encoding/json"
9
+
"fmt"
10
+
"net/http"
11
+
"net/url"
12
+
"time"
13
+
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/gorilla/websocket"
16
+
"tangled.org/core/log"
17
+
)
18
+
19
+
type Handler interface {
20
+
OnEvent(ctx context.Context, evt Event) error
21
+
OnError(ctx context.Context, err error)
22
+
}
23
+
24
+
type Client struct {
25
+
Url string
26
+
AdminPassword string
27
+
HTTPClient *http.Client
28
+
}
29
+
30
+
func NewClient(url, adminPassword string) Client {
31
+
return Client{
32
+
Url: url,
33
+
AdminPassword: adminPassword,
34
+
HTTPClient: &http.Client{},
35
+
}
36
+
}
37
+
38
+
func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
39
+
body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
40
+
if err != nil {
41
+
return err
42
+
}
43
+
req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/add", bytes.NewReader(body))
44
+
if err != nil {
45
+
return err
46
+
}
47
+
req.SetBasicAuth("admin", c.AdminPassword)
48
+
req.Header.Set("Content-Type", "application/json")
49
+
50
+
resp, err := c.HTTPClient.Do(req)
51
+
if err != nil {
52
+
return err
53
+
}
54
+
defer resp.Body.Close()
55
+
if resp.StatusCode != http.StatusOK {
56
+
return fmt.Errorf("tap: /repos/add failed with status %d", resp.StatusCode)
57
+
}
58
+
return nil
59
+
}
60
+
61
+
func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
62
+
body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
63
+
if err != nil {
64
+
return err
65
+
}
66
+
req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/remove", bytes.NewReader(body))
67
+
if err != nil {
68
+
return err
69
+
}
70
+
req.SetBasicAuth("admin", c.AdminPassword)
71
+
req.Header.Set("Content-Type", "application/json")
72
+
73
+
resp, err := c.HTTPClient.Do(req)
74
+
if err != nil {
75
+
return err
76
+
}
77
+
defer resp.Body.Close()
78
+
if resp.StatusCode != http.StatusOK {
79
+
return fmt.Errorf("tap: /repos/remove failed with status %d", resp.StatusCode)
80
+
}
81
+
return nil
82
+
}
83
+
84
+
func (c *Client) Connect(ctx context.Context, handler Handler) error {
85
+
l := log.FromContext(ctx)
86
+
87
+
u, err := url.Parse(c.Url)
88
+
if err != nil {
89
+
return err
90
+
}
91
+
if u.Scheme == "https" {
92
+
u.Scheme = "wss"
93
+
} else {
94
+
u.Scheme = "ws"
95
+
}
96
+
u.Path = "/channel"
97
+
98
+
// TODO: set auth on dial
99
+
100
+
url := u.String()
101
+
102
+
var backoff int
103
+
for {
104
+
select {
105
+
case <-ctx.Done():
106
+
return ctx.Err()
107
+
default:
108
+
}
109
+
110
+
header := http.Header{
111
+
"Authorization": []string{""},
112
+
}
113
+
conn, res, err := websocket.DefaultDialer.DialContext(ctx, url, header)
114
+
if err != nil {
115
+
l.Warn("dialing failed", "url", url, "err", err, "backoff", backoff)
116
+
time.Sleep(time.Duration(5+backoff) * time.Second)
117
+
backoff++
118
+
119
+
continue
120
+
}
121
+
l.Info("connected to tap service")
122
+
123
+
l.Info("tap event subscription response", "code", res.StatusCode)
124
+
125
+
if err = c.handleConnection(ctx, conn, handler); err != nil {
126
+
l.Warn("tap connection failed", "err", err, "backoff", backoff)
127
+
}
128
+
}
129
+
}
130
+
131
+
func (c *Client) handleConnection(ctx context.Context, conn *websocket.Conn, handler Handler) error {
132
+
l := log.FromContext(ctx)
133
+
134
+
defer func() {
135
+
conn.Close()
136
+
l.Warn("closed tap conection")
137
+
}()
138
+
l.Info("established tap conection")
139
+
140
+
for {
141
+
select {
142
+
case <-ctx.Done():
143
+
return ctx.Err()
144
+
default:
145
+
}
146
+
_, message, err := conn.ReadMessage()
147
+
if err != nil {
148
+
return err
149
+
}
150
+
151
+
var ev Event
152
+
if err := json.Unmarshal(message, &ev); err != nil {
153
+
handler.OnError(ctx, fmt.Errorf("failed to parse message: %w", err))
154
+
continue
155
+
}
156
+
if err := handler.OnEvent(ctx, ev); err != nil {
157
+
handler.OnError(ctx, fmt.Errorf("failed to process event %d: %w", ev.ID, err))
158
+
continue
159
+
}
160
+
161
+
ack := map[string]any{
162
+
"type": "ack",
163
+
"id": ev.ID,
164
+
}
165
+
if err := conn.WriteJSON(ack); err != nil {
166
+
l.Warn("failed to send ack", "err", err)
167
+
continue
168
+
}
169
+
}
170
+
}
+62
tapc/types.go
+62
tapc/types.go
···
1
+
package tapc
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
)
9
+
10
+
type EventType string
11
+
12
+
const (
13
+
EvtRecord EventType = "record"
14
+
EvtIdentity EventType = "identity"
15
+
)
16
+
17
+
type Event struct {
18
+
ID int64 `json:"id"`
19
+
Type EventType `json:"type"`
20
+
Record *RecordEventData `json:"record,omitempty"`
21
+
Identity *IdentityEventData `json:"identity,omitempty"`
22
+
}
23
+
24
+
type RecordEventData struct {
25
+
Live bool `json:"live"`
26
+
Did syntax.DID `json:"did"`
27
+
Rev string `json:"rev"`
28
+
Collection syntax.NSID `json:"collection"`
29
+
Rkey syntax.RecordKey `json:"rkey"`
30
+
Action RecordAction `json:"action"`
31
+
Record json.RawMessage `json:"record,omitempty"`
32
+
CID *syntax.CID `json:"cid,omitempty"`
33
+
}
34
+
35
+
func (r *RecordEventData) AtUri() syntax.ATURI {
36
+
return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, r.Collection, r.Rkey))
37
+
}
38
+
39
+
type RecordAction string
40
+
41
+
const (
42
+
RecordCreateAction RecordAction = "create"
43
+
RecordUpdateAction RecordAction = "update"
44
+
RecordDeleteAction RecordAction = "delete"
45
+
)
46
+
47
+
type IdentityEventData struct {
48
+
DID syntax.DID `json:"did"`
49
+
Handle string `json:"handle"`
50
+
IsActive bool `json:"is_active"`
51
+
Status RepoStatus `json:"status"`
52
+
}
53
+
54
+
type RepoStatus string
55
+
56
+
const (
57
+
RepoStatusActive RepoStatus = "active"
58
+
RepoStatusTakendown RepoStatus = "takendown"
59
+
RepoStatusSuspended RepoStatus = "suspended"
60
+
RepoStatusDeactivated RepoStatus = "deactivated"
61
+
RepoStatusDeleted RepoStatus = "deleted"
62
+
)
History
5 rounds
20 comments
expand 5 comments
i definitely see 42 changed files with +2214/-1481 on here! (you may be viewing an earlier round perhaps?)!
commit message is perfect, i do have some minor code change comments, but i am happy to handle that outside of this PR.
ah yes you're right, i didn't realize i was still on an earlier round! great, lmk if you need anything else :)
Not really related to this PR, but somehow Bun's lockfile is not collapsed by default. Maybe a bug in go-enry?
yup it is a known issue in enry!
pull request successfully merged
23 commits
expand
collapse
refactor: move og card rendering to external worker service
replace local go-based card rendering with calls to an external
ogcard worker (cloudflare workers). this offloads image processing
and simplifies the main application code.
changes:
- remove local ogcard rendering code from appview/ogcard/card.go
- add ogcard client package for external worker communication
- update issues opengraph handler to use new client
- add format parameter support to avatar service (webp/jpeg/png)
- add OgcardConfig for worker host configuration
update packages to their latest version
wire up opengraph handlers to use ogcard HTTP client
Replace local Go image rendering with HTTP calls to the remote TypeScript ogcard service. Add OgcardConfig for service URL configuration and extend the client to support repository, issue, and pull request card rendering.
appview/ogcard: improve fixtures.ts mock text
appview/ogcard: remove comment from logo.tsx
appview/ogcard: add knip dependency
knip helps find unused files, dependencies and exports in ts/js
projects. https://knip.dev/
appview/ogcard: commit bun.lock
appview/ogcard/components: add more font constants and use them
appview/ogcard/icons: remove unused exports
appview/ogcard: remove unused export from validation.ts
appview/ogcard: simplify wrangler.jsonc
appview/ogcard: fix language circles not being draw properly
appview/ogcard: show comments and reaction count only when larger than 0
appview/ogcard: switch repo card "updated at" to "created at"
appview/ogcard: replace remnant s with s
appview/ogcard: fix type issues
appview/ogcard: add TypeScript declaration for wasm module imports
appview/ogcard: pin satori to exact version to ensure wasm compatibility
appview/ogcard: add global_navigator compatibility flag for cf workers
appview/ogcard: load satori wasm module directly instead of from cdn
appview/ogcard: switch to wasm resvg and bundled fonts for cf Workers
appview/ogcard: extract satori/resvg runtime into separate package
appview/ogcard: format code with prettier
expand 0 comments
5 commits
expand
collapse
refactor: move og card rendering to external worker service
replace local go-based card rendering with calls to an external
ogcard worker (cloudflare workers). this offloads image processing
and simplifies the main application code.
changes:
- remove local ogcard rendering code from appview/ogcard/card.go
- add ogcard client package for external worker communication
- update issues opengraph handler to use new client
- add format parameter support to avatar service (webp/jpeg/png)
- add OgcardConfig for worker host configuration
update packages to their latest version
wire up opengraph handlers to use ogcard HTTP client
Replace local Go image rendering with HTTP calls to the remote TypeScript ogcard service. Add OgcardConfig for service URL configuration and extend the client to support repository, issue, and pull request card rendering.
appview/ogcard: improve fixtures.ts mock text
appview/ogcard: remove comment from logo.tsx
expand 11 comments
appview/ogcard/src/index.tsx:85
I think that if we 500 out, we should not show the stack to the public, just as a security measure
appview/ogcard/wrangler.jsonc:6 is this subbed out at runtime?
appview/ogcard/.gitignore:2 bun huh? :P I think it would be nice to commit the lockfile in fact
appview/ogcard/src/components/shared/constants.ts:20-26 is this going to be used somewhere?
appview/ogcard/src/components/shared/language-circles.tsx:9 Another unused constant, or maybe ctrl+f on this PR page doesn't catch the full picture
appview/ogcard/src/components/shared/footer-stats.tsx:19 would be cool if we did conditional rendering here for the 0 reactions case so that people don't get sad that their repo has 0 reactions advertised so blatantly heh
appview/repo/opengraph.go:83 we don't have an updatedAt on git repos, do we...
appview/repo/opengraph.go:92 doesn't this just render the actual timestamp itself?
appview/ogcard/src/lib/render.ts:17 I'm not super familiar with cf workers but fetching inside a worker doesn't seem super nice. could we inline it? I also noticed that package.json has this at ^0.25.0 which would make a mismatch pretty quickly
appview/ogcard/src/lib/fonts.ts:19-21 more inlining opportunities
my only comment: can we squash the to 1 (and rebase on latest master perhaps)?
would be ripe if you could update the new commit to adhere to the commit guidelines, it would also need DCO.
don't pay attention to #3... it was uh... nothing feel free to take a look at the latest squashed commit, let me know your thoughts!
3 commits
expand
collapse
refactor: move og card rendering to external worker service
replace local go-based card rendering with calls to an external
ogcard worker (cloudflare workers). this offloads image processing
and simplifies the main application code.
changes:
- remove local ogcard rendering code from appview/ogcard/card.go
- add ogcard client package for external worker communication
- update issues opengraph handler to use new client
- add format parameter support to avatar service (webp/jpeg/png)
- add OgcardConfig for worker host configuration
update packages to their latest version
wire up opengraph handlers to use ogcard HTTP client
Replace local Go image rendering with HTTP calls to the remote TypeScript ogcard service. Add OgcardConfig for service URL configuration and extend the client to support repository, issue, and pull request card rendering.
expand 2 comments
appview/ogcard/src/components/shared/logo.tsx:1
nit: this has been customized already, yes? :P
good catch 😸
3 commits
expand
collapse
refactor: move og card rendering to external worker service
replace local go-based card rendering with calls to an external
ogcard worker (cloudflare workers). this offloads image processing
and simplifies the main application code.
changes:
- remove local ogcard rendering code from appview/ogcard/card.go
- add ogcard client package for external worker communication
- update issues opengraph handler to use new client
- add format parameter support to avatar service (webp/jpeg/png)
- add OgcardConfig for worker host configuration
update packages to their latest version
wire up opengraph handlers to use ogcard HTTP client
Replace local Go image rendering with HTTP calls to the remote TypeScript ogcard service. Add OgcardConfig for service URL configuration and extend the client to support repository, issue, and pull request card rendering.
for some reason the amount of files changed here (36) is lower than the amount of files changed on my fork (42).. any ideas?
0927d254it seems like this pr doesn't have the exact same content as my commit?