// Package gitclient is a small git protocol v2 client over smart http, just // enough to read refs, trees and blobs from a knot without cloning. package gitclient import ( "bytes" "context" "errors" "fmt" "io" "maps" "net/http" "slices" "strings" "sync" ) var ( ErrRefNotFound = errors.New("branch or tag not found") ErrPathNotFound = errors.New("path not found at the commit") ErrTooLarge = errors.New("pack exceeds the read limit") ErrUnsupported = errors.New("server lacks a required v2 capability") ) // hex object id, sha1 or sha256 depending on the repo's object-format type Oid string type Client struct { http *http.Client url string // repo base, e.g. https://knot.example/did:plc:abc maxBytes int64 // cap on one fetch's pack mu sync.Mutex caps *Caps } func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client { if httpClient == nil { httpClient = http.DefaultClient } return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes} } // from GET info/refs?service=git-upload-pack with Git-Protocol: version=2 type Caps struct { ObjectFormat string // "sha1" or "sha256" LsRefs bool Fetch []string // fetch= features: shallow, filter, ... } func (c *Client) Caps(ctx context.Context) (Caps, error) { c.mu.Lock() defer c.mu.Unlock() if c.caps != nil { return *c.caps, nil } u := c.url + "/info/refs?service=git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return Caps{}, err } req.Header.Set("Git-Protocol", "version=2") resp, err := c.http.Do(req) if err != nil { return Caps{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return Caps{}, httpError(resp) } caps := Caps{ObjectFormat: "sha1"} p := newPktReader(io.LimitReader(resp.Body, 1<<20)) v2 := false for { kind, line, err := p.next() if err != nil && !v2 { return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u) } if err != nil { return Caps{}, fmt.Errorf("reading %s: %w", u, err) } if kind != pktData { // a v0-style "# service=" banner ends in a flush before the v2 advert if v2 { break } continue } key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=") switch key { case "version 2": v2 = true case "ls-refs": caps.LsRefs = true case "fetch": caps.Fetch = strings.Fields(val) case "object-format": caps.ObjectFormat = val } } if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" { return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat) } c.caps = &caps return caps, nil } type Ref struct { Name string Oid Oid Peeled Oid // set for annotated tags when Peel was asked Symref string // target of HEAD when Symrefs was asked } type LsRefsOpts struct { Prefixes []string // ref-prefix lines Peel bool Symrefs bool } // command=ls-refs func (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if !caps.LsRefs { return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported) } var args []string if opts.Peel { args = append(args, "peel") } if opts.Symrefs { args = append(args, "symrefs") } for _, prefix := range opts.Prefixes { args = append(args, "ref-prefix "+prefix) } resp, err := c.command(ctx, caps, "ls-refs", args) if err != nil { return nil, err } defer resp.Body.Close() p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes}) var refs []Ref for { kind, line, err := p.next() if err != nil { return nil, fmt.Errorf("ls-refs: %w", err) } if kind != pktData { return refs, nil } text := strings.TrimSuffix(string(line), "\n") if msg, ok := strings.CutPrefix(text, "ERR "); ok { return nil, fmt.Errorf("remote: %s", msg) } fields := strings.Split(text, " ") if len(fields) < 2 || !isOid(fields[0]) { return nil, fmt.Errorf("ls-refs: bad line %q", text) } ref := Ref{Oid: Oid(fields[0]), Name: fields[1]} for _, attr := range fields[2:] { if v, ok := strings.CutPrefix(attr, "peeled:"); ok { ref.Peeled = Oid(v) } else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok { ref.Symref = v } } refs = append(refs, ref) } } type FetchOpts struct { Wants []Oid Deepen int // 0 means no deepen line Filter string // e.g. "tree:0", "blob:none"; empty means none } // command=fetch with done and no haves, since there's no local object store func (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") { return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported) } if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") { return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported) } if len(opts.Wants) == 0 { return Objects{}, nil } args := []string{"no-progress", "ofs-delta"} for _, want := range opts.Wants { if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) { return nil, fmt.Errorf("bad object id %q", want) } args = append(args, "want "+string(want)) } if opts.Deepen > 0 { args = append(args, fmt.Sprintf("deepen %d", opts.Deepen)) } if opts.Filter != "" { args = append(args, "filter "+opts.Filter) } args = append(args, "done") resp, err := c.command(ctx, caps, "fetch", args) if err != nil { return nil, err } defer resp.Body.Close() pack, err := packfileSection(newPktReader(resp.Body)) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } return objs, nil } // commit oid for refs/heads/x, refs/tags/x or a bare name (branches before // tags, like git), tags peeled; ErrRefNotFound when ls-refs lists none of them func (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) { if ref == "" || strings.ContainsAny(ref, "\x00\n ") { return "", ErrRefNotFound } candidates := []string{ref} if !strings.HasPrefix(ref, "refs/") { candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref} } refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true}) if err != nil { return "", err } for _, name := range candidates { for _, r := range refs { if r.Name != name { continue } if r.Peeled != "" { return r.Peeled, nil } return r.Oid, nil } } return "", ErrRefNotFound } type File struct { Name string Mode string // octal, as in the tree Oid Oid Data []byte // nil for subtrees and submodules } // entries of dir at commit with blob contents. knot2 walks its whole repo to // check each want that isn't a commit, so these are kept to a minimum: the // commit comes with its root tree (deepen 1, tree:1, which every server counts // from the commit alike), middle trees come alone (tree:0), and the last tree // asks for tree:1, which knot2 answers with its blobs too. filter depth for a // wanted tree differs between servers, so blobs still missing are wanted by // oid. ErrPathNotFound when a component is missing or isn't a tree func (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"}) if err != nil { return nil, err } obj, ok := objs[commit] if !ok || obj.Kind != KindCommit { return nil, fmt.Errorf("fetch of commit %s didn't return it", commit) } tree, err := commitTree(obj.Data) if err != nil { return nil, err } var names []string for _, name := range strings.Split(dir, "/") { if name != "" { names = append(names, name) } } entries, err := c.tree(ctx, caps, objs, tree, "tree:0") if err != nil { return nil, err } for n, name := range names { i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name }) if i < 0 || strings.TrimLeft(entries[i].Mode, "0") != "40000" { return nil, fmt.Errorf("%s: %w", dir, ErrPathNotFound) } filter := "tree:0" if n == len(names)-1 { filter = "tree:1" } if entries, err = c.tree(ctx, caps, objs, entries[i].Oid, filter); err != nil { return nil, err } } var missing []Oid for _, e := range entries { if _, ok := objs[e.Oid]; isBlobMode(e.Mode) && !ok && !slices.Contains(missing, e.Oid) { missing = append(missing, e.Oid) } } if len(missing) > 0 { // wanted objects pass any filter; it keeps the server on its selective path more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"}) if err != nil { return nil, err } maps.Copy(objs, more) } files := make([]File, 0, len(entries)) for _, e := range entries { f := File{Name: e.Name, Mode: e.Mode, Oid: e.Oid} if isBlobMode(e.Mode) { obj, ok := objs[e.Oid] if !ok || obj.Kind != KindBlob { return nil, fmt.Errorf("fetch of blob %s didn't return it", e.Oid) } f.Data = obj.Data } files = append(files, f) } return files, nil } // entries of tree oid, fetched with filter unless an earlier fetch brought it; // whatever else the fetch returns lands in have func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) { if _, ok := have[oid]; !ok { objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter}) if err != nil { return nil, err } maps.Copy(have, objs) } obj, ok := have[oid] if !ok || obj.Kind != KindTree { return nil, fmt.Errorf("fetch of tree %s didn't return it", oid) } return parseTree(obj.Data, caps.ObjectFormat) } // anything but subtrees and submodules func isBlobMode(mode string) bool { mode = strings.TrimLeft(mode, "0") return mode != "40000" && mode != "160000" } // one POST git-upload-pack with a v2 command body func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) { var body bytes.Buffer writePkt(&body, "command="+command+"\n") // git refuses non-sha1 unless the format is echoed back if caps.ObjectFormat != "sha1" { writePkt(&body, "object-format="+caps.ObjectFormat+"\n") } writeDelim(&body) for _, arg := range args { if err := writePkt(&body, arg+"\n"); err != nil { return nil, err } } writeFlush(&body) u := c.url + "/git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body) if err != nil { return nil, err } req.Header.Set("Git-Protocol", "version=2") req.Header.Set("Content-Type", "application/x-git-upload-pack-request") req.Header.Set("Accept", "application/x-git-upload-pack-result") resp, err := c.http.Do(req) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() return nil, httpError(resp) } return resp, nil } // knot2 says why in a plain text body, like a want it won't serve func httpError(resp *http.Response) error { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status) if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?")) } return err }