// Package gitclient is a small git protocol v2 client over smart http, just // enough to read refs, trees and blobs from a knot without cloning. package gitclient import ( "bytes" "context" "errors" "fmt" "io" "maps" "net/http" "path" "slices" "strings" "sync" ) var ( ErrRefNotFound = errors.New("branch or tag not found") ErrPathNotFound = errors.New("path not found at the commit") ErrNotDir = fmt.Errorf("%w: not a directory", ErrPathNotFound) ErrTooLarge = errors.New("pack exceeds the read limit") ErrUnsupported = errors.New("server lacks a required v2 capability") ) // hex object id, sha1 or sha256 depending on the repo's object-format type Oid string type Client struct { http *http.Client url string // repo base, e.g. https://knot.example/did:plc:abc maxBytes int64 // cap on one fetch's pack mu sync.Mutex caps *Caps } func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client { if httpClient == nil { httpClient = http.DefaultClient } return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes} } type Caps struct { ObjectFormat string // "sha1" or "sha256" LsRefs bool Fetch []string // fetch= features: shallow, filter, ... } func (c *Client) Caps(ctx context.Context) (Caps, error) { c.mu.Lock() defer c.mu.Unlock() if c.caps != nil { return *c.caps, nil } u := c.url + "/info/refs?service=git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return Caps{}, err } req.Header.Set("Git-Protocol", "version=2") resp, err := c.http.Do(req) if err != nil { return Caps{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return Caps{}, httpError(resp) } caps := Caps{ObjectFormat: "sha1"} p := newPktReader(io.LimitReader(resp.Body, 1<<20)) v2 := false for { kind, line, err := p.next() if err != nil && !v2 { return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u) } if err != nil { return Caps{}, fmt.Errorf("reading %s: %w", u, err) } if kind != pktData { // a v0-style "# service=" banner ends in a flush before the v2 advert if v2 { break } continue } key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=") switch key { case "version 2": v2 = true case "ls-refs": caps.LsRefs = true case "fetch": caps.Fetch = strings.Fields(val) case "object-format": caps.ObjectFormat = val } } if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" { return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat) } c.caps = &caps return caps, nil } type Ref struct { Name string Oid Oid Peeled Oid // set for annotated tags when Peel was asked Symref string // target of HEAD when Symrefs was asked } type LsRefsOpts struct { Prefixes []string // ref-prefix lines Peel bool Symrefs bool } // command=ls-refs func (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if !caps.LsRefs { return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported) } var args []string if opts.Peel { args = append(args, "peel") } if opts.Symrefs { args = append(args, "symrefs") } for _, prefix := range opts.Prefixes { args = append(args, "ref-prefix "+prefix) } resp, err := c.command(ctx, caps, "ls-refs", args) if err != nil { return nil, err } defer resp.Body.Close() p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes}) var refs []Ref for { kind, line, err := p.next() if err != nil { return nil, fmt.Errorf("ls-refs: %w", err) } if kind != pktData { return refs, nil } text := strings.TrimSuffix(string(line), "\n") if msg, ok := strings.CutPrefix(text, "ERR "); ok { return nil, fmt.Errorf("remote: %s", msg) } fields := strings.Split(text, " ") if len(fields) < 2 || !isOid(fields[0]) { return nil, fmt.Errorf("ls-refs: bad line %q", text) } ref := Ref{Oid: Oid(fields[0]), Name: fields[1]} for _, attr := range fields[2:] { if v, ok := strings.CutPrefix(attr, "peeled:"); ok { ref.Peeled = Oid(v) } else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok { ref.Symref = v } } refs = append(refs, ref) } } type FetchOpts struct { Wants []Oid Deepen int // 0 means no deepen line Filter string // e.g. "tree:0", "blob:none"; empty means none } // command=fetch with done and no haves, since there's no local object store func (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") { return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported) } if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") { return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported) } if len(opts.Wants) == 0 { return Objects{}, nil } args := []string{"no-progress", "ofs-delta"} for _, want := range opts.Wants { if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) { return nil, fmt.Errorf("bad object id %q", want) } args = append(args, "want "+string(want)) } if opts.Deepen > 0 { args = append(args, fmt.Sprintf("deepen %d", opts.Deepen)) } if opts.Filter != "" { args = append(args, "filter "+opts.Filter) } args = append(args, "done") resp, err := c.command(ctx, caps, "fetch", args) if err != nil { return nil, err } defer resp.Body.Close() pack, err := packfileSection(newPktReader(resp.Body)) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } return objs, nil } // branches before tags, like git; tags peeled func (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) { if ref == "" || strings.ContainsAny(ref, "\x00\n ") { return "", ErrRefNotFound } candidates := []string{ref} if !strings.HasPrefix(ref, "refs/") { candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref} } refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true}) if err != nil { return "", err } for _, name := range candidates { for _, r := range refs { if r.Name != name { continue } if r.Peeled != "" { return r.Peeled, nil } return r.Oid, nil } } return "", ErrRefNotFound } type File struct { Name string Mode string // octal, as in the tree Oid Oid Data []byte // nil for subtrees and submodules } // one level of dir, blobs filled in func (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) { caps, objs, oid, err := c.walk(ctx, commit, dir) if err != nil { return nil, err } entries, err := c.tree(ctx, caps, objs, oid, "tree:1") if err != nil { return nil, err } files := make([]File, 0, len(entries)) for _, e := range entries { files = append(files, File{Name: e.Name, Mode: e.Mode, Oid: e.Oid}) } if err := c.fillBlobs(ctx, objs, files); err != nil { return nil, err } return files, nil } // everything under dir, named relative to it; the dir tree is wanted unfiltered // so one pack brings it all func (c *Client) ReadTree(ctx context.Context, commit Oid, dir string) ([]File, error) { caps, objs, oid, err := c.walk(ctx, commit, dir) if err != nil { return nil, err } var files []File queue := []File{{Oid: oid}} for len(queue) > 0 { t := queue[0] queue = queue[1:] entries, err := c.tree(ctx, caps, objs, t.Oid, "") if err != nil { return nil, err } for _, e := range entries { f := File{Name: path.Join(t.Name, e.Name), Mode: e.Mode, Oid: e.Oid} if isTreeMode(e.Mode) { queue = append(queue, f) } files = append(files, f) } } if err := c.fillBlobs(ctx, objs, files); err != nil { return nil, err } return files, nil } // middle trees come alone (tree:0) so sibling blobs stay on the server; servers // differ on filter depth for a wanted tree, hence fillBlobs func (c *Client) walk(ctx context.Context, commit Oid, dir string) (Caps, Objects, Oid, error) { caps, err := c.Caps(ctx) if err != nil { return Caps{}, nil, "", err } objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"}) if err != nil { return Caps{}, nil, "", err } obj, ok := objs[commit] if !ok || obj.Kind != KindCommit { return Caps{}, nil, "", fmt.Errorf("fetch of commit %s didn't return it", commit) } tree, err := commitTree(obj.Data) if err != nil { return Caps{}, nil, "", err } for _, name := range strings.Split(dir, "/") { if name == "" { continue } entries, err := c.tree(ctx, caps, objs, tree, "tree:0") if err != nil { return Caps{}, nil, "", err } i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name }) if i < 0 { return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrPathNotFound) } if !isTreeMode(entries[i].Mode) { return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrNotDir) } tree = entries[i].Oid } return caps, objs, tree, nil } func (c *Client) fillBlobs(ctx context.Context, objs Objects, files []File) error { var missing []Oid for _, f := range files { if _, ok := objs[f.Oid]; isBlobMode(f.Mode) && !ok && !slices.Contains(missing, f.Oid) { missing = append(missing, f.Oid) } } if len(missing) > 0 { // wanted objects pass any filter; it keeps the server on its selective path more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"}) if err != nil { return err } maps.Copy(objs, more) } for i, f := range files { if !isBlobMode(f.Mode) { continue } obj, ok := objs[f.Oid] if !ok || obj.Kind != KindBlob { return fmt.Errorf("fetch of blob %s didn't return it", f.Oid) } files[i].Data = obj.Data } return nil } func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) { if _, ok := have[oid]; !ok { objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter}) if err != nil { return nil, err } maps.Copy(have, objs) } obj, ok := have[oid] if !ok || obj.Kind != KindTree { return nil, fmt.Errorf("fetch of tree %s didn't return it", oid) } return parseTree(obj.Data, caps.ObjectFormat) } func isBlobMode(mode string) bool { mode = strings.TrimLeft(mode, "0") return mode != "40000" && mode != "160000" } func isTreeMode(mode string) bool { return strings.TrimLeft(mode, "0") == "40000" } func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) { var body bytes.Buffer writePkt(&body, "command="+command+"\n") // git refuses non-sha1 unless the format is echoed back if caps.ObjectFormat != "sha1" { writePkt(&body, "object-format="+caps.ObjectFormat+"\n") } writeDelim(&body) for _, arg := range args { if err := writePkt(&body, arg+"\n"); err != nil { return nil, err } } writeFlush(&body) u := c.url + "/git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body) if err != nil { return nil, err } req.Header.Set("Git-Protocol", "version=2") req.Header.Set("Content-Type", "application/x-git-upload-pack-request") req.Header.Set("Accept", "application/x-git-upload-pack-result") resp, err := c.http.Do(req) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() return nil, httpError(resp) } return resp, nil } // knot2 says why in a plain text body, like a want it won't serve func httpError(resp *http.Response) error { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status) if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?")) } return err }