From f8f30045ec38dcbb4e8f35807bd18528581bf03d Mon Sep 17 00:00:00 2001 From: Willow Sparks Date: Mon, 9 Nov 2026 18:22:43 +0000 Subject: [PATCH] feat(treesitter): add Go tree-sitter provider helper --- runtime/plugins/treesitter/repo.json | 7 + runtime/plugins/treesitter/treesitter.lua | 299 +++++++++ treesitter/.gitignore | 3 + treesitter/README.md | 253 ++++++++ treesitter/cmd/micro-treesitter/main.go | 81 +++ treesitter/go.mod | 10 + treesitter/go.sum | 36 + treesitter/internal/protocol/protocol.go | 91 +++ treesitter/internal/protocol/protocol_test.go | 69 ++ treesitter/internal/tsprovider/provider.go | 614 ++++++++++++++++++ .../internal/tsprovider/provider_test.go | 309 +++++++++ treesitter/queries/go/LICENSE | 21 + treesitter/queries/go/highlights.scm | 123 ++++ treesitter/testdata/demo.py | 36 + treesitter/testdata/lua_harness.lua | 109 ++++ treesitter/testdata/sample.go | 22 + 16 files changed, 2083 insertions(+) create mode 100644 runtime/plugins/treesitter/repo.json create mode 100644 runtime/plugins/treesitter/treesitter.lua create mode 100644 treesitter/.gitignore create mode 100644 treesitter/README.md create mode 100644 treesitter/cmd/micro-treesitter/main.go create mode 100644 treesitter/go.mod create mode 100644 treesitter/go.sum create mode 100644 treesitter/internal/protocol/protocol.go create mode 100644 treesitter/internal/protocol/protocol_test.go create mode 100644 treesitter/internal/tsprovider/provider.go create mode 100644 treesitter/internal/tsprovider/provider_test.go create mode 100644 treesitter/queries/go/LICENSE create mode 100644 treesitter/queries/go/highlights.scm create mode 100755 treesitter/testdata/demo.py create mode 100644 treesitter/testdata/lua_harness.lua create mode 100644 treesitter/testdata/sample.go diff --git a/runtime/plugins/treesitter/repo.json b/runtime/plugins/treesitter/repo.json new file mode 100644 index 00000000..1f06d373 --- /dev/null +++ b/runtime/plugins/treesitter/repo.json @@ -0,0 +1,7 @@ +[ + { + "Name": "treesitter", + "Description": "Tree-sitter highlighting for Go through an external helper", + "Website": "https://github.com/tree-sitter/tree-sitter" + } +] diff --git a/runtime/plugins/treesitter/treesitter.lua b/runtime/plugins/treesitter/treesitter.lua new file mode 100644 index 00000000..b3e38855 --- /dev/null +++ b/runtime/plugins/treesitter/treesitter.lua @@ -0,0 +1,299 @@ +VERSION = "0.1.0" + +-- Tree-sitter highlighting for Go, driving an out-of-process helper. +-- The helper owns parsing, the grammar, the query, and capture mapping; this +-- plugin owns the process, request framing, and reply generation checks. + +local syntax = import("micro/syntax") +local shell = import("micro/shell") +local micro = import("micro") +local os = import("os") + +local PROVIDER_NAME = "treesitter-go" +local PROVIDER_FILETYPE = "go" + +local job = nil +local disabled = false +local logged = false +local providerRegistered = false +local spawnGeneration = 0 +local helperPath = nil + +local outputBuffer = "" +local collecting = nil +local replies = {} +local bufferIDs = {} +local nextBufferID = 0 +local lastHelperError = nil + +local function logOnce(message) + if not logged then + logged = true + micro.Log("treesitter: " .. message) + end +end + +local function nextLine() + local newline = outputBuffer:find("\n", 1, true) + if not newline then + return nil + end + local line = outputBuffer:sub(1, newline - 1) + outputBuffer = outputBuffer:sub(newline + 1) + return line +end + +local function deliver(id, generation, spans, errMessage) + local pending = replies[id] + if pending == nil then + return + end + if tostring(pending.generation) ~= tostring(generation) then + return + end + replies[id] = nil + if errMessage ~= nil then + if errMessage ~= lastHelperError then + lastHelperError = errMessage + micro.Log("treesitter: helper error: " .. errMessage) + end + -- A failed request carries no spans. Clear the previous ones so the + -- buffer is never coloured from a stale generation. + pcall(pending.reply, {}) + return + end + lastHelperError = nil + local ok, err = pcall(pending.reply, spans) + if not ok then + micro.Log("treesitter: reply failed: " .. tostring(err)) + end +end + +local function stopJob(generation) + if generation ~= spawnGeneration then + return + end + spawnGeneration = spawnGeneration + 1 + if job ~= nil then + shell.JobStop(job) + job = nil + end + replies = {} + collecting = nil + outputBuffer = "" +end + +local function protocolFailure(generation, message) + if generation ~= spawnGeneration then + return + end + logOnce("helper protocol error: " .. message) + disabled = true + stopJob(generation) +end + +local process -- forward declaration + +local function handleHeader(generation, line) + local kind, id, gen, rest = line:match("^(%S+)\t(%S+)\t(%S+)\t(.*)$") + if kind == nil then + protocolFailure(generation, "malformed header: " .. line) + return + end + if kind == "SPANS" then + local count = tonumber(rest) + if count == nil or count < 0 or count ~= math.floor(count) then + protocolFailure(generation, "bad span count: " .. rest) + return + end + if count == 0 then + deliver(id, gen, {}) + return + end + collecting = { id = id, generation = gen, count = count, spans = {} } + elseif kind == "ERR" then + deliver(id, gen, nil, rest) + else + protocolFailure(generation, "unknown frame: " .. kind) + end +end + +local function handleSpanLine(generation, line) + local lineNo, startByte, endByte, group = + line:match("^(%-?%d+)\t(%-?%d+)\t(%-?%d+)\t(.*)$") + if lineNo == nil then + protocolFailure(generation, "malformed span line: " .. line) + return + end + local spans = collecting.spans + local n = #spans + spans[n + 1] = tonumber(lineNo) + spans[n + 2] = tonumber(startByte) + spans[n + 3] = tonumber(endByte) + spans[n + 4] = group + if #spans >= collecting.count * 4 then + local finished = collecting + collecting = nil + deliver(finished.id, finished.generation, finished.spans) + end +end + +process = function(generation) + if generation ~= spawnGeneration then + return + end + while true do + local line = nextLine() + if line == nil then + return + end + if collecting ~= nil then + handleSpanLine(generation, line) + else + handleHeader(generation, line) + end + if generation ~= spawnGeneration then + return + end + end +end + +local function bufferID(ctx) + if ctx.id ~= nil then + return tostring(ctx.id) + end + -- Older core builds may omit ctx.id. Derive a stable numeric id from the + -- buffer handle so the helper can still key its parse state. + local key = tostring(ctx.buf) + local id = bufferIDs[key] + if id == nil then + nextBufferID = nextBufferID + 1 + id = nextBufferID + bufferIDs[key] = id + end + return tostring(id) +end + +local function request(ctx, reply) + if disabled or job == nil then + reply({}) + return + end + local id = bufferID(ctx) + local generation = ctx.generation or 0 + local filetype = ctx.filetype or PROVIDER_FILETYPE + local text = ctx.text or "" + replies[id] = { generation = generation, reply = reply } + local frame = "REQ\t" .. id .. "\t" .. tostring(generation) .. "\t" + .. filetype .. "\t" .. tostring(#text) .. "\n" .. text + shell.JobSend(job, frame) +end + +local function startHelper() + spawnGeneration = (spawnGeneration or 0) + 1 + local generation = spawnGeneration + outputBuffer = "" + collecting = nil + replies = {} + + local function onStdout(data, args) + if generation ~= spawnGeneration then + return + end + outputBuffer = outputBuffer .. data + process(generation) + end + + local function onStderr(data, args) + if generation ~= spawnGeneration then + return + end + local detail = tostring(data or "") + if #detail > 400 then + detail = detail:sub(1, 400) .. "..." + end + lastHelperError = detail + micro.Log("treesitter: " .. detail) + end + + local function onExit(output, args) + if generation ~= spawnGeneration then + return + end + job = nil + disabled = true + collecting = nil + outputBuffer = "" + replies = {} + local detail = tostring(output or "") + if #detail > 400 then + detail = detail:sub(1, 400) .. "..." + end + detail = detail:gsub("%s+$", "") + if detail ~= "" then + logOnce("helper exited (" .. helperPath .. "): " .. detail) + else + logOnce("could not run helper " .. helperPath .. " (set MICRO_TS or install micro-treesitter)") + end + end + + job = shell.JobSpawn(helperPath, {}, onStdout, onStderr, onExit) +end + +function init() + if syntax == nil then + logOnce("this micro build has no micro/syntax provider API") + disabled = true + return + end + if job ~= nil then + return + end + + disabled = false + logged = false + lastHelperError = nil + helperPath = os.Getenv("MICRO_TS") + if helperPath == nil or helperPath == "" then + helperPath = "micro-treesitter" + end + + startHelper() + + if providerRegistered then + return + end + local ok, err = pcall(syntax.register, { + name = PROVIDER_NAME, + filetype = PROVIDER_FILETYPE, + filename = "\\.go$", + priority = 100, + request = request, + }) + if not ok then + disabled = true + logOnce("register failed: " .. tostring(err)) + if job ~= nil then + stopJob(spawnGeneration) + end + else + providerRegistered = true + end +end + +function deinit() + spawnGeneration = spawnGeneration + 1 + if job ~= nil then + shell.JobStop(job) + job = nil + end + outputBuffer = "" + collecting = nil + replies = {} + bufferIDs = {} + nextBufferID = 0 + disabled = false + logged = false + providerRegistered = false + lastHelperError = nil +end diff --git a/treesitter/.gitignore b/treesitter/.gitignore new file mode 100644 index 00000000..96f09079 --- /dev/null +++ b/treesitter/.gitignore @@ -0,0 +1,3 @@ +/micro-treesitter +/.gocache/ +/.freshmod/ diff --git a/treesitter/README.md b/treesitter/README.md new file mode 100644 index 00000000..64694230 --- /dev/null +++ b/treesitter/README.md @@ -0,0 +1,253 @@ +# micro-treesitter + +An out-of-process tree-sitter highlighter for the micro fork's +`micro/syntax` provider API. The Go helper parses a document, runs a +tree-sitter highlight query, and returns single-line byte spans. The Lua +plugin owns the helper process and forwards spans to the editor. + +This is the first provider extension for the fork: it keeps cgo, grammars, +queries, and capture names out of the editor core. Removing the plugin and +the helper removes tree-sitter entirely. + +## Layout + +| Path | Purpose | +|---|---| +| `cmd/micro-treesitter/` | helper binary: stdin/stdout protocol loop | +| `internal/protocol/` | frame reader/writer | +| `internal/tsprovider/` | grammars, queries, parse sessions, span flattening | +| `queries/go/highlights.scm` | Go highlight query, copied from tree-sitter-go v0.25.0 (MIT) | +| `runtime/plugins/treesitter/` | Lua plugin and `repo.json` | +| `testdata/` | Go sample, standalone helper driver, Lua harness | + +## Build and test + +The helper uses cgo through `github.com/tree-sitter/go-tree-sitter` and the +official Go grammar binding: + +```sh +nix shell nixpkgs#go_1_27 -c bash -lc 'CGO_ENABLED=1 go build ./cmd/micro-treesitter' +nix shell nixpkgs#go_1_27 -c bash -lc 'CGO_ENABLED=1 go test ./...' +``` + +A C compiler must be available to cgo. On this machine the build was also +verified with `nix shell nixpkgs#go_1_27 nixpkgs#gcc`; `CGO_CFLAGS=-O2` is +optional and only affects parse throughput. + +Drive the helper directly: + +```sh +python3 testdata/demo.py ./micro-treesitter testdata/sample.go go +``` + +Check the Lua state machine without micro: + +```sh +lua5_1 testdata/lua_harness.lua +``` + +## Grammar and query lookup + +The helper reads `//highlights.scm`. The grammar +directory is resolved in this order: + +1. `--grammar-dir` flag +2. `MICRO_TS_GRAMMAR_DIR` +3. `./queries` + +`/.scm` and `/highlights.scm` are accepted +as fallbacks, so a single-grammar directory also works. A missing or invalid +query becomes an `ERR` reply for that filetype; the helper keeps running. + +Only the Go grammar is linked in this version. `filetype` is mapped to a +grammar through `internal/tsprovider.filetypeGrammar`; adding a language means +adding its Go module to `go.mod`, one `grammarLanguages` entry, one mapping +entry, a query directory, and a second Lua `syntax.register` call. + +## Wire protocol + +Requests and replies are newline-framed headers followed by an exact byte +payload or a known number of lines. All integers are decimal. Fields are +separated by one tab. Text is UTF-8 but is never decoded into runes. + +Request: + +```text +REQ\t\t\t\t\n + +``` + +Reply: + +```text +SPANS\t\t\t\n +\t\t\t\n (count lines) +``` + +Error: + +```text +ERR\t\t\t\n +``` + +`line` is 0-based. `startByte` and `endByte` are 0-based byte offsets within +that line, end exclusive. Lines are `\n`-separated in the payload and contain +no line terminator. + +The helper keeps parse state per `id`: the previous text and tree are reused +through a single contiguous `InputEdit` computed from the common prefix and +suffix, then reparsed. If tree-sitter declines to produce a tree, the helper +falls back to a full parse. Sessions are evicted least-recently-used beyond +16 ids. Parse sessions are not shared across ids, and requests are processed +sequentially. + +## Span construction + +- Multi-line nodes are split per line. The first line runs from the node's + start byte to the end of the line; middle lines are covered fully; the last + line runs from byte 0 to the node's end column. A node that ends at column 0 + emits nothing on that line. +- Empty and zero-width spans are dropped. +- Overlapping captures are painted in precedence order. Later captures win, + except for identical ranges: the more specific capture name wins, then the + earlier pattern. This keeps `@function.builtin` over `@function`, and + `@function` over a later `@variable` capture of the same node. +- Adjacent spans with the same group are merged. +- Output is sorted by line, then start byte. + +Capture names are resolved by longest dotted prefix through +`internal/tsprovider.captureGroups`. The groups emitted are limited to names +that micro colorschemes define (`comment`, `identifier`, `constant`, +`statement`, `symbol`, `preproc`, `type`, `special`, `error`, `todo` and their +subgroups). Unknown captures produce no span. The main mappings: + +| tree-sitter capture | micro group | +|---|---| +| `comment*` | `comment` | +| `string*`, `character*` | `constant.string` | +| `string.escape`, `escape` | `constant.specialChar` | +| `number*` | `constant.number` | +| `constant`, `constant.macro` | `constant` | +| `constant.builtin`, `boolean` | `constant.bool` | +| `keyword*` | `statement` (Go keywords are refined below) | +| `keyword.function/type/import/package/declaration` | `preproc` | +| `function*`, `variable*`, `parameter`, `property`, `identifier` | `identifier` | +| `function.builtin`, `function.macro` | `preproc` | +| `type*`, `constructor`, `module`, `namespace`, `attribute` | `type` | +| `operator` | `symbol.operator` | +| `punctuation.bracket` | `symbol.brackets` | +| `punctuation*` | `symbol` | +| `preproc` | `preproc` | +| `tag` | `symbol.tag` | +| `error` | `error` | +| `todo`, `comment.todo`, `comment.warning` | `todo` | + +The Go grammar emits one generic `@keyword` capture for every keyword. The +helper refines the keyword text using the old `go.yaml` vocabulary: `package`, +`import`, `const`, `var`, `type`, `struct`, `func`, `interface`, `map`, +`chan`, `defer` become `preproc`; `break`, `case`, `continue`, `default`, +`go`, `goto`, `range`, `return`, `fallthrough` become `special`; `else`, +`for`, `if`, `switch`, `select` stay `statement`. + +## Lua plugin + +`runtime/plugins/treesitter/treesitter.lua` registers one provider: + +```lua +{ + name = "treesitter-go", + filetype = "go", + filename = "\\.go$", + priority = 100, + request = request, -- request(ctx, reply) +} +``` + +On `init` it spawns `$MICRO_TS` if set, otherwise `micro-treesitter` from +`PATH`. It logs once and disables highlighting when the helper exits before +serving requests. On `deinit` it stops the job and drops pending replies. + +The plugin keeps one helper process for all buffers and routes replies by +`ctx.id`. A pending reply is stored per id with its generation; a `SPANS` or +`ERR` frame is delivered only when the generation matches. Chunk boundaries +from the process pipe are arbitrary; the plugin buffers stdout and reassembles +frames line by line. + +Install the helper on `PATH` through your package manager, or set: + +```sh +export MICRO_TS=/path/to/micro-treesitter +export MICRO_TS_GRAMMAR_DIR=/path/to/grammar-bundle +``` + +Copy `runtime/plugins/treesitter/` into `$MICRO_CONFIG_HOME/plug/` (or ship it +in the editor's embedded runtime plugins). The plugin directory must contain +only `treesitter.lua` and `repo.json`; micro executes every `.lua` file it +finds there. + +The plugin assumes the core provides `micro/syntax` with +`register{...}` and `request(ctx, reply)`, and that `ctx` carries `id`, +`generation`, `text`, `filetype`, `buf`, `path`, `first`, `last`, and `cause`. +`ctx.id` must be an integer that is stable for a buffer: the helper parses it +as the decimal request key. `MICRO_TS` is one executable path; arguments are +not split, so use a wrapper script if extra flags are needed. + +## Tested sample + +`testdata/sample.go` exercises a multi-line block comment, a backtick raw +string, a struct tag, and UTF-8 identifiers. The helper's reply begins: + +```text +SPANS 1 1 39 +0 0 7 preproc +2 0 42 comment +3 0 16 comment +4 0 17 comment +5 0 11 comment +6 0 4 preproc +6 5 11 type +6 12 18 preproc +7 1 4 identifier +7 6 12 type +7 13 35 constant.string +11 0 4 preproc +... +16 1 7 special +... +``` + +`Héllo` (6 bytes, 5 runes) is reported as byte span `[5,11)`, and both the +three-line block comment starting at line 3 and the raw string on lines 12-14 +are split across their lines. + +## Deliberate choices and deviations + +- `ERR` replies with a matching generation call `reply({})` after logging, so + a failed refresh cannot leave spans from a previous generation on screen. +- `ctx.id` is used as the numeric request key. If a core build omits it, the + plugin derives a stable numeric id per buffer handle; that keeps the helper + usable but is intended only as a compatibility fallback. +- A malformed request header is answered with `ERR` and the helper exits, + because the unread payload length is unknown. The plugin never sends one. +- The helper reparses incrementally even though v1 permits a full reparse on + every request. A differential test compares incremental spans with a fresh + parse over random edits, including UTF-8 and invalid UTF-8 bytes. +- Request handling is sequential and has no timeout. A helper that hangs will + eventually block `JobSend` in the editor. Process supervision beyond the + exit callback (restart with backoff, watchdog) is not implemented. +- `queries/go/highlights.scm` and `queries/go/LICENSE` are copied from + tree-sitter-go v0.25.0. + +## Known gaps + +- Only Go is linked. Other filetypes get an `ERR`, and the plugin only + registers `go`. +- Queries run over the whole document; there is no windowed query, no + debounce policy in the plugin (core owns the request cadence), and no + injections or embedded languages. +- One helper serves all buffers. A helper crash disables highlighting until + the plugin is reloaded. +- Spans are generated from the whole tree on every request; the previous tree + only saves reparsing, not re-querying. +- No grammar manifest. Distribution must provide `queries//highlights.scm` + and set `MICRO_TS_GRAMMAR_DIR`; there is no compiled default path. diff --git a/treesitter/cmd/micro-treesitter/main.go b/treesitter/cmd/micro-treesitter/main.go new file mode 100644 index 00000000..987aaabd --- /dev/null +++ b/treesitter/cmd/micro-treesitter/main.go @@ -0,0 +1,81 @@ +// Command micro-treesitter is the tree-sitter highlight helper for micro. +// +// It reads newline-framed requests from stdin and writes span replies to +// stdout. See ../../README.md for the wire format. +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + + "micro-treesitter/internal/protocol" + "micro-treesitter/internal/tsprovider" +) + +var defaultGrammarDir = "./queries" + +func main() { + grammarDirFlag := flag.String("grammar-dir", "", "directory containing /highlights.scm (default $MICRO_TS_GRAMMAR_DIR or "+defaultGrammarDir+")") + verbose := flag.Bool("verbose", false, "log request errors to stderr") + flag.Parse() + + grammarDir := *grammarDirFlag + if grammarDir == "" { + grammarDir = os.Getenv("MICRO_TS_GRAMMAR_DIR") + } + if grammarDir == "" { + grammarDir = defaultGrammarDir + } + + engine := tsprovider.New(grammarDir) + defer engine.Close() + + reader := bufio.NewReaderSize(os.Stdin, 1<<20) + writer := bufio.NewWriterSize(os.Stdout, 1<<20) + + if *verbose { + log.Printf("micro-treesitter: grammar dir %s", grammarDir) + } + + for { + request, err := protocol.ReadRequest(reader) + if err != nil { + if errors.Is(err, io.EOF) { + return + } + var headerErr *protocol.HeaderError + if errors.As(err, &headerErr) { + _ = protocol.WriteError(writer, headerErr.ID, headerErr.Gen, headerErr.Message) + if *verbose { + log.Printf("micro-treesitter: %v", err) + } + // The unread payload length is unknown, so the stream cannot be + // resynchronised. Exit after reporting the error. + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "micro-treesitter: %v\n", err) + os.Exit(1) + } + + spans, err := engine.Spans(request.ID, request.Filetype, request.Payload) + if err != nil { + if *verbose { + log.Printf("micro-treesitter: request %d (%s): %v", request.ID, request.Filetype, err) + } + if err := protocol.WriteError(writer, request.ID, request.Gen, err.Error()); err != nil { + fmt.Fprintf(os.Stderr, "micro-treesitter: write: %v\n", err) + os.Exit(1) + } + continue + } + if err := protocol.WriteSpans(writer, request.ID, request.Gen, spans); err != nil { + fmt.Fprintf(os.Stderr, "micro-treesitter: write: %v\n", err) + os.Exit(1) + } + } +} diff --git a/treesitter/go.mod b/treesitter/go.mod new file mode 100644 index 00000000..9bd23601 --- /dev/null +++ b/treesitter/go.mod @@ -0,0 +1,10 @@ +module micro-treesitter + +go 1.27.0 + +require ( + github.com/tree-sitter/go-tree-sitter v0.25.0 + github.com/tree-sitter/tree-sitter-go v0.25.0 +) + +require github.com/mattn/go-pointer v0.0.1 // indirect diff --git a/treesitter/go.sum b/treesitter/go.sum new file mode 100644 index 00000000..7bc1cf2a --- /dev/null +++ b/treesitter/go.sum @@ -0,0 +1,36 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU= +github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c= +github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw= +github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w= +github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ= +github.com/tree-sitter/tree-sitter-go v0.25.0 h1:cEB0Q3LHgZtS+ECHx9wcP7AwzoOddJFQCVmytX42cVU= +github.com/tree-sitter/tree-sitter-go v0.25.0/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040= +github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc= +github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og= +github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig= +github.com/tree-sitter/tree-sitter-java v0.23.5/go.mod h1:NRKlI8+EznxA7t1Yt3xtraPk1Wzqh3GAIC46wxvc320= +github.com/tree-sitter/tree-sitter-javascript v0.23.1 h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo= +github.com/tree-sitter/tree-sitter-javascript v0.23.1/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs= +github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ= +github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo= +github.com/tree-sitter/tree-sitter-php v0.23.11 h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8= +github.com/tree-sitter/tree-sitter-php v0.23.11/go.mod h1:T/kbfi+UcCywQfUNAJnGTN/fMSUjnwPXA8k4yoIks74= +github.com/tree-sitter/tree-sitter-python v0.23.6 h1:qHnWFR5WhtMQpxBZRwiaU5Hk/29vGju6CVtmvu5Haas= +github.com/tree-sitter/tree-sitter-python v0.23.6/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= +github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo= +github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= +github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90= +github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/treesitter/internal/protocol/protocol.go b/treesitter/internal/protocol/protocol.go new file mode 100644 index 00000000..730c2fb5 --- /dev/null +++ b/treesitter/internal/protocol/protocol.go @@ -0,0 +1,91 @@ +// Package protocol implements the micro-treesitter stdin/stdout framing. +package protocol + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" + + "micro-treesitter/internal/tsprovider" +) + +// MaxPayload bounds one request body. A larger declared length is rejected +// without reading it, because the stream cannot be resynchronised safely. +const MaxPayload = 64 << 20 + +// Request is one REQ frame: a header plus its exact payload. +type Request struct { + ID int64 + Gen int64 + Filetype string + Payload []byte +} + +// HeaderError reports a frame that could not be parsed. ID and Gen are the +// best-effort values from the header, so the caller can send an ERR frame. +type HeaderError struct { + ID int64 + Gen int64 + Message string +} + +func (e *HeaderError) Error() string { return e.Message } + +// ReadRequest reads one request. It returns io.EOF only for a clean end of +// input before any header byte. +func ReadRequest(r *bufio.Reader) (*Request, error) { + header, err := r.ReadString('\n') + if err != nil { + if err == io.EOF && header == "" { + return nil, io.EOF + } + return nil, fmt.Errorf("read header: %w", err) + } + header = strings.TrimSuffix(header, "\n") + header = strings.TrimSuffix(header, "\r") + fields := strings.Split(header, "\t") + if len(fields) < 3 { + return nil, &HeaderError{Message: "malformed header: want REQ\\t\\t\\t\\t"} + } + id, idErr := strconv.ParseInt(fields[1], 10, 64) + gen, _ := strconv.ParseInt(fields[2], 10, 64) + if idErr != nil { + return nil, &HeaderError{Gen: gen, Message: "malformed header: invalid id"} + } + if len(fields) != 5 || fields[0] != "REQ" { + return nil, &HeaderError{ID: id, Gen: gen, Message: "malformed header: want REQ\\t\\t\\t\\t"} + } + length, lengthErr := strconv.ParseInt(fields[4], 10, 64) + if lengthErr != nil || length < 0 || length > MaxPayload { + return nil, &HeaderError{ID: id, Gen: gen, Message: "malformed header: invalid byte_len"} + } + payload := make([]byte, length) + if _, err := io.ReadFull(r, payload); err != nil { + return nil, fmt.Errorf("read payload of %d bytes: %w", length, err) + } + return &Request{ID: id, Gen: gen, Filetype: fields[3], Payload: payload}, nil +} + +// WriteSpans writes one SPANS frame and its span lines. +func WriteSpans(w *bufio.Writer, id, gen int64, spans []tsprovider.Span) error { + if _, err := fmt.Fprintf(w, "SPANS\t%d\t%d\t%d\n", id, gen, len(spans)); err != nil { + return err + } + for _, span := range spans { + if _, err := fmt.Fprintf(w, "%d\t%d\t%d\t%s\n", span.Line, span.Start, span.End, span.Group); err != nil { + return err + } + } + return w.Flush() +} + +// WriteError writes one ERR frame. +func WriteError(w *bufio.Writer, id, gen int64, message string) error { + message = strings.NewReplacer("\t", " ", "\r", " ", "\n", " ").Replace(message) + if _, err := fmt.Fprintf(w, "ERR\t%d\t%d\t%s\n", id, gen, message); err != nil { + return err + } + return w.Flush() +} diff --git a/treesitter/internal/protocol/protocol_test.go b/treesitter/internal/protocol/protocol_test.go new file mode 100644 index 00000000..bbdbbeb6 --- /dev/null +++ b/treesitter/internal/protocol/protocol_test.go @@ -0,0 +1,69 @@ +package protocol + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "strings" + "testing" + + "micro-treesitter/internal/tsprovider" +) + +func TestReadRequest(t *testing.T) { + payload := "package main\n\nfunc main() {}\n" + frame := fmt.Sprintf("REQ\t42\t7\tgo\t%d\n%s", len(payload), payload) + request, err := ReadRequest(bufio.NewReader(strings.NewReader(frame))) + if err != nil { + t.Fatal(err) + } + if request.ID != 42 || request.Gen != 7 || request.Filetype != "go" { + t.Fatalf("header fields: %+v", request) + } + if string(request.Payload) != payload { + t.Fatalf("payload: %q", request.Payload) + } + if _, err := ReadRequest(bufio.NewReader(strings.NewReader(""))); !errors.Is(err, io.EOF) { + t.Fatalf("empty input: want io.EOF, got %v", err) + } +} + +func TestReadRequestMalformed(t *testing.T) { + _, err := ReadRequest(bufio.NewReader(strings.NewReader("REQ\t3\t9\tgo\tx\n"))) + var headerErr *HeaderError + if !errors.As(err, &headerErr) { + t.Fatalf("want HeaderError, got %v", err) + } + if headerErr.ID != 3 || headerErr.Gen != 9 { + t.Fatalf("best-effort ids: %+v", headerErr) + } +} + +func TestWriteSpans(t *testing.T) { + var buffer bytes.Buffer + writer := bufio.NewWriter(&buffer) + spans := []tsprovider.Span{ + {Line: 0, Start: 0, End: 7, Group: "preproc"}, + {Line: 2, Start: 5, End: 9, Group: "constant.string"}, + } + if err := WriteSpans(writer, 4, 11, spans); err != nil { + t.Fatal(err) + } + want := "SPANS\t4\t11\t2\n0\t0\t7\tpreproc\n2\t5\t9\tconstant.string\n" + if buffer.String() != want { + t.Fatalf("got %q want %q", buffer.String(), want) + } +} + +func TestWriteErrorSanitisesMessage(t *testing.T) { + var buffer bytes.Buffer + writer := bufio.NewWriter(&buffer) + if err := WriteError(writer, 1, 2, "bad\trequest\nsecond line"); err != nil { + t.Fatal(err) + } + if got := buffer.String(); got != "ERR\t1\t2\tbad request second line\n" { + t.Fatalf("got %q", got) + } +} diff --git a/treesitter/internal/tsprovider/provider.go b/treesitter/internal/tsprovider/provider.go new file mode 100644 index 00000000..796a88e5 --- /dev/null +++ b/treesitter/internal/tsprovider/provider.go @@ -0,0 +1,614 @@ +// Package tsprovider turns tree-sitter highlight queries into micro spans. +// +// The package owns one parser and one compiled query per filetype, plus a +// per-request-id parse session. Parse sessions are reused with a single +// contiguous InputEdit derived from the common prefix and suffix of the old +// and new document; a failed incremental parse falls back to a full parse. +package tsprovider + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + sitter "github.com/tree-sitter/go-tree-sitter" + tsgo "github.com/tree-sitter/tree-sitter-go/bindings/go" +) + +// Span is one half-open, single-line byte range. Line is 0-based; Start and +// End are byte offsets within that line, End exclusive. +type Span struct { + Line int + Start int + End int + Group string +} + +// filetypeGrammar names the tree-sitter grammar used for a micro filetype. +var filetypeGrammar = map[string]string{ + "go": "go", +} + +// grammarLanguages links a grammar id to its compiled language. Adding a +// grammar means adding a module to go.mod and one entry here. +var grammarLanguages = map[string]func() *sitter.Language{ + "go": func() *sitter.Language { + return sitter.NewLanguage(tsgo.Language()) + }, +} + +// captureGroups maps a tree-sitter capture name to a micro highlight group. +// Names are matched exactly first, then by dropping the last dotted +// component. Only group names whose first component exists in micro +// colorschemes (comment, identifier, constant, statement, symbol, preproc, +// type, special, error, todo) are emitted; config.GetColor falls back to the +// default style for anything else. +var captureGroups = map[string]string{ + "comment": "comment", + "comment.documentation": "comment", + "comment.error": "error", + "comment.warning": "todo", + "comment.todo": "todo", + "todo": "todo", + + "string": "constant.string", + "string.escape": "constant.specialChar", + "string.special": "constant.specialChar", + "string.special.key": "constant.string", + "string.regexp": "constant.string", + "escape": "constant.specialChar", + "character": "constant.string", + "character.special": "constant.specialChar", + + "number": "constant.number", + "number.float": "constant.number", + "number.integer": "constant.number", + "float": "constant.number", + + "boolean": "constant.bool", + "constant": "constant", + "constant.builtin": "constant.bool", + "constant.bool": "constant.bool", + "constant.macro": "constant", + + "keyword": "statement", + "keyword.return": "statement", + "keyword.conditional": "statement", + "keyword.control": "statement", + "keyword.exception": "statement", + "keyword.repeat": "statement", + "keyword.coroutine": "statement", + "keyword.function": "preproc", + "keyword.type": "preproc", + "keyword.import": "preproc", + "keyword.package": "preproc", + "keyword.declaration": "preproc", + "keyword.operator": "symbol.operator", + + "function": "identifier", + "function.call": "identifier", + "function.method": "identifier", + "function.builtin": "preproc", + "function.macro": "preproc", + + "type": "type", + "type.builtin": "type", + "type.qualifier": "statement", + "constructor": "type", + "module": "type", + "namespace": "type", + "attribute": "type", + "label": "statement", + + "property": "identifier", + "variable": "identifier", + "variable.builtin": "identifier", + "variable.parameter": "identifier", + "parameter": "identifier", + "identifier": "identifier", + + "operator": "symbol.operator", + "punctuation": "symbol", + "punctuation.bracket": "symbol.brackets", + "punctuation.delimiter": "symbol", + "punctuation.special": "symbol", + "symbol": "symbol", + "tag": "symbol.tag", + "preproc": "preproc", + "error": "error", +} + +// MapCapture resolves a capture name to a micro group, or "" when the +// capture is unknown. Longest dotted prefix wins. +func MapCapture(name string) string { + for name != "" { + if group, ok := captureGroups[name]; ok { + return group + } + i := strings.LastIndexByte(name, '.') + if i < 0 { + break + } + name = name[:i] + } + return "" +} + +// goKeywordGroups refines the grammar's single @keyword capture into the +// vocabulary the old go.yaml used: declarations are preproc, control-flow +// keywords are special, and the rest stay statement. +var goKeywordGroups = map[string]string{ + "package": "preproc", "import": "preproc", "const": "preproc", "var": "preproc", + "type": "preproc", "struct": "preproc", "func": "preproc", "interface": "preproc", + "map": "preproc", "chan": "preproc", "defer": "preproc", + "break": "special", "case": "special", "continue": "special", "default": "special", + "go": "special", "goto": "special", "range": "special", "return": "special", + "fallthrough": "special", + "else": "statement", "for": "statement", "if": "statement", + "switch": "statement", "select": "statement", +} + +type grammar struct { + id string + parser *sitter.Parser + language *sitter.Language + query *sitter.Query + captures []string +} + +func (g *grammar) close() { + if g.query != nil { + g.query.Close() + } + if g.parser != nil { + g.parser.Close() + } +} + +type session struct { + filetype string + text []byte + tree *sitter.Tree + used uint64 +} + +func (s *session) closeTree() { + if s.tree != nil { + s.tree.Close() + s.tree = nil + } +} + +// Engine owns grammars and parse sessions. It is not safe for concurrent use. +type Engine struct { + grammarDir string + grammars map[string]*grammar + failures map[string]error + sessions map[int64]*session + maxSessions int + tick uint64 +} + +// New returns an engine that looks for //highlights.scm. +func New(grammarDir string) *Engine { + return &Engine{ + grammarDir: grammarDir, + grammars: make(map[string]*grammar), + failures: make(map[string]error), + sessions: make(map[int64]*session), + maxSessions: 16, + } +} + +// Close releases all parsers, queries, and cached trees. +func (e *Engine) Close() { + for _, g := range e.grammars { + g.close() + } + for _, s := range e.sessions { + s.closeTree() + } + e.grammars = nil + e.sessions = nil +} + +// Spans reparses text for the request id and returns flattened micro spans. +// id identifies the parse session (normally one buffer); filetype selects +// the grammar and query. +func (e *Engine) Spans(id int64, filetype string, text []byte) ([]Span, error) { + g, err := e.grammarFor(filetype) + if err != nil { + return nil, err + } + s := e.touchSession(id) + tree, err := e.parse(g, s, filetype, text) + if err != nil { + return nil, err + } + return g.spans(tree, text), nil +} + +func (e *Engine) grammarFor(filetype string) (*grammar, error) { + if g, ok := e.grammars[filetype]; ok { + return g, nil + } + if err, ok := e.failures[filetype]; ok { + return nil, err + } + g, err := e.loadGrammar(filetype) + if err != nil { + e.failures[filetype] = err + return nil, err + } + e.grammars[filetype] = g + return g, nil +} + +func (e *Engine) loadGrammar(filetype string) (*grammar, error) { + grammarID, ok := filetypeGrammar[filetype] + if !ok { + return nil, fmt.Errorf("unsupported filetype %q: no linked tree-sitter grammar", filetype) + } + newLanguage, ok := grammarLanguages[grammarID] + if !ok { + return nil, fmt.Errorf("filetype %q maps to unknown grammar %q", filetype, grammarID) + } + language := newLanguage() + if language == nil { + return nil, fmt.Errorf("grammar %q returned a nil language", grammarID) + } + queryPath, err := e.findQuery(grammarID) + if err != nil { + return nil, err + } + querySource, err := os.ReadFile(queryPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", queryPath, err) + } + query, queryErr := sitter.NewQuery(language, string(querySource)) + if queryErr != nil { + return nil, fmt.Errorf("%s: %s", queryPath, queryErr.Message) + } + parser := sitter.NewParser() + if err := parser.SetLanguage(language); err != nil { + query.Close() + parser.Close() + return nil, fmt.Errorf("set language %q: %w", grammarID, err) + } + return &grammar{ + id: grammarID, + parser: parser, + language: language, + query: query, + captures: query.CaptureNames(), + }, nil +} + +func (e *Engine) findQuery(grammarID string) (string, error) { + candidates := []string{ + filepath.Join(e.grammarDir, grammarID, "highlights.scm"), + filepath.Join(e.grammarDir, grammarID+".scm"), + filepath.Join(e.grammarDir, "highlights.scm"), + } + for _, path := range candidates { + if info, err := os.Stat(path); err == nil && !info.IsDir() { + return path, nil + } + } + return "", fmt.Errorf("no highlights.scm for grammar %q under %q", grammarID, e.grammarDir) +} + +func (e *Engine) touchSession(id int64) *session { + e.tick++ + s := e.sessions[id] + if s == nil { + s = &session{} + e.sessions[id] = s + } + s.used = e.tick + e.evictSessions() + return s +} + +func (e *Engine) evictSessions() { + for len(e.sessions) > e.maxSessions { + var oldestID int64 + oldestUse := ^uint64(0) + for id, s := range e.sessions { + if s.used < oldestUse { + oldestID, oldestUse = id, s.used + } + } + e.sessions[oldestID].closeTree() + delete(e.sessions, oldestID) + } +} + +func (e *Engine) parse(g *grammar, s *session, filetype string, text []byte) (*sitter.Tree, error) { + if s.tree != nil && s.filetype == filetype { + oldTree := s.tree + if tree := incrementalParse(g.parser, oldTree, s.text, text); tree != nil { + oldTree.Close() + s.tree = tree + s.text = text + s.filetype = filetype + return tree, nil + } + oldTree.Close() + s.tree = nil + } + s.closeTree() + tree := g.parser.Parse(text, nil) + if tree == nil { + s.text = text + s.filetype = filetype + return nil, errors.New("tree-sitter parse failed") + } + s.tree = tree + s.text = text + s.filetype = filetype + return tree, nil +} + +// incrementalParse mutates oldTree with one InputEdit covering the changed +// region and reparses. The edit is always a single contiguous replacement in +// the old and new document, so it is valid even when several distant regions +// changed. Returns nil when tree-sitter declines to produce a tree. +func incrementalParse(parser *sitter.Parser, oldTree *sitter.Tree, oldText, newText []byte) *sitter.Tree { + prefix, suffix := commonAffix(oldText, newText) + if prefix == len(oldText) && prefix == len(newText) { + // Unchanged document: reparse from the existing tree without an edit. + return parser.Parse(newText, oldTree) + } + oldEnd := len(oldText) - suffix + newEnd := len(newText) - suffix + oldTree.Edit(&sitter.InputEdit{ + StartByte: uint(prefix), + OldEndByte: uint(oldEnd), + NewEndByte: uint(newEnd), + StartPosition: pointAt(oldText, prefix), + OldEndPosition: pointAt(oldText, oldEnd), + NewEndPosition: pointAt(newText, newEnd), + }) + return parser.Parse(newText, oldTree) +} + +func commonAffix(a, b []byte) (prefix, suffix int) { + limit := len(a) + if len(b) < limit { + limit = len(b) + } + for prefix < limit && a[prefix] == b[prefix] { + prefix++ + } + for suffix < limit-prefix && a[len(a)-1-suffix] == b[len(b)-1-suffix] { + suffix++ + } + return prefix, suffix +} + +func pointAt(text []byte, offset int) sitter.Point { + if offset > len(text) { + offset = len(text) + } + row, lineStart := 0, 0 + for i := 0; i < offset; i++ { + if text[i] == '\n' { + row++ + lineStart = i + 1 + } + } + return sitter.Point{Row: uint(row), Column: uint(offset - lineStart)} +} + +type lineIndex struct { + starts []int + lengths []int +} + +func newLineIndex(text []byte) lineIndex { + starts := []int{0} + for i, b := range text { + if b == '\n' { + starts = append(starts, i+1) + } + } + lengths := make([]int, len(starts)) + for i, start := range starts { + end := len(text) + if i+1 < len(starts) { + end = starts[i+1] - 1 + } + if end < start { + end = start + } + lengths[i] = end - start + } + return lineIndex{starts: starts, lengths: lengths} +} + +type rawSpan struct { + Span + seq int + spec int +} + +func (g *grammar) spans(tree *sitter.Tree, text []byte) []Span { + lines := newLineIndex(text) + root := tree.RootNode() + cursor := sitter.NewQueryCursor() + defer cursor.Close() + + matches := cursor.Matches(g.query, root, text) + var raw []rawSpan + seq := 0 + for match := matches.Next(); match != nil; match = matches.Next() { + for _, capture := range match.Captures { + name := "" + if int(capture.Index) < len(g.captures) { + name = g.captures[capture.Index] + } + group := MapCapture(name) + if group == "" { + continue + } + node := capture.Node + if g.id == "go" && name == "keyword" { + if refined, ok := goKeywordGroups[nodeText(text, &node)]; ok { + group = refined + } + } + start, end := node.StartPosition(), node.EndPosition() + spec := strings.Count(name, ".") + 1 + for _, span := range splitNode(lines, int(start.Row), int(start.Column), int(end.Row), int(end.Column)) { + raw = append(raw, rawSpan{Span: Span{ + Line: span.Line, + Start: span.Start, + End: span.End, + Group: group, + }, seq: seq, spec: spec}) + } + seq++ + } + } + return resolve(raw) +} + +func nodeText(text []byte, node *sitter.Node) string { + start, end := node.StartByte(), node.EndByte() + if start > end || end > uint(len(text)) { + return "" + } + return string(text[start:end]) +} + +func splitNode(lines lineIndex, startRow, startColumn, endRow, endColumn int) []Span { + if startRow < 0 { + startRow = 0 + } + if endRow < startRow { + return nil + } + last := len(lines.starts) - 1 + if startRow > last { + return nil + } + if endRow > last { + endRow = last + endColumn = lines.lengths[last] + } + clamp := func(row, column int) int { + if column < 0 { + return 0 + } + if column > lines.lengths[row] { + return lines.lengths[row] + } + return column + } + var out []Span + if startRow == endRow { + start, end := clamp(startRow, startColumn), clamp(endRow, endColumn) + if end > start { + out = append(out, Span{Line: startRow, Start: start, End: end}) + } + return out + } + if start := clamp(startRow, startColumn); lines.lengths[startRow] > start { + out = append(out, Span{Line: startRow, Start: start, End: lines.lengths[startRow]}) + } + for row := startRow + 1; row < endRow; row++ { + if lines.lengths[row] > 0 { + out = append(out, Span{Line: row, Start: 0, End: lines.lengths[row]}) + } + } + if end := clamp(endRow, endColumn); end > 0 { + out = append(out, Span{Line: endRow, Start: 0, End: end}) + } + return out +} + +func resolve(raw []rawSpan) []Span { + if len(raw) == 0 { + return nil + } + byLine := make(map[int][]rawSpan) + for _, span := range raw { + byLine[span.Line] = append(byLine[span.Line], span) + } + lineNumbers := make([]int, 0, len(byLine)) + for line := range byLine { + lineNumbers = append(lineNumbers, line) + } + sort.Ints(lineNumbers) + + var out []Span + for _, line := range lineNumbers { + candidates := byLine[line] + // Identical ranges: the most specific capture name wins; on a tie the + // earlier pattern wins (e.g. @function over a later @variable). + type spanKey struct{ start, end int } + best := make(map[spanKey]rawSpan, len(candidates)) + for _, candidate := range candidates { + key := spanKey{candidate.Start, candidate.End} + current, seen := best[key] + if !seen || candidate.spec > current.spec || + (candidate.spec == current.spec && candidate.seq < current.seq) { + best[key] = candidate + } + } + list := make([]rawSpan, 0, len(best)) + for _, candidate := range best { + list = append(list, candidate) + } + sort.SliceStable(list, func(i, j int) bool { + if list[i].spec != list[j].spec { + return list[i].spec < list[j].spec + } + return list[i].seq < list[j].seq + }) + + var painted []Span + for _, candidate := range list { + painted = paint(painted, Span{ + Line: line, + Start: candidate.Start, + End: candidate.End, + Group: candidate.Group, + }) + } + for _, span := range painted { + if n := len(out); n > 0 && out[n-1].Line == span.Line && + out[n-1].End == span.Start && out[n-1].Group == span.Group { + out[n-1].End = span.End + continue + } + out = append(out, span) + } + } + return out +} + +// paint overwrites [span.Start, span.End) in a sorted, non-overlapping list. +// Later spans win; the caller establishes precedence by paint order. +func paint(list []Span, span Span) []Span { + out := list[:0] + for _, existing := range list { + if existing.End <= span.Start || existing.Start >= span.End { + out = append(out, existing) + continue + } + if existing.Start < span.Start { + out = append(out, Span{Line: existing.Line, Start: existing.Start, End: span.Start, Group: existing.Group}) + } + if existing.End > span.End { + out = append(out, Span{Line: existing.Line, Start: span.End, End: existing.End, Group: existing.Group}) + } + } + out = append(out, span) + sort.SliceStable(out, func(i, j int) bool { return out[i].Start < out[j].Start }) + return out +} diff --git a/treesitter/internal/tsprovider/provider_test.go b/treesitter/internal/tsprovider/provider_test.go new file mode 100644 index 00000000..63b3b3ab --- /dev/null +++ b/treesitter/internal/tsprovider/provider_test.go @@ -0,0 +1,309 @@ +package tsprovider + +import ( + "bytes" + "math/rand" + "os" + "testing" +) + +func loadSample(t *testing.T) []byte { + t.Helper() + text, err := os.ReadFile("../../testdata/sample.go") + if err != nil { + t.Fatal(err) + } + return text +} + +func splitLines(text []byte) [][]byte { + return bytes.Split(text, []byte("\n")) +} + +func groupAt(spans []Span, line, column int) string { + for _, span := range spans { + if span.Line == line && column >= span.Start && column < span.End { + return span.Group + } + } + return "" +} + +func haveSpan(spans []Span, line, start, end int, group string) bool { + for _, span := range spans { + if span.Line == line && span.Start == start && span.End == end && span.Group == group { + return true + } + } + return false +} + +func TestGoSampleSpans(t *testing.T) { + engine := New("../../queries") + defer engine.Close() + text := loadSample(t) + spans, err := engine.Spans(1, "go", text) + if err != nil { + t.Fatal(err) + } + lines := splitLines(text) + + // Go keyword refinement: declarations are preproc, control flow is + // special, and conditional keywords stay statement. + if got := groupAt(spans, 0, 0); got != "preproc" { + t.Errorf("package keyword: got %q", got) + } + if got := groupAt(spans, 11, 0); got != "preproc" { + t.Errorf("func keyword: got %q", got) + } + if got := groupAt(spans, 16, 1); got != "special" { + t.Errorf("return keyword: got %q", got) + } + + // Multi-line block comment: first line from the opener, middle line fully + // covered, last line through the closer. + if got := groupAt(spans, 3, 0); got != "comment" { + t.Errorf("block comment opener: got %q", got) + } + if got := groupAt(spans, 4, 0); got != "comment" { + t.Errorf("block comment middle line column 0: got %q", got) + } + if got := groupAt(spans, 4, len(lines[4])-1); got != "comment" { + t.Errorf("block comment middle line last column: got %q", got) + } + if got := groupAt(spans, 5, len(lines[5])-1); got != "comment" { + t.Errorf("block comment closer: got %q", got) + } + + // Raw string with backticks spanning three lines. + rawStart := bytes.IndexByte(lines[12], '`') + if !haveSpan(spans, 12, rawStart, len(lines[12]), "constant.string") { + t.Errorf("raw string first line: want constant.string [%d,%d), spans %v", rawStart, len(lines[12]), lineSpans(spans, 12)) + } + if !haveSpan(spans, 13, 0, len(lines[13]), "constant.string") { + t.Errorf("raw string middle line: spans %v", lineSpans(spans, 13)) + } + if !haveSpan(spans, 14, 0, len(lines[14]), "constant.string") { + t.Errorf("raw string last line: spans %v", lineSpans(spans, 14)) + } + + // Struct tag: the raw string literal is a single span including both + // backticks. + tagStart := bytes.IndexByte(lines[7], '`') + tagEnd := bytes.LastIndexByte(lines[7], '`') + 1 + if !haveSpan(spans, 7, tagStart, tagEnd, "constant.string") { + t.Errorf("struct tag: want constant.string [%d,%d), spans %v", tagStart, tagEnd, lineSpans(spans, 7)) + } + + // UTF-8 identifier: byte columns, not rune columns. + name := []byte("Héllo") + for _, line := range []int{6, 19} { + start := bytes.Index(lines[line], name) + if start < 0 { + t.Fatalf("Héllo missing on line %d", line) + } + if !haveSpan(spans, line, start, start+len(name), "type") { + t.Errorf("UTF-8 type identifier on line %d: want type [%d,%d), spans %v", line, start, start+len(name), lineSpans(spans, line)) + } + } + greet := []byte("Gréet") + start := bytes.Index(lines[11], greet) + if !haveSpan(spans, 11, start, start+len(greet), "identifier") { + t.Errorf("UTF-8 function name: want identifier [%d,%d), spans %v", start, start+len(greet), lineSpans(spans, 11)) + } + + // Every span must be a non-empty, in-bounds byte range on its line. + for _, span := range spans { + if span.Line < 0 || span.Line >= len(lines) { + t.Fatalf("span line out of range: %+v", span) + } + if span.Start < 0 || span.End > len(lines[span.Line]) || span.End <= span.Start { + t.Fatalf("span out of bounds: %+v (line length %d)", span, len(lines[span.Line])) + } + } +} + +func lineSpans(spans []Span, line int) []Span { + var out []Span + for _, span := range spans { + if span.Line == line { + out = append(out, span) + } + } + return out +} + +func TestIncrementalMatchesFullParse(t *testing.T) { + engine := New("../../queries") + defer engine.Close() + text := append([]byte(nil), loadSample(t)...) + + insertions := []string{ + "x", "\n", "// note\n", "func f() {}\n", "é", "`", "/*", "\tvar _ = 1\n", + "日本語", "\r\n", "string", "}", "package p\n", + } + random := rand.New(rand.NewSource(7)) + for step := 0; step < 40; step++ { + switch random.Intn(3) { + case 0: + at := random.Intn(len(text) + 1) + insert := insertions[random.Intn(len(insertions))] + text = append(text[:at:at], append([]byte(insert), text[at:]...)...) + case 1: + if len(text) > 1 { + at := random.Intn(len(text)) + n := 1 + random.Intn(min(5, len(text)-at)) + text = append(text[:at:at], text[at+n:]...) + } + default: + if len(text) > 1 { + at := random.Intn(len(text)) + n := 1 + random.Intn(min(5, len(text)-at)) + text = append(text[:at:at], append([]byte(insertions[random.Intn(len(insertions))]), text[at+n:]...)...) + } + } + + got, err := engine.Spans(1, "go", text) + if err != nil { + t.Fatalf("step %d: incremental parse: %v", step, err) + } + fresh := New("../../queries") + want, err := fresh.Spans(1, "go", text) + fresh.Close() + if err != nil { + t.Fatalf("step %d: full parse: %v", step, err) + } + if !equalSpans(got, want) { + t.Fatalf("step %d: incremental spans differ from full parse\ngot %v\nwant %v\ntext:\n%s", step, got, want, text) + } + } +} + +func equalSpans(a, b []Span) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestSplitNode(t *testing.T) { + lines := newLineIndex([]byte("aa\n\nbbbb\ncc")) + got := splitNode(lines, 0, 1, 2, 1) + want := []Span{ + {Line: 0, Start: 1, End: 2}, + {Line: 2, Start: 0, End: 1}, + } + if !equalSpans(got, want) { + t.Fatalf("split: got %v want %v", got, want) + } + // A node ending at column 0 of the next line must not emit a zero-width + // span on that line. + got = splitNode(lines, 0, 0, 1, 0) + if len(got) != 1 || got[0] != (Span{Line: 0, Start: 0, End: 2}) { + t.Fatalf("end at line start: got %v", got) + } + // A node covering an empty line emits nothing for it. + got = splitNode(lines, 1, 0, 2, 0) + if len(got) != 0 { + t.Fatalf("empty middle/end line: got %v", got) + } +} + +func TestCapturePrecedence(t *testing.T) { + engine := New("../../queries") + defer engine.Close() + text := []byte("package p\n\nvar s = \"a\\tb\"\n\nfunc f() { foo(len(s)) }\n") + spans, err := engine.Spans(1, "go", text) + if err != nil { + t.Fatal(err) + } + lines := splitLines(text) + + // An escape capture is narrower and later than its enclosing string, so + // it must override the string group inside the quoted literal. + escape := bytes.Index(lines[2], []byte("\\t")) + if escape < 0 { + t.Fatalf("escape sequence missing from %q", lines[2]) + } + if got := groupAt(spans, 2, escape); got != "constant.specialChar" { + t.Errorf("escape group: got %q, spans %v", got, lineSpans(spans, 2)) + } + quote := bytes.IndexByte(lines[2], '"') + if got := groupAt(spans, 2, quote); got != "constant.string" { + t.Errorf("string group: got %q", got) + } + + // @function and @variable cover the same node for a call name. The + // earlier, more specific pattern must win; @function.builtin wins over + // @function by capture-name specificity. + foo := bytes.Index(lines[4], []byte("foo")) + if got := groupAt(spans, 4, foo); got != "identifier" { + t.Errorf("call name group: got %q, spans %v", got, lineSpans(spans, 4)) + } + builtin := bytes.Index(lines[4], []byte("len")) + if got := groupAt(spans, 4, builtin); got != "preproc" { + t.Errorf("builtin call group: got %q, spans %v", got, lineSpans(spans, 4)) + } +} + +func TestSessionsAreBounded(t *testing.T) { + engine := New("../../queries") + defer engine.Close() + text := []byte("package p\n") + for id := int64(1); id <= 20; id++ { + if _, err := engine.Spans(id, "go", text); err != nil { + t.Fatal(err) + } + } + if len(engine.sessions) != engine.maxSessions { + t.Fatalf("session count: got %d want %d", len(engine.sessions), engine.maxSessions) + } + if engine.sessions[20] == nil { + t.Fatal("most recently used session was evicted") + } +} + +func TestUnsupportedFiletype(t *testing.T) { + engine := New("../../queries") + defer engine.Close() + if _, err := engine.Spans(1, "python", []byte("print(1)\n")); err == nil { + t.Fatal("expected an error for an unlinked filetype") + } +} + +func TestMapCapture(t *testing.T) { + cases := map[string]string{ + "comment": "comment", + "comment.documentation": "comment", + "string": "constant.string", + "escape": "constant.specialChar", + "number": "constant.number", + "keyword": "statement", + "keyword.function": "preproc", + "function": "identifier", + "function.builtin": "preproc", + "type": "type", + "constant.builtin": "constant.bool", + "operator": "symbol.operator", + "punctuation.bracket": "symbol.brackets", + "preproc": "preproc", + "unknown.capture": "", + } + for capture, want := range cases { + if got := MapCapture(capture); got != want { + t.Errorf("MapCapture(%q) = %q, want %q", capture, got, want) + } + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/treesitter/queries/go/LICENSE b/treesitter/queries/go/LICENSE new file mode 100644 index 00000000..4b52d191 --- /dev/null +++ b/treesitter/queries/go/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/treesitter/queries/go/highlights.scm b/treesitter/queries/go/highlights.scm new file mode 100644 index 00000000..6a3c0ac8 --- /dev/null +++ b/treesitter/queries/go/highlights.scm @@ -0,0 +1,123 @@ +; Function calls + +(call_expression + function: (identifier) @function) + +(call_expression + function: (identifier) @function.builtin + (#match? @function.builtin "^(append|cap|close|complex|copy|delete|imag|len|make|new|panic|print|println|real|recover)$")) + +(call_expression + function: (selector_expression + field: (field_identifier) @function.method)) + +; Function definitions + +(function_declaration + name: (identifier) @function) + +(method_declaration + name: (field_identifier) @function.method) + +; Identifiers + +(type_identifier) @type +(field_identifier) @property +(identifier) @variable + +; Operators + +[ + "--" + "-" + "-=" + ":=" + "!" + "!=" + "..." + "*" + "*" + "*=" + "/" + "/=" + "&" + "&&" + "&=" + "%" + "%=" + "^" + "^=" + "+" + "++" + "+=" + "<-" + "<" + "<<" + "<<=" + "<=" + "=" + "==" + ">" + ">=" + ">>" + ">>=" + "|" + "|=" + "||" + "~" +] @operator + +; Keywords + +[ + "break" + "case" + "chan" + "const" + "continue" + "default" + "defer" + "else" + "fallthrough" + "for" + "func" + "go" + "goto" + "if" + "import" + "interface" + "map" + "package" + "range" + "return" + "select" + "struct" + "switch" + "type" + "var" +] @keyword + +; Literals + +[ + (interpreted_string_literal) + (raw_string_literal) + (rune_literal) +] @string + +(escape_sequence) @escape + +[ + (int_literal) + (float_literal) + (imaginary_literal) +] @number + +[ + (true) + (false) + (nil) + (iota) +] @constant.builtin + +(comment) @comment diff --git a/treesitter/testdata/demo.py b/treesitter/testdata/demo.py new file mode 100755 index 00000000..04916561 --- /dev/null +++ b/treesitter/testdata/demo.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Drive micro-treesitter over stdin/stdout and print one reply. + +Usage: demo.py [helper] [file] [filetype] +""" +import subprocess +import sys + +helper = sys.argv[1] if len(sys.argv) > 1 else "./micro-treesitter" +path = sys.argv[2] if len(sys.argv) > 2 else "testdata/sample.go" +filetype = sys.argv[3] if len(sys.argv) > 3 else "go" + +with open(path, "rb") as handle: + text = handle.read() + +process = subprocess.Popen( + [helper, "--grammar-dir", "queries"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) +header = b"REQ\t1\t1\t" + filetype.encode() + b"\t" + str(len(text)).encode() + b"\n" +process.stdin.write(header + text) +process.stdin.flush() + +reply = process.stdout.readline().decode() +sys.stdout.write(reply) +fields = reply.rstrip("\n").split("\t") +if fields[0] == "SPANS": + for _ in range(int(fields[3])): + sys.stdout.write(process.stdout.readline().decode()) +elif fields[0] == "ERR": + sys.stdout.write(process.stderr.read().decode()) + +process.stdin.close() +process.wait() diff --git a/treesitter/testdata/lua_harness.lua b/treesitter/testdata/lua_harness.lua new file mode 100644 index 00000000..71af39dc --- /dev/null +++ b/treesitter/testdata/lua_harness.lua @@ -0,0 +1,109 @@ +-- Standalone harness for runtime/plugins/treesitter/treesitter.lua. +-- Run from the repository root: lua5_1 testdata/lua_harness.lua +-- +-- It mocks micro's Lua modules, drives the plugin's protocol state machine +-- with chunked helper output, and asserts the reply/generation behaviour. + +local logs = {} +local sent = {} +local stops = 0 +local registered +local onStdout, onStderr, onExit + +function import(name) + if name == "micro/syntax" then + return { register = function(provider) registered = provider end } + elseif name == "micro/shell" then + return { + JobSpawn = function(command, args, out, err, exit) + onStdout, onStderr, onExit = out, err, exit + return { command = command } + end, + JobSend = function(job, data) sent[#sent + 1] = data end, + JobStop = function(job) stops = stops + 1 end, + } + elseif name == "micro" then + return { Log = function(...) logs[#logs + 1] = table.concat({ ... }, " ") end } + elseif name == "os" then + return { getenv = function() return nil end } + end + error("unexpected import: " .. name) +end + +dofile("runtime/plugins/treesitter/treesitter.lua") + +local function check(condition, message) + if not condition then + error(message, 2) + end +end + +local function feed(chunk) + onStdout(chunk, {}) +end + +local function joinFlat(spans) + local parts = {} + for i = 1, #spans do + parts[i] = tostring(spans[i]) + end + return table.concat(parts, "|") +end + +init() +check(registered ~= nil, "provider was not registered") +check(registered.name == "treesitter-go", "provider name") +check(registered.filetype == "go", "provider filetype") +check(registered.filename == "\\.go$", "provider filename regex") +check(registered.priority == 100, "provider priority") + +-- Frame framing: exact bytes, byte length, id and generation from ctx. +local delivered +local function reply(spans) delivered = spans end +local text = "package x\n" +registered.request({ id = 1, generation = 5, filetype = "go", text = text }, reply) +check(#sent == 1, "expected one REQ frame, got " .. #sent) +local wantFrame = "REQ\t1\t5\tgo\t10\n" .. text +check(sent[1] == wantFrame, string.format("frame mismatch: %q", sent[1])) + +-- Chunked reply: header and spans split at arbitrary byte boundaries. +feed("SPA") +check(delivered == nil, "replied before the header was complete") +feed("NS\t1\t5\t2\n0\t0\t7\tprepr") +check(delivered == nil, "replied before the first span line was complete") +feed("oc\n1\t0\t1\tcomment\n") +check(delivered ~= nil, "reply was not delivered") +check(joinFlat(delivered) == "0|0|7|preproc|1|0|1|comment", + "span payload: " .. joinFlat(delivered)) + +-- A stale generation must not overwrite the newer pending reply. +delivered = nil +registered.request({ id = 1, generation = 6, filetype = "go", text = text }, reply) +feed("SPANS\t1\t5\t0\n") +check(delivered == nil, "stale generation was delivered") +feed("SPANS\t1\t6\t0\n") +check(delivered ~= nil and #delivered == 0, "current generation clears spans") + +-- ERR for the current generation clears spans and logs once. +delivered = nil +registered.request({ id = 1, generation = 7, filetype = "go", text = text }, reply) +feed("ERR\t1\t7\tquery failed\n") +check(delivered ~= nil and #delivered == 0, "ERR should clear spans") +local sawError = false +for _, line in ipairs(logs) do + if line:find("query failed", 1, true) then sawError = true end +end +check(sawError, "ERR was not logged") + +-- Unknown frames are a protocol failure: the helper is stopped and disabled. +local stopsBefore = stops +feed("WHAT\t1\t7\tnonsense\n") +check(stops == stopsBefore + 1, "unknown frame did not stop the helper") +delivered = nil +registered.request({ id = 1, generation = 8, filetype = "go", text = text }, reply) +check(delivered ~= nil and #delivered == 0, "disabled plugin must still answer") + +deinit() +check(stops == stopsBefore + 1, "deinit stopped an already stopped helper") + +print("lua harness: ok") diff --git a/treesitter/testdata/sample.go b/treesitter/testdata/sample.go new file mode 100644 index 00000000..609f3213 --- /dev/null +++ b/treesitter/testdata/sample.go @@ -0,0 +1,22 @@ +package sample + +// 日本語のコメント: héllo 世界 +/* block comment + spanning three + lines */ +type Héllo struct { + Nom string `json:"nom,omitempty"` + Note string +} + +func Gréet(nom string) string { + raw := `first line +second "line" +third line` + _ = raw + return "héllo " + nom +} + +func (h *Héllo) Label() string { + return h.Nom +} -- knotmirror 0.0.1