chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
// Hot-path workaround: the upstream tree-sitter/go-tree-sitter
|
||||
// (v0.25.0) registers a Go-routed allocator at package init —
|
||||
// `ts_set_allocator(c_malloc_fn, …)` — and `c_malloc_fn` calls back
|
||||
// into Go (`go_malloc`), which then calls `C.malloc`. Every internal
|
||||
// allocation inside the parser therefore round-trips C → Go → C
|
||||
// instead of going straight to libc. On large repos that round-trip
|
||||
// dominated the cgo cost: `_cgoexp_…go_malloc` showed up at ~35% of
|
||||
// total CPU in our indexer profile, and indexing the gortex repo
|
||||
// itself ran 3× slower than the smacker baseline.
|
||||
//
|
||||
// We undo that by re-calling `ts_set_allocator(NULL, …, NULL)` from
|
||||
// our own init, which the C library treats as "use libc directly."
|
||||
// The symbol `ts_set_allocator` is exported from the upstream
|
||||
// package's CGO archive, so we declare it `extern` and link against
|
||||
// the same archive.
|
||||
//
|
||||
// Init ordering: Go runs init() functions in import-graph order, and
|
||||
// this file is in the `tsitter` package which depends on the upstream
|
||||
// `tree_sitter` package — so the upstream init (which registers the
|
||||
// indirection) runs first, ours runs after, and our reset wins.
|
||||
//
|
||||
// Tree-sitter's docs warn that switching allocators after objects
|
||||
// have been created is unsafe unless those objects are freed first
|
||||
// or the new allocator shares state with the old. Both go-routed and
|
||||
// libc allocators ultimately call `C.malloc`/`C.free`, so they share
|
||||
// state — the round-trip layer drops away cleanly.
|
||||
|
||||
package tsitter
|
||||
|
||||
/*
|
||||
extern void ts_set_allocator(
|
||||
void *(*new_malloc)(unsigned long),
|
||||
void *(*new_calloc)(unsigned long, unsigned long),
|
||||
void *(*new_realloc)(void *, unsigned long),
|
||||
void (*new_free)(void *)
|
||||
);
|
||||
|
||||
static void gortex_use_libc_allocator(void) {
|
||||
ts_set_allocator(0, 0, 0, 0);
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
func init() {
|
||||
C.gortex_use_libc_allocator()
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tsitter
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestNodeArenaStablePointers guards the arena's load-bearing invariant:
|
||||
// a pointer returned by alloc stays valid after later allocations grow the
|
||||
// arena onto a new backing chunk. If alloc ever appended into a single
|
||||
// reallocating slice, earlier &slice[i] pointers would dangle and node data
|
||||
// would silently corrupt mid-walk.
|
||||
func TestNodeArenaStablePointers(t *testing.T) {
|
||||
a := newNodeArena()
|
||||
const n = arenaMaxChunk*2 + 17 // cross several chunk boundaries incl. the max-size cap
|
||||
ptrs := make([]*Node, n)
|
||||
for i := range ptrs {
|
||||
p := a.alloc()
|
||||
if p == nil {
|
||||
t.Fatalf("alloc %d returned nil", i)
|
||||
}
|
||||
p.valid = true
|
||||
ptrs[i] = p
|
||||
}
|
||||
seen := make(map[*Node]bool, n)
|
||||
for i, p := range ptrs {
|
||||
if seen[p] {
|
||||
t.Fatalf("alloc %d aliased an earlier node pointer", i)
|
||||
}
|
||||
seen[p] = true
|
||||
if !p.valid {
|
||||
t.Fatalf("node %d was clobbered by a later chunk allocation", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeArenaGeometricGrowth confirms the first chunk is small (low waste
|
||||
// for tiny files) and chunks grow up to the cap (few objects for deep files).
|
||||
func TestNodeArenaGeometricGrowth(t *testing.T) {
|
||||
a := newNodeArena()
|
||||
a.alloc()
|
||||
if got := len(a.chunks[0]); got != arenaFirstChunk {
|
||||
t.Fatalf("first chunk size = %d, want %d", got, arenaFirstChunk)
|
||||
}
|
||||
// Drain well past the cap; no chunk may exceed the max.
|
||||
for i := 0; i < arenaMaxChunk*3; i++ {
|
||||
a.alloc()
|
||||
if got := len(a.chunks[a.ci]); got > arenaMaxChunk {
|
||||
t.Fatalf("chunk grew to %d, exceeds cap %d", got, arenaMaxChunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeArenaResetReusesChunks is the pooling invariant: reset rewinds the
|
||||
// cursor but KEEPS the backing chunks, so refilling to the same size
|
||||
// allocates no new chunk, and any stale Node from the previous round is
|
||||
// cleared — a recycled arena must never pin a closed tree.
|
||||
func TestNodeArenaResetReusesChunks(t *testing.T) {
|
||||
a := newNodeArena()
|
||||
const n = arenaMaxChunk + 100 // span more than one chunk
|
||||
for i := 0; i < n; i++ {
|
||||
a.alloc().valid = true
|
||||
}
|
||||
want := len(a.chunks)
|
||||
if want < 2 {
|
||||
t.Fatalf("test needs a multi-chunk arena, got %d chunk(s)", want)
|
||||
}
|
||||
|
||||
a.reset()
|
||||
if len(a.chunks) != want {
|
||||
t.Fatalf("reset dropped chunks: have %d, want %d (chunks must be retained)", len(a.chunks), want)
|
||||
}
|
||||
if a.chunks[0][0].valid {
|
||||
t.Fatal("reset left a stale Node valid — a recycled arena would pin its tree")
|
||||
}
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
a.alloc().valid = true
|
||||
}
|
||||
if len(a.chunks) != want {
|
||||
t.Fatalf("refill allocated fresh chunks: have %d, want %d (chunks not reused)", len(a.chunks), want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Package bash re-exports the tree-sitter-bash grammar in a
|
||||
// smacker-compatible shape (GetLanguage() *tsitter.Language).
|
||||
package bash
|
||||
|
||||
import (
|
||||
tree_sitter_bash "github.com/tree-sitter/tree-sitter-bash/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_bash.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package c re-exports the tree-sitter-c grammar.
|
||||
package c
|
||||
|
||||
import (
|
||||
tree_sitter_c "github.com/tree-sitter/tree-sitter-c/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_c.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package cpp re-exports the tree-sitter-cpp grammar.
|
||||
package cpp
|
||||
|
||||
import (
|
||||
tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_cpp.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package csharp re-exports the tree-sitter-c-sharp grammar.
|
||||
package csharp
|
||||
|
||||
import (
|
||||
tree_sitter_csharp "github.com/tree-sitter/tree-sitter-c-sharp/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_csharp.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package css re-exports the tree-sitter-css grammar.
|
||||
package css
|
||||
|
||||
import (
|
||||
tree_sitter_css "github.com/tree-sitter/tree-sitter-css/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_css.Language())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package dart re-exports the tree-sitter-dart grammar. The C parser
|
||||
// lives in the sibling github.com/gortexhq/tree-sitter-dart module;
|
||||
// this file is just the thin shim that bridges the upstream binding
|
||||
// into gortex's *tsitter.Language type.
|
||||
package dart
|
||||
|
||||
import (
|
||||
tree_sitter_dart "github.com/gortexhq/tree-sitter-dart/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Dart language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_dart.Language())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package dockerfile re-exports the tree-sitter-dockerfile grammar.
|
||||
// The C parser lives in the sibling github.com/gortexhq/tree-sitter-dockerfile
|
||||
// module; this file is just the thin shim that bridges the upstream
|
||||
// binding into gortex's *tsitter.Language type.
|
||||
package dockerfile
|
||||
|
||||
import (
|
||||
tree_sitter_dockerfile "github.com/gortexhq/tree-sitter-dockerfile/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Dockerfile language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_dockerfile.Language())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package elixir re-exports tree-sitter-elixir.
|
||||
// The upstream repo declares its module path as
|
||||
// github.com/tree-sitter/tree-sitter-elixir (it was donated from
|
||||
// elixir-lang); go.mod pins a replace directive to the elixir-lang
|
||||
// origin so this import resolves.
|
||||
package elixir
|
||||
|
||||
import (
|
||||
tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_elixir.Language())
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Package golang re-exports the tree-sitter-go grammar.
|
||||
// The package name mirrors smacker's `golang` so existing extractor
|
||||
// imports only need a path change.
|
||||
package golang
|
||||
|
||||
import (
|
||||
tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_go.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package hcl re-exports tree-sitter-grammars/tree-sitter-hcl.
|
||||
package hcl
|
||||
|
||||
import (
|
||||
tree_sitter_hcl "github.com/tree-sitter-grammars/tree-sitter-hcl/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_hcl.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package html re-exports the tree-sitter-html grammar.
|
||||
package html
|
||||
|
||||
import (
|
||||
tree_sitter_html "github.com/tree-sitter/tree-sitter-html/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_html.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package java re-exports the tree-sitter-java grammar.
|
||||
package java
|
||||
|
||||
import (
|
||||
tree_sitter_java "github.com/tree-sitter/tree-sitter-java/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_java.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package javascript re-exports the tree-sitter-javascript grammar.
|
||||
package javascript
|
||||
|
||||
import (
|
||||
tree_sitter_javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_javascript.Language())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package kotlin re-exports fwcd/tree-sitter-kotlin. The
|
||||
// tree-sitter-grammars/tree-sitter-kotlin v1.x grammar renamed many
|
||||
// node kinds (e.g. dropped the `type_identifier` alias), which would
|
||||
// force a substantial rewrite of our Kotlin extractor. fwcd's grammar
|
||||
// matches the node vocabulary smacker shipped, so we keep that.
|
||||
package kotlin
|
||||
|
||||
import (
|
||||
tree_sitter_kotlin "github.com/fwcd/tree-sitter-kotlin/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_kotlin.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package lua re-exports tree-sitter-grammars/tree-sitter-lua.
|
||||
package lua
|
||||
|
||||
import (
|
||||
tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_lua.Language())
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Package markdown re-exports the tree-sitter-markdown grammar. The C
|
||||
// parser lives in the sibling github.com/gortexhq/tree-sitter-markdown
|
||||
// module (a fork of the tree-sitter-recommended
|
||||
// tree-sitter-grammars/tree-sitter-markdown block-level parser with
|
||||
// Go bindings added); this file is just the thin shim that bridges
|
||||
// the upstream binding into gortex's *tsitter.Language type.
|
||||
package markdown
|
||||
|
||||
import (
|
||||
tree_sitter_markdown "github.com/gortexhq/tree-sitter-markdown/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Markdown language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_markdown.Language())
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package tsitter
|
||||
|
||||
// Direct C navigation for the hot child-access path.
|
||||
//
|
||||
// go-tree-sitter's Node navigation (Child / NamedChild / Parent, the cursor,
|
||||
// even the []Node value-slice helpers) funnels through newNode, which
|
||||
// heap-allocates a *ts.Node for every visited node. Profiling a large index
|
||||
// put that at ~43% of all bytes allocated — 95% of it NamedChild + Child +
|
||||
// Parent in the extractor child loops — and the resulting GC churn dominated
|
||||
// CPU under a memory cap.
|
||||
//
|
||||
// The tree-sitter C API returns nodes BY VALUE (TSNode is a 32-byte struct).
|
||||
// We call those C functions directly and drop the returned value straight
|
||||
// into the pooled Node arena, so a child visit costs one bump-allocated
|
||||
// wrapper and zero heap garbage.
|
||||
//
|
||||
// The functions are declared here rather than pulled from <tree_sitter/api.h>
|
||||
// so this file needs no -I path into the go-tree-sitter module cache (which
|
||||
// is version- and machine-specific). The symbols themselves are defined in
|
||||
// the tree-sitter C library that go-tree-sitter already compiles and links
|
||||
// into the binary; C has no name mangling, so a local prototype with the
|
||||
// identical ABI resolves to them at link time.
|
||||
|
||||
/*
|
||||
#include <stdint.h>
|
||||
|
||||
// Mirror of tree-sitter's TSNode (include/tree_sitter/api.h): a 4-word
|
||||
// context plus two opaque pointers. Layout must match exactly — it is
|
||||
// reinterpreted from ts.Node, whose sole field is this struct.
|
||||
typedef struct { uint32_t context[4]; const void *id; const void *tree; } GxTSNode;
|
||||
|
||||
// Mirror of TSTreeCursor — reinterpreted from *ts.TreeCursor (whose sole
|
||||
// field is this struct) so the cursor's current node can be read by value.
|
||||
typedef struct { const void *tree; const void *id; uint32_t context[3]; } GxTSTreeCursor;
|
||||
|
||||
extern GxTSNode ts_node_child(GxTSNode, uint32_t);
|
||||
extern GxTSNode ts_node_named_child(GxTSNode, uint32_t);
|
||||
extern GxTSNode ts_node_parent(GxTSNode);
|
||||
extern GxTSNode ts_node_next_sibling(GxTSNode);
|
||||
extern GxTSNode ts_node_prev_sibling(GxTSNode);
|
||||
extern GxTSNode ts_node_next_named_sibling(GxTSNode);
|
||||
extern GxTSNode ts_node_prev_named_sibling(GxTSNode);
|
||||
extern GxTSNode ts_node_child_by_field_name(GxTSNode, const char *, uint32_t);
|
||||
extern GxTSNode ts_tree_cursor_current_node(const GxTSTreeCursor *);
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
|
||||
ts "github.com/tree-sitter/go-tree-sitter"
|
||||
)
|
||||
|
||||
// init fails fast if a tree-sitter upgrade ever changes the TSNode layout
|
||||
// out from under the reinterpret casts below, rather than silently corrupting
|
||||
// navigated nodes at runtime.
|
||||
func init() {
|
||||
if unsafe.Sizeof(ts.Node{}) != unsafe.Sizeof(C.GxTSNode{}) {
|
||||
panic("tsitter: ts.Node and TSNode size mismatch — tree-sitter ABI changed, update node_cnav.go")
|
||||
}
|
||||
if unsafe.Sizeof(ts.TreeCursor{}) != unsafe.Sizeof(C.GxTSTreeCursor{}) {
|
||||
panic("tsitter: ts.TreeCursor and TSTreeCursor size mismatch — tree-sitter ABI changed, update node_cnav.go")
|
||||
}
|
||||
}
|
||||
|
||||
// asC reinterprets an upstream ts.Node as the locally-declared C node. ts.Node
|
||||
// is `struct { _inner C.TSNode }`, so its bytes are exactly a TSNode and the
|
||||
// two share an identical ABI layout.
|
||||
func asC(n ts.Node) C.GxTSNode { return *(*C.GxTSNode)(unsafe.Pointer(&n)) }
|
||||
|
||||
// asGo reinterprets a C node back into an upstream ts.Node value.
|
||||
func asGo(c C.GxTSNode) ts.Node { return *(*ts.Node)(unsafe.Pointer(&c)) }
|
||||
|
||||
// childDirect returns parent's i-th child as a value, with no heap
|
||||
// allocation. ok is false for a null child (index past the end).
|
||||
func childDirect(parent ts.Node, i int) (ts.Node, bool) {
|
||||
c := C.ts_node_child(asC(parent), C.uint32_t(i))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
// namedChildDirect is childDirect over named children only.
|
||||
func namedChildDirect(parent ts.Node, i int) (ts.Node, bool) {
|
||||
c := C.ts_node_named_child(asC(parent), C.uint32_t(i))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
// parentDirect returns n's parent as a value. ok is false at the root.
|
||||
func parentDirect(n ts.Node) (ts.Node, bool) {
|
||||
c := C.ts_node_parent(asC(n))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
func nextSiblingDirect(n ts.Node) (ts.Node, bool) {
|
||||
c := C.ts_node_next_sibling(asC(n))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
func prevSiblingDirect(n ts.Node) (ts.Node, bool) {
|
||||
c := C.ts_node_prev_sibling(asC(n))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
func nextNamedSiblingDirect(n ts.Node) (ts.Node, bool) {
|
||||
c := C.ts_node_next_named_sibling(asC(n))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
func prevNamedSiblingDirect(n ts.Node) (ts.Node, bool) {
|
||||
c := C.ts_node_prev_named_sibling(asC(n))
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
|
||||
// cursorCurrentNode returns a tree cursor's current node by value, avoiding
|
||||
// TreeCursor.Node's per-step heap *Node. The cursor walk itself (GotoFirstChild
|
||||
// / GotoNextSibling) stays O(total children); only the node read is changed.
|
||||
func cursorCurrentNode(cursor *ts.TreeCursor) ts.Node {
|
||||
return asGo(C.ts_tree_cursor_current_node((*C.GxTSTreeCursor)(unsafe.Pointer(cursor))))
|
||||
}
|
||||
|
||||
// childByFieldNameDirect returns parent's child for a grammar field name. The
|
||||
// name's bytes are passed to C by pointer+length (ts_node_child_by_field_name
|
||||
// does not require NUL termination and reads them only for the duration of the
|
||||
// call), so no C string is allocated. ok is false when no such field exists.
|
||||
func childByFieldNameDirect(parent ts.Node, name string) (ts.Node, bool) {
|
||||
if name == "" {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
c := C.ts_node_child_by_field_name(
|
||||
asC(parent),
|
||||
(*C.char)(unsafe.Pointer(unsafe.StringData(name))),
|
||||
C.uint32_t(len(name)),
|
||||
)
|
||||
if c.id == nil {
|
||||
return ts.Node{}, false
|
||||
}
|
||||
return asGo(c), true
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Package ocaml re-exports the tree-sitter-ocaml grammar (OCaml
|
||||
// implementation variant, .ml files). Interface (.mli) and type
|
||||
// bindings exist in sister packages but gortex only indexes .ml.
|
||||
package ocaml
|
||||
|
||||
import (
|
||||
tree_sitter_ocaml "github.com/tree-sitter/tree-sitter-ocaml/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_ocaml.LanguageOCaml())
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
// Package orgmode re-exports the tree-sitter-org-mode grammar. The C
|
||||
// parser lives in the sibling github.com/gortexhq/tree-sitter-org-mode
|
||||
// module (a vendored fork of zac-garby/tree-sitter-org-mode); this file
|
||||
// is the thin shim that bridges the upstream binding into gortex's
|
||||
// *tsitter.Language type.
|
||||
package orgmode
|
||||
|
||||
import (
|
||||
tree_sitter_orgmode "github.com/gortexhq/tree-sitter-org-mode/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Org-mode language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_orgmode.Language())
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Package php re-exports the tree-sitter-php grammar (full-mode PHP
|
||||
// with HTML framing; matches smacker's default php binding).
|
||||
package php
|
||||
|
||||
import (
|
||||
tree_sitter_php "github.com/tree-sitter/tree-sitter-php/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_php.LanguagePHP())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package protobuf re-exports the tree-sitter-proto grammar. The C
|
||||
// parser lives in the sibling github.com/gortexhq/tree-sitter-protobuf
|
||||
// module; this file is just the thin shim that bridges the upstream
|
||||
// binding into gortex's *tsitter.Language type.
|
||||
package protobuf
|
||||
|
||||
import (
|
||||
tree_sitter_protobuf "github.com/gortexhq/tree-sitter-protobuf/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Protobuf language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_protobuf.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package python re-exports the tree-sitter-python grammar.
|
||||
package python
|
||||
|
||||
import (
|
||||
tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_python.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package ruby re-exports the tree-sitter-ruby grammar.
|
||||
package ruby
|
||||
|
||||
import (
|
||||
tree_sitter_ruby "github.com/tree-sitter/tree-sitter-ruby/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_ruby.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package rust re-exports the tree-sitter-rust grammar.
|
||||
package rust
|
||||
|
||||
import (
|
||||
tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_rust.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package scala re-exports the tree-sitter-scala grammar.
|
||||
package scala
|
||||
|
||||
import (
|
||||
tree_sitter_scala "github.com/tree-sitter/tree-sitter-scala/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_scala.Language())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package sql re-exports the tree-sitter-sql grammar. The C parser
|
||||
// lives in the sibling github.com/gortexhq/tree-sitter-sql module;
|
||||
// this file is just the thin shim that bridges the upstream binding
|
||||
// into gortex's *tsitter.Language type.
|
||||
package sql
|
||||
|
||||
import (
|
||||
tree_sitter_sql "github.com/gortexhq/tree-sitter-sql/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled SQL language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_sql.Language())
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Package swift re-exports the tree-sitter-swift grammar. The C parser
|
||||
// lives in the sibling github.com/gortexhq/tree-sitter-swift module (a
|
||||
// fork of the tree-sitter-recommended alex-pinkus/tree-sitter-swift
|
||||
// with a regenerated parser.c committed); this file is just the thin
|
||||
// shim that bridges the upstream binding into gortex's *tsitter.Language
|
||||
// type.
|
||||
package swift
|
||||
|
||||
import (
|
||||
tree_sitter_swift "github.com/gortexhq/tree-sitter-swift/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
// GetLanguage returns the compiled Swift language.
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_swift.Language())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package toml re-exports tree-sitter-grammars/tree-sitter-toml.
|
||||
package toml
|
||||
|
||||
import (
|
||||
tree_sitter_toml "github.com/tree-sitter-grammars/tree-sitter-toml/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_toml.Language())
|
||||
}
|
||||
@@ -0,0 +1,606 @@
|
||||
// Package tsitter is a thin compatibility shim over
|
||||
// github.com/tree-sitter/go-tree-sitter. Its surface intentionally
|
||||
// mirrors the smacker/go-tree-sitter API so that the ~90 language
|
||||
// extractors in gortex can be migrated by changing only import paths —
|
||||
// method names and signatures stay the same.
|
||||
//
|
||||
// The shim wraps native tree-sitter types (Node, Tree, Parser, Query)
|
||||
// and adapts:
|
||||
// - Type() → Kind()
|
||||
// - Content(src) → Utf8Text(src)
|
||||
// - StartPoint/EndPoint → StartPosition/EndPosition (uint32 Row/Column)
|
||||
// - int-indexed children → uint-indexed (internally converted)
|
||||
// - ParseCtx(ctx, old, src) built on top of ParseWithOptions
|
||||
//
|
||||
// The cursor-based query iteration is not re-exposed here: it lives in
|
||||
// the parent parser package, which uses the official API directly.
|
||||
package tsitter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
ts "github.com/tree-sitter/go-tree-sitter"
|
||||
)
|
||||
|
||||
// Language is a parser language. Exposed as an alias so grammar
|
||||
// sub-packages can return *ts.Language directly.
|
||||
type Language = ts.Language
|
||||
|
||||
// NewLanguage constructs a Language from a grammar's raw C pointer —
|
||||
// used by the per-language shim sub-packages.
|
||||
func NewLanguage(ptr unsafe.Pointer) *Language { return ts.NewLanguage(ptr) }
|
||||
|
||||
// Point mirrors the smacker Point layout (uint32 row/column).
|
||||
type Point struct {
|
||||
Row uint32
|
||||
Column uint32
|
||||
}
|
||||
|
||||
func fromTSPoint(p ts.Point) Point {
|
||||
return Point{Row: uint32(p.Row), Column: uint32(p.Column)}
|
||||
}
|
||||
|
||||
// Node wraps *ts.Node with smacker-compatible method names. Nodes are
|
||||
// valid for the lifetime of their Tree; copying by value is cheap
|
||||
// (single C struct field).
|
||||
type Node struct {
|
||||
inner ts.Node
|
||||
// set true when constructed; distinguishes the zero value from a real node.
|
||||
valid bool
|
||||
// langKey is the stable C-pointer identity of this node's language,
|
||||
// propagated from the root through every navigation step so Type()
|
||||
// can resolve the node kind from a per-language table with no CGO
|
||||
// call and no allocation. Nil means "not stamped" — Type() then
|
||||
// derives the language, which is slower but still correct.
|
||||
langKey unsafe.Pointer
|
||||
// arena bump-allocates Node wrappers in chunks so a deep tree walk
|
||||
// produces a few large backing arrays instead of millions of tiny heap
|
||||
// objects. The per-object GC mark cost dominated CPU when indexing a
|
||||
// large TS monorepo (vscode: ~70% of cycles in GC, scanning millions of
|
||||
// 1-node spans). Chunks are never explicitly freed — they are reachable
|
||||
// only through the Nodes that point into them and are reclaimed by the
|
||||
// GC once the tree's nodes are dropped. A nil arena falls back to a plain
|
||||
// heap Node (zero-value and SetInner-pooled nodes have no arena).
|
||||
arena *nodeArena
|
||||
}
|
||||
|
||||
// nodeArena is a per-tree bump allocator for Node wrappers. It is not
|
||||
// safe for concurrent use; each parse tree is walked by a single
|
||||
// goroutine, and distinct files use distinct trees (and arenas).
|
||||
//
|
||||
// Arenas are pooled and reused across files (see arenaPool): a Tree takes
|
||||
// one on its first RootNode() and returns it on Close(). reset() rewinds
|
||||
// the allocation cursor but RETAINS the backing chunks, so a warm pool
|
||||
// serves each file's node count with no fresh allocation. This is the
|
||||
// dominant GC-pressure lever on a large index: profiling vscode and
|
||||
// kubernetes put tree-sitter Node wrappers at 70–82% of every byte
|
||||
// allocated, almost all of it per-file chunk garbage. Retaining the chunks
|
||||
// turns that churn into a bounded, reused working set.
|
||||
type nodeArena struct {
|
||||
chunks [][]Node // backing arrays, retained across resets for reuse
|
||||
ci int // index of the current chunk within chunks
|
||||
used int // slots used in chunks[ci]
|
||||
}
|
||||
|
||||
const (
|
||||
// arenaFirstChunk keeps the first backing array small so a file with a
|
||||
// handful of nodes (the common case in a many-small-files repo) wastes
|
||||
// little; chunks then double up to arenaMaxChunk so a deep file still
|
||||
// ends up with only a few large objects.
|
||||
arenaFirstChunk = 64
|
||||
arenaMaxChunk = 4096
|
||||
)
|
||||
|
||||
func newNodeArena() *nodeArena { return &nodeArena{} }
|
||||
|
||||
// alloc returns a pointer to a fresh Node. The pointer is stable for the
|
||||
// life of the arena: a chunk is never resized in place — when the current
|
||||
// chunk fills, allocation advances to the next retained chunk, or appends a
|
||||
// geometrically larger one past the high-water mark, so earlier &chunk[i]
|
||||
// pointers never move. Callers overwrite all fields of the returned Node
|
||||
// immediately, so a reused slot needs no zeroing here; reset() clears stale
|
||||
// slots when the arena is recycled.
|
||||
func (a *nodeArena) alloc() *Node {
|
||||
switch {
|
||||
case len(a.chunks) == 0:
|
||||
a.chunks = append(a.chunks, make([]Node, arenaFirstChunk))
|
||||
a.ci, a.used = 0, 0
|
||||
case a.used >= len(a.chunks[a.ci]):
|
||||
a.ci++
|
||||
if a.ci >= len(a.chunks) {
|
||||
size := len(a.chunks[a.ci-1]) * 2
|
||||
if size > arenaMaxChunk {
|
||||
size = arenaMaxChunk
|
||||
}
|
||||
a.chunks = append(a.chunks, make([]Node, size))
|
||||
}
|
||||
a.used = 0
|
||||
}
|
||||
n := &a.chunks[a.ci][a.used]
|
||||
a.used++
|
||||
return n
|
||||
}
|
||||
|
||||
// reset rewinds the allocation cursor to the start while retaining the
|
||||
// backing chunks for reuse. It clears the slots touched since the last
|
||||
// reset so a stale Node value — whose embedded ts.Node pins its now-closed
|
||||
// *ts.Tree — cannot survive into the next file and leak. Clearing only the
|
||||
// used prefix keeps the cost proportional to the file just processed, not
|
||||
// the high-water capacity.
|
||||
func (a *nodeArena) reset() {
|
||||
for i := 0; i <= a.ci && i < len(a.chunks); i++ {
|
||||
end := len(a.chunks[i])
|
||||
if i == a.ci {
|
||||
end = a.used
|
||||
}
|
||||
clear(a.chunks[i][:end])
|
||||
}
|
||||
a.ci, a.used = 0, 0
|
||||
}
|
||||
|
||||
// arenaPool recycles per-tree arenas — with their retained chunks — across
|
||||
// files. A Tree gets one lazily on its first RootNode() and returns it on
|
||||
// Close(). sync.Pool may drop entries under GC pressure; a cold Get then
|
||||
// just starts with no chunks and warms up again, so correctness never
|
||||
// depends on retention.
|
||||
var arenaPool = sync.Pool{New: func() any { return &nodeArena{} }}
|
||||
|
||||
func getArena() *nodeArena { return arenaPool.Get().(*nodeArena) }
|
||||
|
||||
func putArena(a *nodeArena) {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.reset()
|
||||
arenaPool.Put(a)
|
||||
}
|
||||
|
||||
// WrapNode wraps a value Node from the new API into our shim. It derives
|
||||
// the language key eagerly so navigation from the result stays alloc-free,
|
||||
// and seeds a fresh arena so the subtree walk below it allocates in chunks.
|
||||
func WrapNode(n ts.Node) *Node {
|
||||
a := newNodeArena()
|
||||
nn := a.alloc()
|
||||
nn.inner = n
|
||||
nn.valid = true
|
||||
nn.langKey = unsafe.Pointer(n.Language().Inner)
|
||||
nn.arena = a
|
||||
return nn
|
||||
}
|
||||
|
||||
// WrapVal wraps a ts.Node reached from n (e.g. a query capture),
|
||||
// carrying n's language key so Type() on the result and its descendants
|
||||
// needs neither CGO nor allocation.
|
||||
func (n *Node) WrapVal(c ts.Node) *Node {
|
||||
if n.arena == nil {
|
||||
return &Node{inner: c, valid: true, langKey: n.langKey}
|
||||
}
|
||||
nn := n.arena.alloc()
|
||||
nn.inner = c
|
||||
nn.valid = true
|
||||
nn.langKey = n.langKey
|
||||
nn.arena = n.arena
|
||||
return nn
|
||||
}
|
||||
|
||||
// SetInner overwrites the receiver's wrapped ts.Node and marks it
|
||||
// valid. Lets callers reuse a *Node out of a pool / backing slice
|
||||
// instead of allocating a new one per query match — see EachMatch in
|
||||
// internal/parser/treesitter.go. The receiver must already exist
|
||||
// (caller-owned), so SetInner cannot be used on a nil pointer.
|
||||
func (n *Node) SetInner(inner ts.Node) {
|
||||
n.inner = inner
|
||||
n.valid = true
|
||||
}
|
||||
|
||||
// Inner returns a pointer to the underlying ts.Node. Internal use by
|
||||
// the parser package's query runners.
|
||||
func (n *Node) Inner() *ts.Node {
|
||||
if n == nil || !n.valid {
|
||||
return nil
|
||||
}
|
||||
return &n.inner
|
||||
}
|
||||
|
||||
// Type returns the node kind string ("identifier", "function_declaration", …).
|
||||
func (n *Node) Type() string { return internedKind(n.inner, n.langKey) }
|
||||
|
||||
// kindTables memoises per-language node-kind name tables. tree-sitter
|
||||
// node kinds are a small fixed set (NodeKindCount — typically <300),
|
||||
// but ts.Node.Kind() crosses CGO and allocates a fresh Go string on
|
||||
// every call; a single index walks millions of nodes, and profiling
|
||||
// put node-kind GoString conversions at ~22% of all allocations. The
|
||||
// table turns Type() into a slice index — zero CGO, zero allocation —
|
||||
// after a one-time build per language.
|
||||
//
|
||||
// Keyed by the C TSLanguage pointer (stable per registered grammar);
|
||||
// sync.Map because indexing runs many languages concurrently.
|
||||
var kindTables sync.Map // unsafe.Pointer(*C.TSLanguage) -> []string
|
||||
|
||||
// internedKind returns a node's type name from the per-language table,
|
||||
// building it on first use. Equivalent to ts.Node.Kind() because
|
||||
// ts_node_type is itself ts_language_symbol_name(language, symbol).
|
||||
// Falls back to the allocating Kind() only for an out-of-range symbol
|
||||
// id, which a well-formed grammar never produces.
|
||||
func internedKind(n ts.Node, key unsafe.Pointer) string {
|
||||
// key is the node's language identity, stamped at wrap time. A nil
|
||||
// key means the node was built on a path that didn't stamp it —
|
||||
// derive it the slow way so Type() still returns the right answer.
|
||||
if key == nil {
|
||||
key = unsafe.Pointer(n.Language().Inner)
|
||||
}
|
||||
id := n.KindId()
|
||||
v, ok := kindTables.Load(key)
|
||||
if !ok {
|
||||
lang := n.Language()
|
||||
cnt := lang.NodeKindCount()
|
||||
names := make([]string, cnt)
|
||||
for i := uint32(0); i < cnt; i++ {
|
||||
names[i] = lang.NodeKindForId(uint16(i))
|
||||
}
|
||||
v, _ = kindTables.LoadOrStore(key, names)
|
||||
}
|
||||
names := v.([]string)
|
||||
if int(id) < len(names) {
|
||||
return names[id]
|
||||
}
|
||||
return n.Kind()
|
||||
}
|
||||
|
||||
// Content returns the UTF-8 text of the node as a slice of src.
|
||||
func (n *Node) Content(src []byte) string { return n.inner.Utf8Text(src) }
|
||||
|
||||
// StartPoint returns the (row, column) position of the node start.
|
||||
func (n *Node) StartPoint() Point { return fromTSPoint(n.inner.StartPosition()) }
|
||||
|
||||
// EndPoint returns the (row, column) position one past the node end.
|
||||
func (n *Node) EndPoint() Point { return fromTSPoint(n.inner.EndPosition()) }
|
||||
|
||||
// StartByte returns the byte offset of the node start.
|
||||
func (n *Node) StartByte() uint32 { return uint32(n.inner.StartByte()) }
|
||||
|
||||
// EndByte returns the byte offset one past the node end.
|
||||
func (n *Node) EndByte() uint32 { return uint32(n.inner.EndByte()) }
|
||||
|
||||
// ChildCount returns the number of children (named + anonymous).
|
||||
func (n *Node) ChildCount() uint32 { return uint32(n.inner.ChildCount()) }
|
||||
|
||||
// NamedChildCount returns the number of named children.
|
||||
func (n *Node) NamedChildCount() uint32 { return uint32(n.inner.NamedChildCount()) }
|
||||
|
||||
// Child returns the i-th child (named or anonymous) or nil. It reaches the
|
||||
// child through a direct C call that returns the node by value, so the result
|
||||
// is bump-allocated in the arena with no go-tree-sitter heap node (newNode).
|
||||
func (n *Node) Child(i int) *Node {
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
c, ok := childDirect(n.inner, i)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// NamedChild returns the i-th named child or nil. Like Child, it avoids
|
||||
// go-tree-sitter's per-node heap allocation.
|
||||
func (n *Node) NamedChild(i int) *Node {
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
c, ok := namedChildDirect(n.inner, i)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// NamedChildren yields n's named children, in order, walking the sibling
|
||||
// chain once with a tree-sitter cursor. Visiting every named child costs
|
||||
// O(total children). The index form
|
||||
//
|
||||
// for i := 0; i < int(n.NamedChildCount()); i++ { c := n.NamedChild(i); … }
|
||||
//
|
||||
// is O(N^2): each NamedChild(i) re-walks the child list from the first
|
||||
// child to reach position i, so a loop over a very wide node (e.g. a
|
||||
// generated file's program root with thousands of top-level siblings)
|
||||
// degrades quadratically. This iterator stays linear.
|
||||
//
|
||||
// The visited set and order are identical to the NamedChild index form:
|
||||
// anonymous (unnamed) children are skipped and named children are
|
||||
// yielded in their natural child order.
|
||||
func (n *Node) NamedChildren() iter.Seq[*Node] {
|
||||
return func(yield func(*Node) bool) {
|
||||
if n == nil || !n.valid {
|
||||
return
|
||||
}
|
||||
cursor := n.inner.Walk()
|
||||
defer cursor.Close()
|
||||
if !cursor.GotoFirstChild() {
|
||||
return
|
||||
}
|
||||
for {
|
||||
c := cursorCurrentNode(cursor)
|
||||
if c.IsNamed() {
|
||||
if !yield(n.WrapVal(c)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if !cursor.GotoNextSibling() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ChildByFieldName returns the first child with the given field name or nil.
|
||||
// Uses a direct C call so the result is arena-allocated with no heap node.
|
||||
func (n *Node) ChildByFieldName(name string) *Node {
|
||||
c, ok := childByFieldNameDirect(n.inner, name)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// FieldNameForChild returns the field name of the i-th child, or "" if none.
|
||||
func (n *Node) FieldNameForChild(i int) string {
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
return n.inner.FieldNameForChild(uint32(i))
|
||||
}
|
||||
|
||||
// Parent returns the parent node or nil for the root. Avoids
|
||||
// go-tree-sitter's per-node heap allocation via a direct C call.
|
||||
func (n *Node) Parent() *Node {
|
||||
c, ok := parentDirect(n.inner)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// NextSibling returns the next sibling (named or anonymous) or nil. Direct C
|
||||
// call, arena-allocated result, no heap node.
|
||||
func (n *Node) NextSibling() *Node {
|
||||
c, ok := nextSiblingDirect(n.inner)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// PrevSibling returns the previous sibling (named or anonymous) or nil.
|
||||
func (n *Node) PrevSibling() *Node {
|
||||
c, ok := prevSiblingDirect(n.inner)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// NextNamedSibling returns the next named sibling or nil.
|
||||
func (n *Node) NextNamedSibling() *Node {
|
||||
c, ok := nextNamedSiblingDirect(n.inner)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// PrevNamedSibling returns the previous named sibling or nil.
|
||||
func (n *Node) PrevNamedSibling() *Node {
|
||||
c, ok := prevNamedSiblingDirect(n.inner)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return n.WrapVal(c)
|
||||
}
|
||||
|
||||
// IsNamed reports whether the node corresponds to a named grammar rule.
|
||||
func (n *Node) IsNamed() bool { return n.inner.IsNamed() }
|
||||
|
||||
// IsMissing reports whether the parser inserted this node to recover from an error.
|
||||
func (n *Node) IsMissing() bool { return n.inner.IsMissing() }
|
||||
|
||||
// IsError reports whether this is a synthetic ERROR node.
|
||||
func (n *Node) IsError() bool { return n.inner.IsError() }
|
||||
|
||||
// HasError reports whether the subtree under this node contains any ERROR nodes.
|
||||
func (n *Node) HasError() bool { return n.inner.HasError() }
|
||||
|
||||
// String returns the s-expression representation of the node.
|
||||
func (n *Node) String() string { return n.inner.ToSexp() }
|
||||
|
||||
// Id returns a stable numeric identity for the underlying node. Safe
|
||||
// to use as a map key; equal across multiple wrappers of the same
|
||||
// tree-sitter node. (Required because our shim creates a fresh *Node
|
||||
// on every traversal, so pointer identity is not meaningful.)
|
||||
func (n *Node) Id() uintptr {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
return n.inner.Id()
|
||||
}
|
||||
|
||||
// Equal reports whether two shim Nodes wrap the same underlying
|
||||
// tree-sitter node. Prefer this to `==` pointer comparison — our
|
||||
// wrappers are freshly allocated on every navigation.
|
||||
func (n *Node) Equal(other *Node) bool {
|
||||
if n == nil || other == nil {
|
||||
return n == other
|
||||
}
|
||||
return n.inner.Equals(other.inner)
|
||||
}
|
||||
|
||||
// Tree wraps *ts.Tree.
|
||||
type Tree struct {
|
||||
inner *ts.Tree
|
||||
arena *nodeArena // pooled; taken lazily on first RootNode, returned on Close
|
||||
}
|
||||
|
||||
// WrapTree wraps a *ts.Tree for internal use by the parser package.
|
||||
func WrapTree(t *ts.Tree) *Tree { return &Tree{inner: t} }
|
||||
|
||||
// Inner exposes the underlying *ts.Tree for internal use.
|
||||
func (t *Tree) Inner() *ts.Tree { return t.inner }
|
||||
|
||||
// RootNode returns the root node of the parse tree, stamped with the
|
||||
// tree's language so Type() lookups across the walk need no CGO call.
|
||||
func (t *Tree) RootNode() *Node {
|
||||
root := t.inner.RootNode()
|
||||
if root == nil {
|
||||
return nil
|
||||
}
|
||||
// Take a pooled arena on first use and reuse it for any later RootNode
|
||||
// call on the same tree, so every node walked from this tree allocates
|
||||
// into one recycled arena. Close() returns it to the pool.
|
||||
if t.arena == nil {
|
||||
t.arena = getArena()
|
||||
}
|
||||
a := t.arena
|
||||
nn := a.alloc()
|
||||
nn.inner = *root
|
||||
nn.valid = true
|
||||
nn.langKey = unsafe.Pointer(root.Language().Inner)
|
||||
nn.arena = a
|
||||
return nn
|
||||
}
|
||||
|
||||
// Close releases the tree's C resources and recycles its node arena.
|
||||
//
|
||||
// The arena is returned to the pool AFTER the C tree is freed: by the
|
||||
// Tree's contract every Node wrapper is dead once Close returns, so the
|
||||
// chunks the arena retains hold nothing live. putArena's reset() clears any
|
||||
// stale slot, so a recycled arena never pins a closed tree.
|
||||
func (t *Tree) Close() {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
if t.inner != nil {
|
||||
t.inner.Close()
|
||||
t.inner = nil
|
||||
}
|
||||
if t.arena != nil {
|
||||
putArena(t.arena)
|
||||
t.arena = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Parser wraps *ts.Parser with a ParseCtx that honours ctx cancellation
|
||||
// via the new API's progress callback hook.
|
||||
type Parser struct {
|
||||
inner *ts.Parser
|
||||
}
|
||||
|
||||
// NewParser allocates a fresh parser. The caller must Close it.
|
||||
func NewParser() *Parser { return &Parser{inner: ts.NewParser()} }
|
||||
|
||||
// Close releases the parser's C resources.
|
||||
func (p *Parser) Close() {
|
||||
if p != nil && p.inner != nil {
|
||||
p.inner.Close()
|
||||
p.inner = nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetLanguage binds a grammar to the parser. Errors from the new API
|
||||
// (incompatible ABI versions) are swallowed to keep the smacker-style
|
||||
// void return; callers trust build-time grammar selection.
|
||||
func (p *Parser) SetLanguage(lang *Language) { _ = p.inner.SetLanguage(lang) }
|
||||
|
||||
// Reset clears retained parse state (finished tree, old-tree refs,
|
||||
// stack, cached token) so a parser that parsed cleanly can be reused
|
||||
// for an unrelated document. It does not clear the bound language.
|
||||
//
|
||||
// Reset does NOT fully sanitise a parser whose parse was cancelled:
|
||||
// ts_parser_reset leaves the C parser's canceled_balancing flag set,
|
||||
// so a parse cancelled during the balancing phase poisons the parser
|
||||
// permanently — the next Parse jumps to the balance label and aborts
|
||||
// the process on an internal assertion. Discard (Close) an errored
|
||||
// parser; never Reset-and-reuse it. See parser.ParseFile.
|
||||
func (p *Parser) Reset() {
|
||||
if p != nil && p.inner != nil {
|
||||
p.inner.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
// ParseCtx parses src under ctx's deadline, returning a *Tree the
|
||||
// caller must Close. Cancellation is polled via a ProgressCallback;
|
||||
// exact-to-the-byte interruption isn't guaranteed — tree-sitter calls
|
||||
// the callback at its own cadence.
|
||||
func (p *Parser) ParseCtx(ctx context.Context, old *Tree, src []byte) (*Tree, error) {
|
||||
var oldTree *ts.Tree
|
||||
if old != nil {
|
||||
oldTree = old.inner
|
||||
}
|
||||
cancelled := false
|
||||
opts := &ts.ParseOptions{
|
||||
ProgressCallback: func(_ ts.ParseState) bool {
|
||||
if ctx.Err() != nil {
|
||||
cancelled = true
|
||||
return true // true aborts the parse
|
||||
}
|
||||
return false
|
||||
},
|
||||
}
|
||||
tree := p.inner.ParseWithOptions(func(offset int, _ ts.Point) []byte {
|
||||
if offset >= len(src) {
|
||||
return nil
|
||||
}
|
||||
return src[offset:]
|
||||
}, oldTree, opts)
|
||||
if tree == nil {
|
||||
if cancelled {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.New("tree-sitter: parse cancelled")
|
||||
}
|
||||
return nil, fmt.Errorf("tree-sitter: parse returned nil")
|
||||
}
|
||||
return &Tree{inner: tree}, nil
|
||||
}
|
||||
|
||||
// Query is a compiled tree-sitter query. It caches CaptureNames so
|
||||
// capture-id → name lookups are O(1) and don't cross into CGO.
|
||||
type Query struct {
|
||||
inner *ts.Query
|
||||
names []string
|
||||
}
|
||||
|
||||
// NewQuery compiles a query pattern against a language. Signature
|
||||
// matches smacker's (pattern, lang) order, which is the argument order
|
||||
// most of our language adapters expect.
|
||||
func NewQuery(pattern []byte, lang *Language) (*Query, error) {
|
||||
q, qerr := ts.NewQuery(lang, string(pattern))
|
||||
if qerr != nil {
|
||||
return nil, errors.New(qerr.Error())
|
||||
}
|
||||
return &Query{inner: q, names: q.CaptureNames()}, nil
|
||||
}
|
||||
|
||||
// Inner exposes the underlying *ts.Query for internal query runners.
|
||||
func (q *Query) Inner() *ts.Query { return q.inner }
|
||||
|
||||
// Close releases the query's C resources.
|
||||
func (q *Query) Close() {
|
||||
if q != nil && q.inner != nil {
|
||||
q.inner.Close()
|
||||
q.inner = nil
|
||||
}
|
||||
}
|
||||
|
||||
// CaptureNameForId returns the capture name for a capture index.
|
||||
func (q *Query) CaptureNameForId(id uint32) string {
|
||||
if int(id) >= len(q.names) {
|
||||
return ""
|
||||
}
|
||||
return q.names[id]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Package tsx re-exports the TSX variant of tree-sitter-typescript
|
||||
// for .tsx files.
|
||||
package tsx
|
||||
|
||||
import (
|
||||
tree_sitter_typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_typescript.LanguageTSX())
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Package typescript re-exports the tree-sitter-typescript grammar.
|
||||
// Smacker's path was `typescript/typescript`; the flat `typescript`
|
||||
// package here is the shorter equivalent.
|
||||
package typescript
|
||||
|
||||
import (
|
||||
tree_sitter_typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_typescript.LanguageTypescript())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Package yaml re-exports tree-sitter-grammars/tree-sitter-yaml.
|
||||
package yaml
|
||||
|
||||
import (
|
||||
tree_sitter_yaml "github.com/tree-sitter-grammars/tree-sitter-yaml/bindings/go"
|
||||
"github.com/zzet/gortex/internal/parser/tsitter"
|
||||
)
|
||||
|
||||
func GetLanguage() *tsitter.Language {
|
||||
return tsitter.NewLanguage(tree_sitter_yaml.Language())
|
||||
}
|
||||
Reference in New Issue
Block a user