chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:33:42 +08:00
commit a06f331eb8
3186 changed files with 689843 additions and 0 deletions
@@ -0,0 +1,46 @@
// Hot-path workaround: the upstream tree-sitter/go-tree-sitter
// (v0.25.0) registers a Go-routed allocator at package init —
// `ts_set_allocator(c_malloc_fn, …)` — and `c_malloc_fn` calls back
// into Go (`go_malloc`), which then calls `C.malloc`. Every internal
// allocation inside the parser therefore round-trips C → Go → C
// instead of going straight to libc. On large repos that round-trip
// dominated the cgo cost: `_cgoexp_…go_malloc` showed up at ~35% of
// total CPU in our indexer profile, and indexing the gortex repo
// itself ran 3× slower than the smacker baseline.
//
// We undo that by re-calling `ts_set_allocator(NULL, …, NULL)` from
// our own init, which the C library treats as "use libc directly."
// The symbol `ts_set_allocator` is exported from the upstream
// package's CGO archive, so we declare it `extern` and link against
// the same archive.
//
// Init ordering: Go runs init() functions in import-graph order, and
// this file is in the `tsitter` package which depends on the upstream
// `tree_sitter` package — so the upstream init (which registers the
// indirection) runs first, ours runs after, and our reset wins.
//
// Tree-sitter's docs warn that switching allocators after objects
// have been created is unsafe unless those objects are freed first
// or the new allocator shares state with the old. Both go-routed and
// libc allocators ultimately call `C.malloc`/`C.free`, so they share
// state — the round-trip layer drops away cleanly.
package tsitter
/*
extern void ts_set_allocator(
void *(*new_malloc)(unsigned long),
void *(*new_calloc)(unsigned long, unsigned long),
void *(*new_realloc)(void *, unsigned long),
void (*new_free)(void *)
);
static void gortex_use_libc_allocator(void) {
ts_set_allocator(0, 0, 0, 0);
}
*/
import "C"
func init() {
C.gortex_use_libc_allocator()
}
+80
View File
@@ -0,0 +1,80 @@
package tsitter
import "testing"
// TestNodeArenaStablePointers guards the arena's load-bearing invariant:
// a pointer returned by alloc stays valid after later allocations grow the
// arena onto a new backing chunk. If alloc ever appended into a single
// reallocating slice, earlier &slice[i] pointers would dangle and node data
// would silently corrupt mid-walk.
func TestNodeArenaStablePointers(t *testing.T) {
a := newNodeArena()
const n = arenaMaxChunk*2 + 17 // cross several chunk boundaries incl. the max-size cap
ptrs := make([]*Node, n)
for i := range ptrs {
p := a.alloc()
if p == nil {
t.Fatalf("alloc %d returned nil", i)
}
p.valid = true
ptrs[i] = p
}
seen := make(map[*Node]bool, n)
for i, p := range ptrs {
if seen[p] {
t.Fatalf("alloc %d aliased an earlier node pointer", i)
}
seen[p] = true
if !p.valid {
t.Fatalf("node %d was clobbered by a later chunk allocation", i)
}
}
}
// TestNodeArenaGeometricGrowth confirms the first chunk is small (low waste
// for tiny files) and chunks grow up to the cap (few objects for deep files).
func TestNodeArenaGeometricGrowth(t *testing.T) {
a := newNodeArena()
a.alloc()
if got := len(a.chunks[0]); got != arenaFirstChunk {
t.Fatalf("first chunk size = %d, want %d", got, arenaFirstChunk)
}
// Drain well past the cap; no chunk may exceed the max.
for i := 0; i < arenaMaxChunk*3; i++ {
a.alloc()
if got := len(a.chunks[a.ci]); got > arenaMaxChunk {
t.Fatalf("chunk grew to %d, exceeds cap %d", got, arenaMaxChunk)
}
}
}
// TestNodeArenaResetReusesChunks is the pooling invariant: reset rewinds the
// cursor but KEEPS the backing chunks, so refilling to the same size
// allocates no new chunk, and any stale Node from the previous round is
// cleared — a recycled arena must never pin a closed tree.
func TestNodeArenaResetReusesChunks(t *testing.T) {
a := newNodeArena()
const n = arenaMaxChunk + 100 // span more than one chunk
for i := 0; i < n; i++ {
a.alloc().valid = true
}
want := len(a.chunks)
if want < 2 {
t.Fatalf("test needs a multi-chunk arena, got %d chunk(s)", want)
}
a.reset()
if len(a.chunks) != want {
t.Fatalf("reset dropped chunks: have %d, want %d (chunks must be retained)", len(a.chunks), want)
}
if a.chunks[0][0].valid {
t.Fatal("reset left a stale Node valid — a recycled arena would pin its tree")
}
for i := 0; i < n; i++ {
a.alloc().valid = true
}
if len(a.chunks) != want {
t.Fatalf("refill allocated fresh chunks: have %d, want %d (chunks not reused)", len(a.chunks), want)
}
}
+12
View File
@@ -0,0 +1,12 @@
// Package bash re-exports the tree-sitter-bash grammar in a
// smacker-compatible shape (GetLanguage() *tsitter.Language).
package bash
import (
tree_sitter_bash "github.com/tree-sitter/tree-sitter-bash/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_bash.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package c re-exports the tree-sitter-c grammar.
package c
import (
tree_sitter_c "github.com/tree-sitter/tree-sitter-c/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_c.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package cpp re-exports the tree-sitter-cpp grammar.
package cpp
import (
tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_cpp.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package csharp re-exports the tree-sitter-c-sharp grammar.
package csharp
import (
tree_sitter_csharp "github.com/tree-sitter/tree-sitter-c-sharp/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_csharp.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package css re-exports the tree-sitter-css grammar.
package css
import (
tree_sitter_css "github.com/tree-sitter/tree-sitter-css/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_css.Language())
}
+15
View File
@@ -0,0 +1,15 @@
// Package dart re-exports the tree-sitter-dart grammar. The C parser
// lives in the sibling github.com/gortexhq/tree-sitter-dart module;
// this file is just the thin shim that bridges the upstream binding
// into gortex's *tsitter.Language type.
package dart
import (
tree_sitter_dart "github.com/gortexhq/tree-sitter-dart/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Dart language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_dart.Language())
}
@@ -0,0 +1,15 @@
// Package dockerfile re-exports the tree-sitter-dockerfile grammar.
// The C parser lives in the sibling github.com/gortexhq/tree-sitter-dockerfile
// module; this file is just the thin shim that bridges the upstream
// binding into gortex's *tsitter.Language type.
package dockerfile
import (
tree_sitter_dockerfile "github.com/gortexhq/tree-sitter-dockerfile/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Dockerfile language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_dockerfile.Language())
}
+15
View File
@@ -0,0 +1,15 @@
// Package elixir re-exports tree-sitter-elixir.
// The upstream repo declares its module path as
// github.com/tree-sitter/tree-sitter-elixir (it was donated from
// elixir-lang); go.mod pins a replace directive to the elixir-lang
// origin so this import resolves.
package elixir
import (
tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_elixir.Language())
}
+13
View File
@@ -0,0 +1,13 @@
// Package golang re-exports the tree-sitter-go grammar.
// The package name mirrors smacker's `golang` so existing extractor
// imports only need a path change.
package golang
import (
tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_go.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package hcl re-exports tree-sitter-grammars/tree-sitter-hcl.
package hcl
import (
tree_sitter_hcl "github.com/tree-sitter-grammars/tree-sitter-hcl/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_hcl.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package html re-exports the tree-sitter-html grammar.
package html
import (
tree_sitter_html "github.com/tree-sitter/tree-sitter-html/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_html.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package java re-exports the tree-sitter-java grammar.
package java
import (
tree_sitter_java "github.com/tree-sitter/tree-sitter-java/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_java.Language())
}
@@ -0,0 +1,11 @@
// Package javascript re-exports the tree-sitter-javascript grammar.
package javascript
import (
tree_sitter_javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_javascript.Language())
}
+15
View File
@@ -0,0 +1,15 @@
// Package kotlin re-exports fwcd/tree-sitter-kotlin. The
// tree-sitter-grammars/tree-sitter-kotlin v1.x grammar renamed many
// node kinds (e.g. dropped the `type_identifier` alias), which would
// force a substantial rewrite of our Kotlin extractor. fwcd's grammar
// matches the node vocabulary smacker shipped, so we keep that.
package kotlin
import (
tree_sitter_kotlin "github.com/fwcd/tree-sitter-kotlin/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_kotlin.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package lua re-exports tree-sitter-grammars/tree-sitter-lua.
package lua
import (
tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_lua.Language())
}
@@ -0,0 +1,17 @@
// Package markdown re-exports the tree-sitter-markdown grammar. The C
// parser lives in the sibling github.com/gortexhq/tree-sitter-markdown
// module (a fork of the tree-sitter-recommended
// tree-sitter-grammars/tree-sitter-markdown block-level parser with
// Go bindings added); this file is just the thin shim that bridges
// the upstream binding into gortex's *tsitter.Language type.
package markdown
import (
tree_sitter_markdown "github.com/gortexhq/tree-sitter-markdown/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Markdown language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_markdown.Language())
}
+158
View File
@@ -0,0 +1,158 @@
package tsitter
// Direct C navigation for the hot child-access path.
//
// go-tree-sitter's Node navigation (Child / NamedChild / Parent, the cursor,
// even the []Node value-slice helpers) funnels through newNode, which
// heap-allocates a *ts.Node for every visited node. Profiling a large index
// put that at ~43% of all bytes allocated — 95% of it NamedChild + Child +
// Parent in the extractor child loops — and the resulting GC churn dominated
// CPU under a memory cap.
//
// The tree-sitter C API returns nodes BY VALUE (TSNode is a 32-byte struct).
// We call those C functions directly and drop the returned value straight
// into the pooled Node arena, so a child visit costs one bump-allocated
// wrapper and zero heap garbage.
//
// The functions are declared here rather than pulled from <tree_sitter/api.h>
// so this file needs no -I path into the go-tree-sitter module cache (which
// is version- and machine-specific). The symbols themselves are defined in
// the tree-sitter C library that go-tree-sitter already compiles and links
// into the binary; C has no name mangling, so a local prototype with the
// identical ABI resolves to them at link time.
/*
#include <stdint.h>
// Mirror of tree-sitter's TSNode (include/tree_sitter/api.h): a 4-word
// context plus two opaque pointers. Layout must match exactly — it is
// reinterpreted from ts.Node, whose sole field is this struct.
typedef struct { uint32_t context[4]; const void *id; const void *tree; } GxTSNode;
// Mirror of TSTreeCursor — reinterpreted from *ts.TreeCursor (whose sole
// field is this struct) so the cursor's current node can be read by value.
typedef struct { const void *tree; const void *id; uint32_t context[3]; } GxTSTreeCursor;
extern GxTSNode ts_node_child(GxTSNode, uint32_t);
extern GxTSNode ts_node_named_child(GxTSNode, uint32_t);
extern GxTSNode ts_node_parent(GxTSNode);
extern GxTSNode ts_node_next_sibling(GxTSNode);
extern GxTSNode ts_node_prev_sibling(GxTSNode);
extern GxTSNode ts_node_next_named_sibling(GxTSNode);
extern GxTSNode ts_node_prev_named_sibling(GxTSNode);
extern GxTSNode ts_node_child_by_field_name(GxTSNode, const char *, uint32_t);
extern GxTSNode ts_tree_cursor_current_node(const GxTSTreeCursor *);
*/
import "C"
import (
"unsafe"
ts "github.com/tree-sitter/go-tree-sitter"
)
// init fails fast if a tree-sitter upgrade ever changes the TSNode layout
// out from under the reinterpret casts below, rather than silently corrupting
// navigated nodes at runtime.
func init() {
if unsafe.Sizeof(ts.Node{}) != unsafe.Sizeof(C.GxTSNode{}) {
panic("tsitter: ts.Node and TSNode size mismatch — tree-sitter ABI changed, update node_cnav.go")
}
if unsafe.Sizeof(ts.TreeCursor{}) != unsafe.Sizeof(C.GxTSTreeCursor{}) {
panic("tsitter: ts.TreeCursor and TSTreeCursor size mismatch — tree-sitter ABI changed, update node_cnav.go")
}
}
// asC reinterprets an upstream ts.Node as the locally-declared C node. ts.Node
// is `struct { _inner C.TSNode }`, so its bytes are exactly a TSNode and the
// two share an identical ABI layout.
func asC(n ts.Node) C.GxTSNode { return *(*C.GxTSNode)(unsafe.Pointer(&n)) }
// asGo reinterprets a C node back into an upstream ts.Node value.
func asGo(c C.GxTSNode) ts.Node { return *(*ts.Node)(unsafe.Pointer(&c)) }
// childDirect returns parent's i-th child as a value, with no heap
// allocation. ok is false for a null child (index past the end).
func childDirect(parent ts.Node, i int) (ts.Node, bool) {
c := C.ts_node_child(asC(parent), C.uint32_t(i))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
// namedChildDirect is childDirect over named children only.
func namedChildDirect(parent ts.Node, i int) (ts.Node, bool) {
c := C.ts_node_named_child(asC(parent), C.uint32_t(i))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
// parentDirect returns n's parent as a value. ok is false at the root.
func parentDirect(n ts.Node) (ts.Node, bool) {
c := C.ts_node_parent(asC(n))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
func nextSiblingDirect(n ts.Node) (ts.Node, bool) {
c := C.ts_node_next_sibling(asC(n))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
func prevSiblingDirect(n ts.Node) (ts.Node, bool) {
c := C.ts_node_prev_sibling(asC(n))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
func nextNamedSiblingDirect(n ts.Node) (ts.Node, bool) {
c := C.ts_node_next_named_sibling(asC(n))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
func prevNamedSiblingDirect(n ts.Node) (ts.Node, bool) {
c := C.ts_node_prev_named_sibling(asC(n))
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
// cursorCurrentNode returns a tree cursor's current node by value, avoiding
// TreeCursor.Node's per-step heap *Node. The cursor walk itself (GotoFirstChild
// / GotoNextSibling) stays O(total children); only the node read is changed.
func cursorCurrentNode(cursor *ts.TreeCursor) ts.Node {
return asGo(C.ts_tree_cursor_current_node((*C.GxTSTreeCursor)(unsafe.Pointer(cursor))))
}
// childByFieldNameDirect returns parent's child for a grammar field name. The
// name's bytes are passed to C by pointer+length (ts_node_child_by_field_name
// does not require NUL termination and reads them only for the duration of the
// call), so no C string is allocated. ok is false when no such field exists.
func childByFieldNameDirect(parent ts.Node, name string) (ts.Node, bool) {
if name == "" {
return ts.Node{}, false
}
c := C.ts_node_child_by_field_name(
asC(parent),
(*C.char)(unsafe.Pointer(unsafe.StringData(name))),
C.uint32_t(len(name)),
)
if c.id == nil {
return ts.Node{}, false
}
return asGo(c), true
}
+13
View File
@@ -0,0 +1,13 @@
// Package ocaml re-exports the tree-sitter-ocaml grammar (OCaml
// implementation variant, .ml files). Interface (.mli) and type
// bindings exist in sister packages but gortex only indexes .ml.
package ocaml
import (
tree_sitter_ocaml "github.com/tree-sitter/tree-sitter-ocaml/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_ocaml.LanguageOCaml())
}
@@ -0,0 +1,16 @@
// Package orgmode re-exports the tree-sitter-org-mode grammar. The C
// parser lives in the sibling github.com/gortexhq/tree-sitter-org-mode
// module (a vendored fork of zac-garby/tree-sitter-org-mode); this file
// is the thin shim that bridges the upstream binding into gortex's
// *tsitter.Language type.
package orgmode
import (
tree_sitter_orgmode "github.com/gortexhq/tree-sitter-org-mode/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Org-mode language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_orgmode.Language())
}
+12
View File
@@ -0,0 +1,12 @@
// Package php re-exports the tree-sitter-php grammar (full-mode PHP
// with HTML framing; matches smacker's default php binding).
package php
import (
tree_sitter_php "github.com/tree-sitter/tree-sitter-php/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_php.LanguagePHP())
}
@@ -0,0 +1,15 @@
// Package protobuf re-exports the tree-sitter-proto grammar. The C
// parser lives in the sibling github.com/gortexhq/tree-sitter-protobuf
// module; this file is just the thin shim that bridges the upstream
// binding into gortex's *tsitter.Language type.
package protobuf
import (
tree_sitter_protobuf "github.com/gortexhq/tree-sitter-protobuf/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Protobuf language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_protobuf.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package python re-exports the tree-sitter-python grammar.
package python
import (
tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_python.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package ruby re-exports the tree-sitter-ruby grammar.
package ruby
import (
tree_sitter_ruby "github.com/tree-sitter/tree-sitter-ruby/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_ruby.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package rust re-exports the tree-sitter-rust grammar.
package rust
import (
tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_rust.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package scala re-exports the tree-sitter-scala grammar.
package scala
import (
tree_sitter_scala "github.com/tree-sitter/tree-sitter-scala/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_scala.Language())
}
+15
View File
@@ -0,0 +1,15 @@
// Package sql re-exports the tree-sitter-sql grammar. The C parser
// lives in the sibling github.com/gortexhq/tree-sitter-sql module;
// this file is just the thin shim that bridges the upstream binding
// into gortex's *tsitter.Language type.
package sql
import (
tree_sitter_sql "github.com/gortexhq/tree-sitter-sql/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled SQL language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_sql.Language())
}
+17
View File
@@ -0,0 +1,17 @@
// Package swift re-exports the tree-sitter-swift grammar. The C parser
// lives in the sibling github.com/gortexhq/tree-sitter-swift module (a
// fork of the tree-sitter-recommended alex-pinkus/tree-sitter-swift
// with a regenerated parser.c committed); this file is just the thin
// shim that bridges the upstream binding into gortex's *tsitter.Language
// type.
package swift
import (
tree_sitter_swift "github.com/gortexhq/tree-sitter-swift/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
// GetLanguage returns the compiled Swift language.
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_swift.Language())
}
+11
View File
@@ -0,0 +1,11 @@
// Package toml re-exports tree-sitter-grammars/tree-sitter-toml.
package toml
import (
tree_sitter_toml "github.com/tree-sitter-grammars/tree-sitter-toml/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_toml.Language())
}
+606
View File
@@ -0,0 +1,606 @@
// Package tsitter is a thin compatibility shim over
// github.com/tree-sitter/go-tree-sitter. Its surface intentionally
// mirrors the smacker/go-tree-sitter API so that the ~90 language
// extractors in gortex can be migrated by changing only import paths —
// method names and signatures stay the same.
//
// The shim wraps native tree-sitter types (Node, Tree, Parser, Query)
// and adapts:
// - Type() → Kind()
// - Content(src) → Utf8Text(src)
// - StartPoint/EndPoint → StartPosition/EndPosition (uint32 Row/Column)
// - int-indexed children → uint-indexed (internally converted)
// - ParseCtx(ctx, old, src) built on top of ParseWithOptions
//
// The cursor-based query iteration is not re-exposed here: it lives in
// the parent parser package, which uses the official API directly.
package tsitter
import (
"context"
"errors"
"fmt"
"iter"
"sync"
"unsafe"
ts "github.com/tree-sitter/go-tree-sitter"
)
// Language is a parser language. Exposed as an alias so grammar
// sub-packages can return *ts.Language directly.
type Language = ts.Language
// NewLanguage constructs a Language from a grammar's raw C pointer —
// used by the per-language shim sub-packages.
func NewLanguage(ptr unsafe.Pointer) *Language { return ts.NewLanguage(ptr) }
// Point mirrors the smacker Point layout (uint32 row/column).
type Point struct {
Row uint32
Column uint32
}
func fromTSPoint(p ts.Point) Point {
return Point{Row: uint32(p.Row), Column: uint32(p.Column)}
}
// Node wraps *ts.Node with smacker-compatible method names. Nodes are
// valid for the lifetime of their Tree; copying by value is cheap
// (single C struct field).
type Node struct {
inner ts.Node
// set true when constructed; distinguishes the zero value from a real node.
valid bool
// langKey is the stable C-pointer identity of this node's language,
// propagated from the root through every navigation step so Type()
// can resolve the node kind from a per-language table with no CGO
// call and no allocation. Nil means "not stamped" — Type() then
// derives the language, which is slower but still correct.
langKey unsafe.Pointer
// arena bump-allocates Node wrappers in chunks so a deep tree walk
// produces a few large backing arrays instead of millions of tiny heap
// objects. The per-object GC mark cost dominated CPU when indexing a
// large TS monorepo (vscode: ~70% of cycles in GC, scanning millions of
// 1-node spans). Chunks are never explicitly freed — they are reachable
// only through the Nodes that point into them and are reclaimed by the
// GC once the tree's nodes are dropped. A nil arena falls back to a plain
// heap Node (zero-value and SetInner-pooled nodes have no arena).
arena *nodeArena
}
// nodeArena is a per-tree bump allocator for Node wrappers. It is not
// safe for concurrent use; each parse tree is walked by a single
// goroutine, and distinct files use distinct trees (and arenas).
//
// Arenas are pooled and reused across files (see arenaPool): a Tree takes
// one on its first RootNode() and returns it on Close(). reset() rewinds
// the allocation cursor but RETAINS the backing chunks, so a warm pool
// serves each file's node count with no fresh allocation. This is the
// dominant GC-pressure lever on a large index: profiling vscode and
// kubernetes put tree-sitter Node wrappers at 7082% of every byte
// allocated, almost all of it per-file chunk garbage. Retaining the chunks
// turns that churn into a bounded, reused working set.
type nodeArena struct {
chunks [][]Node // backing arrays, retained across resets for reuse
ci int // index of the current chunk within chunks
used int // slots used in chunks[ci]
}
const (
// arenaFirstChunk keeps the first backing array small so a file with a
// handful of nodes (the common case in a many-small-files repo) wastes
// little; chunks then double up to arenaMaxChunk so a deep file still
// ends up with only a few large objects.
arenaFirstChunk = 64
arenaMaxChunk = 4096
)
func newNodeArena() *nodeArena { return &nodeArena{} }
// alloc returns a pointer to a fresh Node. The pointer is stable for the
// life of the arena: a chunk is never resized in place — when the current
// chunk fills, allocation advances to the next retained chunk, or appends a
// geometrically larger one past the high-water mark, so earlier &chunk[i]
// pointers never move. Callers overwrite all fields of the returned Node
// immediately, so a reused slot needs no zeroing here; reset() clears stale
// slots when the arena is recycled.
func (a *nodeArena) alloc() *Node {
switch {
case len(a.chunks) == 0:
a.chunks = append(a.chunks, make([]Node, arenaFirstChunk))
a.ci, a.used = 0, 0
case a.used >= len(a.chunks[a.ci]):
a.ci++
if a.ci >= len(a.chunks) {
size := len(a.chunks[a.ci-1]) * 2
if size > arenaMaxChunk {
size = arenaMaxChunk
}
a.chunks = append(a.chunks, make([]Node, size))
}
a.used = 0
}
n := &a.chunks[a.ci][a.used]
a.used++
return n
}
// reset rewinds the allocation cursor to the start while retaining the
// backing chunks for reuse. It clears the slots touched since the last
// reset so a stale Node value — whose embedded ts.Node pins its now-closed
// *ts.Tree — cannot survive into the next file and leak. Clearing only the
// used prefix keeps the cost proportional to the file just processed, not
// the high-water capacity.
func (a *nodeArena) reset() {
for i := 0; i <= a.ci && i < len(a.chunks); i++ {
end := len(a.chunks[i])
if i == a.ci {
end = a.used
}
clear(a.chunks[i][:end])
}
a.ci, a.used = 0, 0
}
// arenaPool recycles per-tree arenas — with their retained chunks — across
// files. A Tree gets one lazily on its first RootNode() and returns it on
// Close(). sync.Pool may drop entries under GC pressure; a cold Get then
// just starts with no chunks and warms up again, so correctness never
// depends on retention.
var arenaPool = sync.Pool{New: func() any { return &nodeArena{} }}
func getArena() *nodeArena { return arenaPool.Get().(*nodeArena) }
func putArena(a *nodeArena) {
if a == nil {
return
}
a.reset()
arenaPool.Put(a)
}
// WrapNode wraps a value Node from the new API into our shim. It derives
// the language key eagerly so navigation from the result stays alloc-free,
// and seeds a fresh arena so the subtree walk below it allocates in chunks.
func WrapNode(n ts.Node) *Node {
a := newNodeArena()
nn := a.alloc()
nn.inner = n
nn.valid = true
nn.langKey = unsafe.Pointer(n.Language().Inner)
nn.arena = a
return nn
}
// WrapVal wraps a ts.Node reached from n (e.g. a query capture),
// carrying n's language key so Type() on the result and its descendants
// needs neither CGO nor allocation.
func (n *Node) WrapVal(c ts.Node) *Node {
if n.arena == nil {
return &Node{inner: c, valid: true, langKey: n.langKey}
}
nn := n.arena.alloc()
nn.inner = c
nn.valid = true
nn.langKey = n.langKey
nn.arena = n.arena
return nn
}
// SetInner overwrites the receiver's wrapped ts.Node and marks it
// valid. Lets callers reuse a *Node out of a pool / backing slice
// instead of allocating a new one per query match — see EachMatch in
// internal/parser/treesitter.go. The receiver must already exist
// (caller-owned), so SetInner cannot be used on a nil pointer.
func (n *Node) SetInner(inner ts.Node) {
n.inner = inner
n.valid = true
}
// Inner returns a pointer to the underlying ts.Node. Internal use by
// the parser package's query runners.
func (n *Node) Inner() *ts.Node {
if n == nil || !n.valid {
return nil
}
return &n.inner
}
// Type returns the node kind string ("identifier", "function_declaration", …).
func (n *Node) Type() string { return internedKind(n.inner, n.langKey) }
// kindTables memoises per-language node-kind name tables. tree-sitter
// node kinds are a small fixed set (NodeKindCount — typically <300),
// but ts.Node.Kind() crosses CGO and allocates a fresh Go string on
// every call; a single index walks millions of nodes, and profiling
// put node-kind GoString conversions at ~22% of all allocations. The
// table turns Type() into a slice index — zero CGO, zero allocation —
// after a one-time build per language.
//
// Keyed by the C TSLanguage pointer (stable per registered grammar);
// sync.Map because indexing runs many languages concurrently.
var kindTables sync.Map // unsafe.Pointer(*C.TSLanguage) -> []string
// internedKind returns a node's type name from the per-language table,
// building it on first use. Equivalent to ts.Node.Kind() because
// ts_node_type is itself ts_language_symbol_name(language, symbol).
// Falls back to the allocating Kind() only for an out-of-range symbol
// id, which a well-formed grammar never produces.
func internedKind(n ts.Node, key unsafe.Pointer) string {
// key is the node's language identity, stamped at wrap time. A nil
// key means the node was built on a path that didn't stamp it —
// derive it the slow way so Type() still returns the right answer.
if key == nil {
key = unsafe.Pointer(n.Language().Inner)
}
id := n.KindId()
v, ok := kindTables.Load(key)
if !ok {
lang := n.Language()
cnt := lang.NodeKindCount()
names := make([]string, cnt)
for i := uint32(0); i < cnt; i++ {
names[i] = lang.NodeKindForId(uint16(i))
}
v, _ = kindTables.LoadOrStore(key, names)
}
names := v.([]string)
if int(id) < len(names) {
return names[id]
}
return n.Kind()
}
// Content returns the UTF-8 text of the node as a slice of src.
func (n *Node) Content(src []byte) string { return n.inner.Utf8Text(src) }
// StartPoint returns the (row, column) position of the node start.
func (n *Node) StartPoint() Point { return fromTSPoint(n.inner.StartPosition()) }
// EndPoint returns the (row, column) position one past the node end.
func (n *Node) EndPoint() Point { return fromTSPoint(n.inner.EndPosition()) }
// StartByte returns the byte offset of the node start.
func (n *Node) StartByte() uint32 { return uint32(n.inner.StartByte()) }
// EndByte returns the byte offset one past the node end.
func (n *Node) EndByte() uint32 { return uint32(n.inner.EndByte()) }
// ChildCount returns the number of children (named + anonymous).
func (n *Node) ChildCount() uint32 { return uint32(n.inner.ChildCount()) }
// NamedChildCount returns the number of named children.
func (n *Node) NamedChildCount() uint32 { return uint32(n.inner.NamedChildCount()) }
// Child returns the i-th child (named or anonymous) or nil. It reaches the
// child through a direct C call that returns the node by value, so the result
// is bump-allocated in the arena with no go-tree-sitter heap node (newNode).
func (n *Node) Child(i int) *Node {
if i < 0 {
return nil
}
c, ok := childDirect(n.inner, i)
if !ok {
return nil
}
return n.WrapVal(c)
}
// NamedChild returns the i-th named child or nil. Like Child, it avoids
// go-tree-sitter's per-node heap allocation.
func (n *Node) NamedChild(i int) *Node {
if i < 0 {
return nil
}
c, ok := namedChildDirect(n.inner, i)
if !ok {
return nil
}
return n.WrapVal(c)
}
// NamedChildren yields n's named children, in order, walking the sibling
// chain once with a tree-sitter cursor. Visiting every named child costs
// O(total children). The index form
//
// for i := 0; i < int(n.NamedChildCount()); i++ { c := n.NamedChild(i); … }
//
// is O(N^2): each NamedChild(i) re-walks the child list from the first
// child to reach position i, so a loop over a very wide node (e.g. a
// generated file's program root with thousands of top-level siblings)
// degrades quadratically. This iterator stays linear.
//
// The visited set and order are identical to the NamedChild index form:
// anonymous (unnamed) children are skipped and named children are
// yielded in their natural child order.
func (n *Node) NamedChildren() iter.Seq[*Node] {
return func(yield func(*Node) bool) {
if n == nil || !n.valid {
return
}
cursor := n.inner.Walk()
defer cursor.Close()
if !cursor.GotoFirstChild() {
return
}
for {
c := cursorCurrentNode(cursor)
if c.IsNamed() {
if !yield(n.WrapVal(c)) {
return
}
}
if !cursor.GotoNextSibling() {
return
}
}
}
}
// ChildByFieldName returns the first child with the given field name or nil.
// Uses a direct C call so the result is arena-allocated with no heap node.
func (n *Node) ChildByFieldName(name string) *Node {
c, ok := childByFieldNameDirect(n.inner, name)
if !ok {
return nil
}
return n.WrapVal(c)
}
// FieldNameForChild returns the field name of the i-th child, or "" if none.
func (n *Node) FieldNameForChild(i int) string {
if i < 0 {
return ""
}
return n.inner.FieldNameForChild(uint32(i))
}
// Parent returns the parent node or nil for the root. Avoids
// go-tree-sitter's per-node heap allocation via a direct C call.
func (n *Node) Parent() *Node {
c, ok := parentDirect(n.inner)
if !ok {
return nil
}
return n.WrapVal(c)
}
// NextSibling returns the next sibling (named or anonymous) or nil. Direct C
// call, arena-allocated result, no heap node.
func (n *Node) NextSibling() *Node {
c, ok := nextSiblingDirect(n.inner)
if !ok {
return nil
}
return n.WrapVal(c)
}
// PrevSibling returns the previous sibling (named or anonymous) or nil.
func (n *Node) PrevSibling() *Node {
c, ok := prevSiblingDirect(n.inner)
if !ok {
return nil
}
return n.WrapVal(c)
}
// NextNamedSibling returns the next named sibling or nil.
func (n *Node) NextNamedSibling() *Node {
c, ok := nextNamedSiblingDirect(n.inner)
if !ok {
return nil
}
return n.WrapVal(c)
}
// PrevNamedSibling returns the previous named sibling or nil.
func (n *Node) PrevNamedSibling() *Node {
c, ok := prevNamedSiblingDirect(n.inner)
if !ok {
return nil
}
return n.WrapVal(c)
}
// IsNamed reports whether the node corresponds to a named grammar rule.
func (n *Node) IsNamed() bool { return n.inner.IsNamed() }
// IsMissing reports whether the parser inserted this node to recover from an error.
func (n *Node) IsMissing() bool { return n.inner.IsMissing() }
// IsError reports whether this is a synthetic ERROR node.
func (n *Node) IsError() bool { return n.inner.IsError() }
// HasError reports whether the subtree under this node contains any ERROR nodes.
func (n *Node) HasError() bool { return n.inner.HasError() }
// String returns the s-expression representation of the node.
func (n *Node) String() string { return n.inner.ToSexp() }
// Id returns a stable numeric identity for the underlying node. Safe
// to use as a map key; equal across multiple wrappers of the same
// tree-sitter node. (Required because our shim creates a fresh *Node
// on every traversal, so pointer identity is not meaningful.)
func (n *Node) Id() uintptr {
if n == nil {
return 0
}
return n.inner.Id()
}
// Equal reports whether two shim Nodes wrap the same underlying
// tree-sitter node. Prefer this to `==` pointer comparison — our
// wrappers are freshly allocated on every navigation.
func (n *Node) Equal(other *Node) bool {
if n == nil || other == nil {
return n == other
}
return n.inner.Equals(other.inner)
}
// Tree wraps *ts.Tree.
type Tree struct {
inner *ts.Tree
arena *nodeArena // pooled; taken lazily on first RootNode, returned on Close
}
// WrapTree wraps a *ts.Tree for internal use by the parser package.
func WrapTree(t *ts.Tree) *Tree { return &Tree{inner: t} }
// Inner exposes the underlying *ts.Tree for internal use.
func (t *Tree) Inner() *ts.Tree { return t.inner }
// RootNode returns the root node of the parse tree, stamped with the
// tree's language so Type() lookups across the walk need no CGO call.
func (t *Tree) RootNode() *Node {
root := t.inner.RootNode()
if root == nil {
return nil
}
// Take a pooled arena on first use and reuse it for any later RootNode
// call on the same tree, so every node walked from this tree allocates
// into one recycled arena. Close() returns it to the pool.
if t.arena == nil {
t.arena = getArena()
}
a := t.arena
nn := a.alloc()
nn.inner = *root
nn.valid = true
nn.langKey = unsafe.Pointer(root.Language().Inner)
nn.arena = a
return nn
}
// Close releases the tree's C resources and recycles its node arena.
//
// The arena is returned to the pool AFTER the C tree is freed: by the
// Tree's contract every Node wrapper is dead once Close returns, so the
// chunks the arena retains hold nothing live. putArena's reset() clears any
// stale slot, so a recycled arena never pins a closed tree.
func (t *Tree) Close() {
if t == nil {
return
}
if t.inner != nil {
t.inner.Close()
t.inner = nil
}
if t.arena != nil {
putArena(t.arena)
t.arena = nil
}
}
// Parser wraps *ts.Parser with a ParseCtx that honours ctx cancellation
// via the new API's progress callback hook.
type Parser struct {
inner *ts.Parser
}
// NewParser allocates a fresh parser. The caller must Close it.
func NewParser() *Parser { return &Parser{inner: ts.NewParser()} }
// Close releases the parser's C resources.
func (p *Parser) Close() {
if p != nil && p.inner != nil {
p.inner.Close()
p.inner = nil
}
}
// SetLanguage binds a grammar to the parser. Errors from the new API
// (incompatible ABI versions) are swallowed to keep the smacker-style
// void return; callers trust build-time grammar selection.
func (p *Parser) SetLanguage(lang *Language) { _ = p.inner.SetLanguage(lang) }
// Reset clears retained parse state (finished tree, old-tree refs,
// stack, cached token) so a parser that parsed cleanly can be reused
// for an unrelated document. It does not clear the bound language.
//
// Reset does NOT fully sanitise a parser whose parse was cancelled:
// ts_parser_reset leaves the C parser's canceled_balancing flag set,
// so a parse cancelled during the balancing phase poisons the parser
// permanently — the next Parse jumps to the balance label and aborts
// the process on an internal assertion. Discard (Close) an errored
// parser; never Reset-and-reuse it. See parser.ParseFile.
func (p *Parser) Reset() {
if p != nil && p.inner != nil {
p.inner.Reset()
}
}
// ParseCtx parses src under ctx's deadline, returning a *Tree the
// caller must Close. Cancellation is polled via a ProgressCallback;
// exact-to-the-byte interruption isn't guaranteed — tree-sitter calls
// the callback at its own cadence.
func (p *Parser) ParseCtx(ctx context.Context, old *Tree, src []byte) (*Tree, error) {
var oldTree *ts.Tree
if old != nil {
oldTree = old.inner
}
cancelled := false
opts := &ts.ParseOptions{
ProgressCallback: func(_ ts.ParseState) bool {
if ctx.Err() != nil {
cancelled = true
return true // true aborts the parse
}
return false
},
}
tree := p.inner.ParseWithOptions(func(offset int, _ ts.Point) []byte {
if offset >= len(src) {
return nil
}
return src[offset:]
}, oldTree, opts)
if tree == nil {
if cancelled {
if err := ctx.Err(); err != nil {
return nil, err
}
return nil, errors.New("tree-sitter: parse cancelled")
}
return nil, fmt.Errorf("tree-sitter: parse returned nil")
}
return &Tree{inner: tree}, nil
}
// Query is a compiled tree-sitter query. It caches CaptureNames so
// capture-id → name lookups are O(1) and don't cross into CGO.
type Query struct {
inner *ts.Query
names []string
}
// NewQuery compiles a query pattern against a language. Signature
// matches smacker's (pattern, lang) order, which is the argument order
// most of our language adapters expect.
func NewQuery(pattern []byte, lang *Language) (*Query, error) {
q, qerr := ts.NewQuery(lang, string(pattern))
if qerr != nil {
return nil, errors.New(qerr.Error())
}
return &Query{inner: q, names: q.CaptureNames()}, nil
}
// Inner exposes the underlying *ts.Query for internal query runners.
func (q *Query) Inner() *ts.Query { return q.inner }
// Close releases the query's C resources.
func (q *Query) Close() {
if q != nil && q.inner != nil {
q.inner.Close()
q.inner = nil
}
}
// CaptureNameForId returns the capture name for a capture index.
func (q *Query) CaptureNameForId(id uint32) string {
if int(id) >= len(q.names) {
return ""
}
return q.names[id]
}
+12
View File
@@ -0,0 +1,12 @@
// Package tsx re-exports the TSX variant of tree-sitter-typescript
// for .tsx files.
package tsx
import (
tree_sitter_typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_typescript.LanguageTSX())
}
@@ -0,0 +1,13 @@
// Package typescript re-exports the tree-sitter-typescript grammar.
// Smacker's path was `typescript/typescript`; the flat `typescript`
// package here is the shorter equivalent.
package typescript
import (
tree_sitter_typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_typescript.LanguageTypescript())
}
+11
View File
@@ -0,0 +1,11 @@
// Package yaml re-exports tree-sitter-grammars/tree-sitter-yaml.
package yaml
import (
tree_sitter_yaml "github.com/tree-sitter-grammars/tree-sitter-yaml/bindings/go"
"github.com/zzet/gortex/internal/parser/tsitter"
)
func GetLanguage() *tsitter.Language {
return tsitter.NewLanguage(tree_sitter_yaml.Language())
}