chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:33:42 +08:00
commit a06f331eb8
3186 changed files with 689843 additions and 0 deletions
+105
View File
@@ -0,0 +1,105 @@
package sql
import "testing"
func colNames(cols []CreateColumn) []string {
out := make([]string, len(cols))
for i, c := range cols {
out[i] = c.Name
}
return out
}
func eqStrs(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func TestExtractCreateTablesWithColumns(t *testing.T) {
ddl := `
CREATE TABLE public.users (
id bigint NOT NULL,
email text NOT NULL,
balance NUMERIC(10,2),
PRIMARY KEY (id)
);
CREATE TABLE IF NOT EXISTS "events" (
"id" BIGSERIAL PRIMARY KEY,
name VARCHAR(255),
CONSTRAINT uq_name UNIQUE (name),
FOREIGN KEY (id) REFERENCES users (id)
);
`
tables := ExtractCreateTablesWithColumns(ddl)
if len(tables) != 2 {
t.Fatalf("tables = %d, want 2: %+v", len(tables), tables)
}
byName := map[string]CreateTableDef{}
for _, tb := range tables {
byName[tb.Table] = tb
}
users, ok := byName["users"]
if !ok {
t.Fatal("users table not extracted")
}
if users.Schema != "public" {
t.Errorf("users schema = %q, want public", users.Schema)
}
if got := colNames(users.Columns); !eqStrs(got, []string{"id", "email", "balance"}) {
t.Errorf("users columns = %v, want [id email balance] (PRIMARY KEY skipped)", got)
}
// The inner comma of NUMERIC(10,2) must not split the column entry.
for _, c := range users.Columns {
if c.Name == "balance" && c.Type != "NUMERIC(10,2)" {
t.Errorf("balance type = %q, want NUMERIC(10,2)", c.Type)
}
}
events := byName["events"]
if got := colNames(events.Columns); !eqStrs(got, []string{"id", "name"}) {
t.Errorf("events columns = %v, want [id name] (CONSTRAINT/FOREIGN KEY skipped)", got)
}
}
func TestIsGeneratedSchema_AndDialect(t *testing.T) {
ddl := sampleLiveSchema().ToDDL()
if !IsGeneratedSchema([]byte(ddl)) {
t.Error("ToDDL output must be recognised as generated schema")
}
if d := GeneratedSchemaDialect([]byte(ddl)); d != "postgres" {
t.Errorf("dialect = %q, want postgres", d)
}
if IsGeneratedSchema([]byte("CREATE TABLE x (id int);")) {
t.Error("plain DDL must not be flagged as generated")
}
if d := GeneratedSchemaDialect([]byte("CREATE TABLE x (id int);")); d != "" {
t.Errorf("non-generated content dialect = %q, want empty", d)
}
}
func TestGeneratedSchema_RoundTripsColumns(t *testing.T) {
// The point of NEW-GFY-8: a live schema's columns survive the DDL
// round-trip into the column extractor.
ddl := sampleLiveSchema().ToDDL()
tables := ExtractCreateTablesWithColumns(ddl)
cols := map[string][]string{}
for _, tb := range tables {
cols[tb.Table] = colNames(tb.Columns)
}
if !eqStrs(cols["users"], []string{"id", "email"}) {
t.Errorf("users columns = %v, want [id email]", cols["users"])
}
if !eqStrs(cols["orders"], []string{"id", "user_id"}) {
t.Errorf("orders columns = %v, want [id user_id]", cols["orders"])
}
}
+252
View File
@@ -0,0 +1,252 @@
package sql
import (
"context"
"fmt"
"sort"
"strings"
"github.com/jackc/pgx/v5"
)
// generatedSchemaMarker is the signature ToDDL writes in the leading
// comment of every dump. IsGeneratedSchema keys on it so a
// `gortex db schema` output is recognised — and ingested into the graph —
// regardless of where the user saves the file.
const generatedSchemaMarker = "Generated by `gortex db schema`"
// IsGeneratedSchema reports whether src is the output of `gortex db schema`.
func IsGeneratedSchema(src []byte) bool {
head := src
if len(head) > 512 {
head = head[:512]
}
return strings.Contains(string(head), generatedSchemaMarker)
}
// GeneratedSchemaDialect extracts the dialect tag from a generated dump's
// header ("... from a live postgres database ..."), or "" when absent.
func GeneratedSchemaDialect(src []byte) string {
head := src
if len(head) > 512 {
head = head[:512]
}
s := string(head)
const pre = "from a live "
i := strings.Index(s, pre)
if i < 0 {
return ""
}
rest := s[i+len(pre):]
if j := strings.Index(rest, " database"); j >= 0 {
return strings.TrimSpace(rest[:j])
}
return ""
}
// LiveColumn is a column introspected from a live database's
// information_schema.
type LiveColumn struct {
Schema string
Table string
Name string
DataType string
Nullable bool
Ordinal int
IsPrimaryKey bool
}
// LiveForeignKey is a foreign-key relationship introspected from a live
// database.
type LiveForeignKey struct {
Schema string
Table string
Column string
RefSchema string
RefTable string
RefColumn string
}
// LiveSchema is the introspected shape of a database. It is dialect-tagged
// so the generated DDL — and the db::<dialect>:: nodes the SQL extractor
// derives from it — carry the right dialect.
type LiveSchema struct {
Dialect string
Columns []LiveColumn
ForeignKeys []LiveForeignKey
}
// IntrospectPostgres connects to a PostgreSQL DSN and reads its table /
// column / foreign-key shape from information_schema. schema filters to a
// single schema (default "public" when empty); pass "*" for every
// non-system schema.
func IntrospectPostgres(ctx context.Context, dsn, schema string) (*LiveSchema, error) {
if schema == "" {
schema = "public"
}
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
return nil, fmt.Errorf("connect postgres: %w", err)
}
defer func() { _ = conn.Close(ctx) }()
ls := &LiveSchema{Dialect: "postgres"}
schemaFilter := func(base string, col string) (string, []any) {
if schema == "*" {
return base + " AND " + col + " NOT IN ('pg_catalog','information_schema')", nil
}
return base + " AND " + col + " = $1", []any{schema}
}
// Columns.
colSQL, colArgs := schemaFilter(
`SELECT table_schema, table_name, column_name, data_type, is_nullable, ordinal_position
FROM information_schema.columns WHERE true`, "table_schema")
colSQL += " ORDER BY table_schema, table_name, ordinal_position"
rows, err := conn.Query(ctx, colSQL, colArgs...)
if err != nil {
return nil, fmt.Errorf("query columns: %w", err)
}
for rows.Next() {
var c LiveColumn
var nullable string
if err := rows.Scan(&c.Schema, &c.Table, &c.Name, &c.DataType, &nullable, &c.Ordinal); err != nil {
rows.Close()
return nil, err
}
c.Nullable = strings.EqualFold(nullable, "YES")
ls.Columns = append(ls.Columns, c)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
// Primary keys — mark the matching columns.
pkSQL, pkArgs := schemaFilter(
`SELECT tc.table_schema, tc.table_name, kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
WHERE tc.constraint_type = 'PRIMARY KEY'`, "tc.table_schema")
pkSet := map[string]bool{}
if rows, err := conn.Query(ctx, pkSQL, pkArgs...); err == nil {
for rows.Next() {
var s, t, c string
if err := rows.Scan(&s, &t, &c); err == nil {
pkSet[s+"."+t+"."+c] = true
}
}
rows.Close()
}
for i := range ls.Columns {
c := &ls.Columns[i]
if pkSet[c.Schema+"."+c.Table+"."+c.Name] {
c.IsPrimaryKey = true
}
}
// Foreign keys.
fkSQL, fkArgs := schemaFilter(
`SELECT tc.table_schema, tc.table_name, kcu.column_name,
ccu.table_schema, ccu.table_name, ccu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage ccu
ON tc.constraint_name = ccu.constraint_name AND tc.table_schema = ccu.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY'`, "tc.table_schema")
if rows, err := conn.Query(ctx, fkSQL, fkArgs...); err == nil {
for rows.Next() {
var fk LiveForeignKey
if err := rows.Scan(&fk.Schema, &fk.Table, &fk.Column, &fk.RefSchema, &fk.RefTable, &fk.RefColumn); err == nil {
ls.ForeignKeys = append(ls.ForeignKeys, fk)
}
}
rows.Close()
}
return ls, nil
}
// ToDDL renders the introspected schema as standard CREATE TABLE +
// foreign-key DDL. Feeding this through Gortex's SQL extractor produces
// the same db::<dialect>:: table / column nodes (and reference edges) as
// indexing hand-written migrations — so a live database becomes graph
// nodes with no new ingestion path. The output is deterministic
// (tables sorted; columns in ordinal order).
func (ls *LiveSchema) ToDDL() string {
type tableKey struct{ schema, table string }
order := []tableKey{}
byTable := map[tableKey][]LiveColumn{}
for _, c := range ls.Columns {
k := tableKey{c.Schema, c.Table}
if _, ok := byTable[k]; !ok {
order = append(order, k)
}
byTable[k] = append(byTable[k], c)
}
sort.Slice(order, func(i, j int) bool {
if order[i].schema != order[j].schema {
return order[i].schema < order[j].schema
}
return order[i].table < order[j].table
})
qualify := func(schema, table string) string {
if schema == "" || schema == "public" {
return table
}
return schema + "." + table
}
var b strings.Builder
fmt.Fprintf(&b, "-- %s from a live %s database. Do not edit.\n\n", generatedSchemaMarker, ls.Dialect)
for _, k := range order {
cols := byTable[k]
sort.SliceStable(cols, func(i, j int) bool { return cols[i].Ordinal < cols[j].Ordinal })
fmt.Fprintf(&b, "CREATE TABLE %s (\n", qualify(k.schema, k.table))
var pks []string
for i, c := range cols {
line := " " + c.Name + " " + sqlType(c.DataType)
if !c.Nullable {
line += " NOT NULL"
}
if i < len(cols)-1 || hasPK(cols) {
line += ","
}
b.WriteString(line + "\n")
if c.IsPrimaryKey {
pks = append(pks, c.Name)
}
}
if len(pks) > 0 {
fmt.Fprintf(&b, " PRIMARY KEY (%s)\n", strings.Join(pks, ", "))
}
b.WriteString(");\n\n")
}
for _, fk := range ls.ForeignKeys {
fmt.Fprintf(&b, "ALTER TABLE %s ADD FOREIGN KEY (%s) REFERENCES %s (%s);\n",
qualify(fk.Schema, fk.Table), fk.Column,
qualify(fk.RefSchema, fk.RefTable), fk.RefColumn)
}
return b.String()
}
func hasPK(cols []LiveColumn) bool {
for _, c := range cols {
if c.IsPrimaryKey {
return true
}
}
return false
}
// sqlType normalises an information_schema data_type to a portable column
// type token the SQL extractor recognises; unknown types pass through.
func sqlType(t string) string {
t = strings.TrimSpace(t)
if t == "" {
return "text"
}
return t
}
+75
View File
@@ -0,0 +1,75 @@
package sql
import (
"strings"
"testing"
)
func sampleLiveSchema() *LiveSchema {
return &LiveSchema{
Dialect: "postgres",
Columns: []LiveColumn{
{Schema: "public", Table: "users", Name: "id", DataType: "bigint", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
{Schema: "public", Table: "users", Name: "email", DataType: "text", Nullable: false, Ordinal: 2},
{Schema: "public", Table: "orders", Name: "id", DataType: "bigint", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
{Schema: "public", Table: "orders", Name: "user_id", DataType: "bigint", Nullable: true, Ordinal: 2},
},
ForeignKeys: []LiveForeignKey{
{Schema: "public", Table: "orders", Column: "user_id", RefSchema: "public", RefTable: "users", RefColumn: "id"},
},
}
}
func TestLiveSchema_ToDDL(t *testing.T) {
ddl := sampleLiveSchema().ToDDL()
for _, want := range []string{
"CREATE TABLE orders (",
"CREATE TABLE users (",
"id bigint NOT NULL",
"email text NOT NULL",
"PRIMARY KEY (id)",
"ALTER TABLE orders ADD FOREIGN KEY (user_id) REFERENCES users (id);",
} {
if !strings.Contains(ddl, want) {
t.Errorf("DDL missing %q\n---\n%s", want, ddl)
}
}
}
func TestLiveSchema_ToDDL_RoundTripsThroughExtractor(t *testing.T) {
// The whole point of emitting DDL is that the existing SQL extractor
// turns it into the same table nodes as a migration would.
ddl := sampleLiveSchema().ToDDL()
tables := ExtractCreateTables(ddl)
got := map[string]bool{}
for _, tr := range tables {
got[tr.Table] = true
if tr.Op != "create" {
t.Errorf("table %s op = %q, want create", tr.Table, tr.Op)
}
}
for _, want := range []string{"users", "orders"} {
if !got[want] {
t.Errorf("ExtractCreateTables did not recover table %q from generated DDL (got %v)", want, got)
}
}
// The generated table IDs match the canonical db::postgres:: scheme.
if id := TableNodeID("postgres", "", "users"); id != "db::postgres::users" {
t.Errorf("unexpected table node id %q", id)
}
}
func TestLiveSchema_ToDDL_NonPublicSchemaQualified(t *testing.T) {
ls := &LiveSchema{
Dialect: "postgres",
Columns: []LiveColumn{
{Schema: "billing", Table: "invoices", Name: "id", DataType: "uuid", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
},
}
ddl := ls.ToDDL()
if !strings.Contains(ddl, "CREATE TABLE billing.invoices (") {
t.Errorf("non-public schema should be qualified: %s", ddl)
}
}
+263
View File
@@ -0,0 +1,263 @@
package sql
import (
"sort"
"github.com/zzet/gortex/internal/graph"
)
// RebuildStats summarises a RebuildTablesFromStringRegistry pass. All
// counts are net (created — already-present). EmittersLinked counts the
// unique (caller, table) pairs that produced an EdgeQueries edge.
type RebuildStats struct {
StringsVisited int `json:"strings_visited"`
TablesCreated int `json:"tables_created"`
ColumnsCreated int `json:"columns_created"`
QueryEdges int `json:"query_edges_created"`
ReadColEdges int `json:"reads_col_edges_created"`
WriteColEdges int `json:"writes_col_edges_created"`
EmittersLinked int `json:"emitters_linked"`
Skipped int `json:"skipped"`
}
// RebuildTablesFromStringRegistry is the short-circuit: rederive
// the KindTable / KindColumn / EdgeQueries / EdgeReadsCol /
// EdgeWritesCol layer from the KindString context="sql" registry
// already present in g, without re-parsing any source file. Idempotent
// — nodes and edges are deduped via graph.AddNode / AddEdge semantics,
// so running it twice produces the same graph.
//
// For each KindString context="sql" node:
//
// - Re-extract tables and columns from node.Name (the verbatim
// query) using the canonical ExtractTables / ExtractColumns
// parsers — the same shape the source-time extractor uses.
// - For every emitter (EdgeEmits.From → KindString), wire
// EdgeQueries to each derived KindTable and EdgeReadsCol /
// EdgeWritesCol to each derived KindColumn.
//
// Dialect is taken from Meta["dialect"] when present (set by the Go
// SQL extractor) and falls back to "generic". Origin on rebuilt edges
// is text_matched (same tier the source-time extractor uses for
// regex-derived SQL).
//
// Returns counts for telemetry; rebuilt edges idempotently replace
// any existing edges with the same edgeKey, so a second call after
// the first reports tablesCreated=0, emittersLinked=0.
func RebuildTablesFromStringRegistry(g graph.Store) RebuildStats {
if g == nil {
return RebuildStats{}
}
var stats RebuildStats
// Snapshot the node set to avoid mutation-during-iteration; we
// append KindTable / KindColumn nodes below.
nodes := g.AllNodes()
// Track per-call counts of new node/edge insertions; we infer
// "created" by checking presence before AddNode / AddEdge.
preExistingTables := make(map[string]struct{})
preExistingCols := make(map[string]struct{})
for _, n := range nodes {
if n == nil {
continue
}
switch n.Kind {
case graph.KindTable:
preExistingTables[n.ID] = struct{}{}
case graph.KindColumn:
preExistingCols[n.ID] = struct{}{}
}
}
// EdgeQueries / EdgeReadsCol / EdgeWritesCol dedup tracker.
// Seeded from the live graph so a re-run on an already-rebuilt
// graph reports zero new edges (graph.AddEdge is idempotent by
// edgeKey, but the stats counters live in this function so they
// need to know what was already there).
type edgeKey struct {
from, to string
kind graph.EdgeKind
}
seenEdges := make(map[edgeKey]struct{})
for _, n := range nodes {
if n == nil {
continue
}
if n.Kind != graph.KindTable && n.Kind != graph.KindColumn {
continue
}
for _, e := range g.GetInEdges(n.ID) {
if e == nil {
continue
}
switch e.Kind {
case graph.EdgeQueries, graph.EdgeReadsCol, graph.EdgeWritesCol:
seenEdges[edgeKey{from: e.From, to: e.To, kind: e.Kind}] = struct{}{}
}
}
}
for _, n := range nodes {
if n == nil || n.Kind != graph.KindString {
continue
}
if ctx, _ := n.Meta["context"].(string); ctx != "sql" {
continue
}
query := n.Name
if query == "" {
if v, ok := n.Meta["value"].(string); ok {
query = v
}
}
if query == "" {
stats.Skipped++
continue
}
stats.StringsVisited++
dialect, _ := n.Meta["dialect"].(string)
if dialect == "" {
dialect = "generic"
}
tables := ExtractTables(query)
columns := ExtractColumns(query)
if len(tables) == 0 {
stats.Skipped++
continue
}
// Collect emitters from EdgeEmits to this KindString.
emitters := make([]string, 0, 4)
seenEmitter := make(map[string]struct{}, 4)
for _, e := range g.GetInEdges(n.ID) {
if e == nil || e.Kind != graph.EdgeEmits {
continue
}
if _, dup := seenEmitter[e.From]; dup {
continue
}
seenEmitter[e.From] = struct{}{}
emitters = append(emitters, e.From)
}
// Stable ordering — emitter snapshots are otherwise shard-
// order-dependent, which makes test assertions racy.
sort.Strings(emitters)
// Ensure KindTable nodes.
tableNodes := make([]struct {
id string
ref TableRef
added bool
}, 0, len(tables))
for _, ref := range tables {
id := TableNodeID(dialect, ref.Schema, ref.Table)
if _, exists := preExistingTables[id]; !exists {
meta := map[string]any{
"table": ref.Table,
"dialect": dialect,
}
if ref.Schema != "" {
meta["schema"] = ref.Schema
}
g.AddNode(&graph.Node{
ID: id,
Kind: graph.KindTable,
Name: ref.Table,
FilePath: n.FilePath,
Language: "sql",
Meta: meta,
})
preExistingTables[id] = struct{}{}
stats.TablesCreated++
tableNodes = append(tableNodes, struct {
id string
ref TableRef
added bool
}{id, ref, true})
} else {
tableNodes = append(tableNodes, struct {
id string
ref TableRef
added bool
}{id, ref, false})
}
}
// Ensure KindColumn nodes.
colNodes := make([]struct {
id string
ref ColumnRef
}, 0, len(columns))
for _, col := range columns {
id := ColumnNodeID(dialect, col.Schema, col.Table, col.Column)
if _, exists := preExistingCols[id]; !exists {
meta := map[string]any{
"table": col.Table,
"column": col.Column,
"dialect": dialect,
}
if col.Schema != "" {
meta["schema"] = col.Schema
}
g.AddNode(&graph.Node{
ID: id,
Kind: graph.KindColumn,
Name: col.Column,
FilePath: n.FilePath,
Language: "sql",
Meta: meta,
})
preExistingCols[id] = struct{}{}
stats.ColumnsCreated++
}
colNodes = append(colNodes, struct {
id string
ref ColumnRef
}{id, col})
}
// Wire emitters → tables / columns.
for _, emitter := range emitters {
for _, t := range tableNodes {
k := edgeKey{from: emitter, to: t.id, kind: graph.EdgeQueries}
if _, dup := seenEdges[k]; dup {
continue
}
seenEdges[k] = struct{}{}
g.AddEdge(&graph.Edge{
From: emitter,
To: t.id,
Kind: graph.EdgeQueries,
FilePath: n.FilePath,
Origin: graph.OriginTextMatched,
Meta: map[string]any{
"op": t.ref.Op,
"source": "string_registry",
},
})
stats.QueryEdges++
stats.EmittersLinked++
}
for _, c := range colNodes {
kind := graph.EdgeReadsCol
if c.ref.Op == "write" {
kind = graph.EdgeWritesCol
}
k := edgeKey{from: emitter, to: c.id, kind: kind}
if _, dup := seenEdges[k]; dup {
continue
}
seenEdges[k] = struct{}{}
g.AddEdge(&graph.Edge{
From: emitter,
To: c.id,
Kind: kind,
FilePath: n.FilePath,
Origin: graph.OriginTextMatched,
Meta: map[string]any{
"source": "string_registry",
},
})
if kind == graph.EdgeReadsCol {
stats.ReadColEdges++
} else {
stats.WriteColEdges++
}
}
}
}
return stats
}
+175
View File
@@ -0,0 +1,175 @@
package sql
import (
"testing"
"github.com/zzet/gortex/internal/graph"
)
func TestRebuildTablesFromStringRegistry_BuildsTablesAndEdges(t *testing.T) {
g := graph.New()
// Caller (any node kind will do — the rebuild reads emitters
// from EdgeEmits.From regardless).
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
// KindString context="sql" registry node.
strID := "string::sql::SELECT id FROM users"
g.AddNode(&graph.Node{
ID: strID,
Kind: graph.KindString,
Name: "SELECT id FROM users",
Language: "go",
Meta: map[string]any{
"context": "sql",
"value": "SELECT id FROM users",
"dialect": "postgres",
},
})
g.AddEdge(&graph.Edge{
From: "pkg/foo.go::Run",
To: strID,
Kind: graph.EdgeEmits,
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
})
stats := RebuildTablesFromStringRegistry(g)
if stats.StringsVisited != 1 {
t.Errorf("StringsVisited = %d, want 1", stats.StringsVisited)
}
if stats.TablesCreated != 1 {
t.Errorf("TablesCreated = %d, want 1", stats.TablesCreated)
}
if stats.QueryEdges != 1 {
t.Errorf("QueryEdges = %d, want 1", stats.QueryEdges)
}
if stats.EmittersLinked != 1 {
t.Errorf("EmittersLinked = %d, want 1", stats.EmittersLinked)
}
tableID := TableNodeID("postgres", "", "users")
if n := g.GetNode(tableID); n == nil {
t.Fatalf("expected KindTable %q to be created", tableID)
} else if n.Kind != graph.KindTable {
t.Errorf("node kind = %s, want %s", n.Kind, graph.KindTable)
}
// EdgeQueries from caller to table.
in := g.GetInEdges(tableID)
hasQueries := false
for _, e := range in {
if e.Kind == graph.EdgeQueries && e.From == "pkg/foo.go::Run" {
hasQueries = true
if e.Origin != graph.OriginTextMatched {
t.Errorf("Origin = %q, want text_matched", e.Origin)
}
}
}
if !hasQueries {
t.Errorf("missing EdgeQueries from caller to %s", tableID)
}
}
func TestRebuildTablesFromStringRegistry_Idempotent(t *testing.T) {
g := graph.New()
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
g.AddNode(&graph.Node{
ID: "string::sql::UPDATE users SET active = true",
Kind: graph.KindString,
Name: "UPDATE users SET active = true",
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
})
g.AddEdge(&graph.Edge{
From: "pkg/foo.go::Run",
To: "string::sql::UPDATE users SET active = true",
Kind: graph.EdgeEmits,
})
first := RebuildTablesFromStringRegistry(g)
if first.TablesCreated == 0 {
t.Fatalf("first pass created no tables: %+v", first)
}
second := RebuildTablesFromStringRegistry(g)
if second.TablesCreated != 0 {
t.Errorf("second pass created %d tables, expected idempotent", second.TablesCreated)
}
if second.QueryEdges != 0 {
t.Errorf("second pass added %d query edges, expected idempotent", second.QueryEdges)
}
}
func TestRebuildTablesFromStringRegistry_SkipsNonSQLStrings(t *testing.T) {
g := graph.New()
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
// error_msg KindString — should be ignored.
g.AddNode(&graph.Node{
ID: "string::error_msg::bad token",
Kind: graph.KindString,
Name: "bad token",
Meta: map[string]any{"context": "error_msg"},
})
g.AddEdge(&graph.Edge{
From: "pkg/foo.go::Run",
To: "string::error_msg::bad token",
Kind: graph.EdgeEmits,
})
stats := RebuildTablesFromStringRegistry(g)
if stats.StringsVisited != 0 {
t.Errorf("StringsVisited = %d, want 0 (no sql-context strings)", stats.StringsVisited)
}
if stats.TablesCreated != 0 {
t.Errorf("TablesCreated = %d, want 0", stats.TablesCreated)
}
}
func TestRebuildTablesFromStringRegistry_SkipsUnparseableQueries(t *testing.T) {
g := graph.New()
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
g.AddNode(&graph.Node{
ID: "string::sql::not actually sql",
Kind: graph.KindString,
Name: "not actually sql",
Meta: map[string]any{"context": "sql"},
})
stats := RebuildTablesFromStringRegistry(g)
if stats.StringsVisited != 1 {
t.Errorf("StringsVisited = %d, want 1", stats.StringsVisited)
}
if stats.TablesCreated != 0 {
t.Errorf("TablesCreated = %d, want 0", stats.TablesCreated)
}
if stats.Skipped != 1 {
t.Errorf("Skipped = %d, want 1", stats.Skipped)
}
}
func TestRebuildTablesFromStringRegistry_EmitsColumnEdges(t *testing.T) {
g := graph.New()
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
g.AddNode(&graph.Node{
ID: "string::sql::INSERT INTO accounts (id, balance) VALUES (1, 10)",
Kind: graph.KindString,
Name: "INSERT INTO accounts (id, balance) VALUES (1, 10)",
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
})
g.AddEdge(&graph.Edge{
From: "pkg/foo.go::Run",
To: "string::sql::INSERT INTO accounts (id, balance) VALUES (1, 10)",
Kind: graph.EdgeEmits,
})
stats := RebuildTablesFromStringRegistry(g)
if stats.ColumnsCreated != 2 {
t.Errorf("ColumnsCreated = %d, want 2 (id, balance)", stats.ColumnsCreated)
}
if stats.WriteColEdges != 2 {
t.Errorf("WriteColEdges = %d, want 2 (INSERT is a write)", stats.WriteColEdges)
}
}
func TestRebuildTablesFromStringRegistry_NilGraphSafe(t *testing.T) {
stats := RebuildTablesFromStringRegistry(nil)
if stats.StringsVisited != 0 {
t.Errorf("nil graph produced StringsVisited = %d, want 0", stats.StringsVisited)
}
}
+833
View File
@@ -0,0 +1,833 @@
// Package sql parses SQL string literals into the table references
// they touch. Used by language extractors that detect calls into a
// SQL exec API (db.Query, db.Exec, sqlx.NamedExec, etc.) with a
// string-literal first arg — the literal goes through ExtractTables
// to get the names; the caller emits KindTable nodes plus EdgeQueries
// edges.
//
// Scope (v1): regex-based table extraction from FROM / JOIN /
// INSERT INTO / UPDATE / DELETE FROM clauses. The regex picks up
// the canonical patterns without spinning up a full SQL parser.
// Trade-offs:
//
// - Dynamic SQL built by string concatenation or query builders
// is invisible. Agents who care about that will fall back to
// grep — same v1 stance the broader spec takes for noisy
// extractions.
//
// - Quoted identifiers (`"foo"`, `[foo]`) and case-sensitive
// schema-qualified names (`schema.table`) are handled — the
// regex strips quoting and keeps the trailing identifier, with
// schema preserved in the meta when present.
//
// - SQL keywords used as identifiers (`FROM "from"`) misclassify
// as the keyword. A future enhancement could feed the regex
// output through a SQL keyword list to filter them; v1 accepts
// the noise.
//
// - Default-off via the `sql` coverage gate per the spec — string-
// literal SQL is noisy enough that opt-in is the right shape.
package sql
import (
"regexp"
"sort"
"strings"
)
// tableRefPatterns enumerates the SQL clauses that introduce a
// table reference. Each pattern uses a single capture group on the
// table identifier. Case-insensitive match — SQL conventionally
// uppercases keywords but we tolerate either form.
var tableRefPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)\bFROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
regexp.MustCompile(`(?i)\bJOIN\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
regexp.MustCompile(`(?i)\bINSERT\s+INTO\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
regexp.MustCompile(`(?i)\bUPDATE\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
regexp.MustCompile(`(?i)\bDELETE\s+FROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
regexp.MustCompile(`(?i)\bTRUNCATE\s+(?:TABLE\s+)?([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
}
// TableRef is a single resolved table reference.
type TableRef struct {
Table string // unquoted table name (last segment if schema.table)
Schema string // optional schema prefix; "" when none
Op string // canonical operation: select, insert, update, delete, truncate
}
// canonicalOp maps a clause keyword to a stable operation tag for
// downstream queries that scope by op (e.g. "find every site that
// truncates X").
func canonicalOp(clauseHead string) string {
switch strings.ToUpper(strings.Fields(clauseHead)[0]) {
case "FROM", "JOIN":
return "select"
case "INSERT":
return "insert"
case "UPDATE":
return "update"
case "DELETE":
return "delete"
case "TRUNCATE":
return "truncate"
}
return ""
}
// ExtractTables walks query and returns the de-duplicated set of
// table references found. Order follows source-text occurrence so
// the result is diff-able across runs of the same query string.
func ExtractTables(query string) []TableRef {
if query == "" {
return nil
}
seen := make(map[string]struct{})
var refs []TableRef
// `DELETE FROM` matches both the DELETE FROM pattern (correct)
// and the bare FROM pattern (wrong — we'd report the same
// table as both a select and a delete). Process compound
// keywords first, mask out their match ranges so the FROM
// regex doesn't see them, then process the remaining ones.
working := maskDeleteFromForFromPattern(query)
for i, re := range tableRefPatterns {
// The FROM pattern (index 0) sees the masked text; the
// DELETE FROM pattern (index 4) sees the original to find
// its own matches first.
text := query
if i == 0 {
text = working
}
matches := re.FindAllStringSubmatch(text, -1)
for _, m := range matches {
if len(m) < 2 {
continue
}
schema, table := splitSchemaTable(stripQuoting(m[1]))
if table == "" {
continue
}
op := canonicalOp(m[0])
key := op + "::" + schema + "::" + table
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
refs = append(refs, TableRef{
Table: table,
Schema: schema,
Op: op,
})
}
}
sort.Slice(refs, func(i, j int) bool {
if refs[i].Op != refs[j].Op {
return refs[i].Op < refs[j].Op
}
if refs[i].Schema != refs[j].Schema {
return refs[i].Schema < refs[j].Schema
}
return refs[i].Table < refs[j].Table
})
return refs
}
// maskDeleteFromForFromPattern substitutes the FROM keyword in
// "DELETE FROM" with a non-keyword sentinel so the bare FROM
// regex doesn't double-match the same table reference. The
// sentinel `__GFOX_FROM__` won't appear in real SQL and is
// valid in the regex's character class so it gets ignored
// silently. The DELETE FROM pattern still operates on the
// original query string and finds its own match.
var deleteFromMaskRe = regexp.MustCompile(`(?i)\b(DELETE)\s+FROM\b`)
func maskDeleteFromForFromPattern(query string) string {
return deleteFromMaskRe.ReplaceAllString(query, "$1 __GFOX_FROM__")
}
// stripQuoting removes the four shapes of SQL identifier quoting:
// double quotes (ANSI), backticks (MySQL), brackets (T-SQL). The
// inner content is returned unchanged otherwise.
func stripQuoting(name string) string {
name = strings.TrimSpace(name)
if len(name) >= 2 {
first, last := name[0], name[len(name)-1]
switch {
case first == '"' && last == '"',
first == '`' && last == '`',
first == '[' && last == ']':
return name[1 : len(name)-1]
}
}
return name
}
// splitSchemaTable separates `schema.table` into its parts.
// Multi-dot identifiers (`db.schema.table`) collapse to schema=
// "schema", table="table" — the leftmost segment is database-
// scoped and rarely useful for graph queries.
func splitSchemaTable(name string) (schema, table string) {
if i := strings.LastIndex(name, "."); i >= 0 {
schema = name[:i]
table = name[i+1:]
// If the schema piece itself has a database segment, keep
// only the immediate parent.
if j := strings.LastIndex(schema, "."); j >= 0 {
schema = schema[j+1:]
}
return strings.TrimSpace(stripQuoting(schema)), strings.TrimSpace(stripQuoting(table))
}
return "", strings.TrimSpace(name)
}
// createTableRe matches CREATE TABLE [IF NOT EXISTS] declarations
// across the four canonical identifier-quoting styles. Used by
// ExtractCreateTables for migration-file extraction — distinct
// from ExtractTables, which scans query strings rather than DDL.
var createTableRe = regexp.MustCompile(`(?i)\bCREATE\s+(?:GLOBAL\s+TEMPORARY\s+|LOCAL\s+TEMPORARY\s+|TEMPORARY\s+|TEMP\s+|UNLOGGED\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`)
// ExtractCreateTables returns the tables declared by CREATE TABLE
// statements in a SQL source file. Schema-qualified names retain
// their schema in TableRef.Schema; identifier quoting is stripped.
// Op is always "create".
//
// Used by migration-file extraction where the SQL source is a DDL
// script rather than an embedded query string. Drop / alter
// statements are deliberately not extracted — a migration that
// drops a table doesn't *provide* the table to the rest of the
// repo, and modeling alter-as-delta would require maintaining
// per-migration ordering that's out of scope for the v1.
func ExtractCreateTables(source string) []TableRef {
if source == "" {
return nil
}
matches := createTableRe.FindAllStringSubmatch(source, -1)
seen := make(map[string]struct{})
var refs []TableRef
for _, m := range matches {
if len(m) < 2 {
continue
}
schema, table := splitSchemaTable(stripQuoting(m[1]))
if table == "" {
continue
}
key := schema + "::" + table
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
refs = append(refs, TableRef{
Table: table,
Schema: schema,
Op: "create",
})
}
sort.Slice(refs, func(i, j int) bool {
if refs[i].Schema != refs[j].Schema {
return refs[i].Schema < refs[j].Schema
}
return refs[i].Table < refs[j].Table
})
return refs
}
// CreateColumn is a column declared inside a CREATE TABLE body. Type is a
// best-effort leading type token ("" when not determinable).
type CreateColumn struct {
Name string
Type string
}
// CreateTableDef is a CREATE TABLE statement together with its declared
// columns — the schema-ingestion counterpart to TableRef, used where
// columns become first-class graph nodes (migration files, live-DB DDL).
type CreateTableDef struct {
Schema string
Table string
Columns []CreateColumn
}
// tableConstraintHeads are the leading keywords of a table-level
// constraint clause inside a CREATE TABLE body; an entry beginning with
// one of these is not a column.
var tableConstraintHeads = map[string]bool{
"PRIMARY": true, "FOREIGN": true, "CONSTRAINT": true, "UNIQUE": true,
"CHECK": true, "EXCLUDE": true, "LIKE": true, "PARTITION": true,
"INDEX": true, "KEY": true,
}
// ExtractCreateTablesWithColumns parses CREATE TABLE statements and their
// column lists out of SQL DDL. The parenthesised body is scanned with
// paren-depth + quote tracking (so VARCHAR(255) / NUMERIC(10,2) stay
// intact), split on top-level commas, and the leading identifier of each
// non-constraint entry is taken as a column. Best-effort and parser-free,
// matching the rest of this package. Tables are de-duplicated by
// schema::table and returned in a deterministic order.
func ExtractCreateTablesWithColumns(source string) []CreateTableDef {
if source == "" {
return nil
}
locs := createTableRe.FindAllStringSubmatchIndex(source, -1)
seen := make(map[string]struct{})
var out []CreateTableDef
for _, loc := range locs {
if len(loc) < 4 || loc[2] < 0 {
continue
}
schema, table := splitSchemaTable(stripQuoting(source[loc[2]:loc[3]]))
if table == "" {
continue
}
key := schema + "::" + table
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
def := CreateTableDef{Schema: schema, Table: table}
if body := balancedParenBody(source, loc[3]); body != "" {
def.Columns = parseColumnDefs(body)
}
out = append(out, def)
}
sort.Slice(out, func(i, j int) bool {
if out[i].Schema != out[j].Schema {
return out[i].Schema < out[j].Schema
}
return out[i].Table < out[j].Table
})
return out
}
// balancedParenBody returns the contents between the first '(' at or after
// `from` and its matching ')', or "" when there is no balanced group
// (e.g. CREATE TABLE ... AS SELECT, or CREATE TABLE x LIKE y).
func balancedParenBody(s string, from int) string {
open := strings.IndexByte(s[from:], '(')
if open < 0 {
return ""
}
start := from + open
depth := 0
inSingle, inDouble := false, false
for i := start; i < len(s); i++ {
c := s[i]
switch {
case inSingle:
if c == '\'' {
inSingle = false
}
case inDouble:
if c == '"' {
inDouble = false
}
case c == '\'':
inSingle = true
case c == '"':
inDouble = true
case c == '(':
depth++
case c == ')':
depth--
if depth == 0 {
return s[start+1 : i]
}
}
}
return ""
}
// parseColumnDefs splits a CREATE TABLE body on top-level commas and reads
// the leading identifier of each non-constraint entry as a column.
func parseColumnDefs(body string) []CreateColumn {
var cols []CreateColumn
seen := make(map[string]struct{})
for _, seg := range splitTopLevel(body, ',') {
seg = strings.TrimSpace(seg)
if seg == "" {
continue
}
fields := strings.Fields(seg)
if len(fields) == 0 {
continue
}
if tableConstraintHeads[strings.ToUpper(stripQuoting(fields[0]))] {
continue
}
name := stripQuoting(fields[0])
if name == "" {
continue
}
if _, ok := seen[name]; ok {
continue
}
seen[name] = struct{}{}
typ := ""
if len(fields) > 1 {
typ = fields[1]
}
cols = append(cols, CreateColumn{Name: name, Type: typ})
}
return cols
}
// splitTopLevel splits s on sep at paren depth 0, ignoring sep inside
// single/double quotes or parentheses.
func splitTopLevel(s string, sep byte) []string {
var parts []string
depth := 0
inSingle, inDouble := false, false
last := 0
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case inSingle:
if c == '\'' {
inSingle = false
}
case inDouble:
if c == '"' {
inDouble = false
}
case c == '\'':
inSingle = true
case c == '"':
inDouble = true
case c == '(':
depth++
case c == ')':
if depth > 0 {
depth--
}
case c == sep && depth == 0:
parts = append(parts, s[last:i])
last = i + 1
}
}
return append(parts, s[last:])
}
// ColumnRef is a single resolved column reference. Op is "read" for
// columns appearing in SELECT projections, WHERE clauses, ORDER BY,
// or GROUP BY; "write" for columns in INSERT INTO (col-list) and
// UPDATE … SET col = … assignments. Table identifies the table the
// column belongs to; multi-table queries (joins) are restricted to
// the first table reference because column-table association would
// otherwise require a real SQL parser.
type ColumnRef struct {
Schema string
Table string
Column string
Op string // "read" | "write"
}
// insertColsRe matches `INSERT INTO tbl (col1, col2, …)` with
// optional schema-qualified table.
var insertColsRe = regexp.MustCompile(`(?is)\bINSERT\s+INTO\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\s*\(([^)]*)\)`)
// updateSetRe matches `UPDATE tbl SET col = …, col2 = …` capturing
// the table name and the SET clause's content (greedy until WHERE
// or end of statement).
var updateSetRe = regexp.MustCompile(`(?is)\bUPDATE\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\s+SET\s+(.+?)(?:\bWHERE\b|\bRETURNING\b|;|$)`)
// selectFromRe matches `SELECT cols FROM tbl` for single-table
// queries (no JOIN). Multi-table SELECTs return no column edges
// because v1 can't disambiguate which table each column lives on.
var selectFromRe = regexp.MustCompile(`(?is)\bSELECT\s+(.+?)\s+FROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\b`)
// joinDetectRe is used to suppress SELECT column extraction when
// the query has any kind of JOIN — preserves correctness over
// completeness in v1.
var joinDetectRe = regexp.MustCompile(`(?i)\bJOIN\b`)
// ExtractColumns walks a query string and returns the column
// references it touches. The "Op" field distinguishes reads from
// writes so the caller can emit EdgeReadsCol vs EdgeWritesCol.
//
// Limitations (intentional for v1):
// - SELECT * does not produce edges (wildcard).
// - Multi-table SELECTs (with JOIN) produce no column edges.
// - Functions, expressions, and CASE statements degrade to no edge
// for that particular projection slot.
// - WHERE-clause column reads are extracted only when the value-side
// reference is a bare identifier.
func ExtractColumns(query string) []ColumnRef {
if query == "" {
return nil
}
out := []ColumnRef{}
seen := map[string]struct{}{}
add := func(c ColumnRef) {
key := c.Op + "::" + c.Schema + "::" + c.Table + "::" + c.Column
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
out = append(out, c)
}
// INSERT INTO tbl (col1, col2, …) → writes.
for _, m := range insertColsRe.FindAllStringSubmatch(query, -1) {
schema, table := splitSchemaTable(stripQuoting(m[1]))
if table == "" {
continue
}
for _, c := range splitColumnList(m[2]) {
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "write"})
}
}
// UPDATE tbl SET col = …, col2 = … → writes.
for _, m := range updateSetRe.FindAllStringSubmatch(query, -1) {
schema, table := splitSchemaTable(stripQuoting(m[1]))
if table == "" {
continue
}
for _, c := range splitSetAssignments(m[2]) {
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "write"})
}
}
// SELECT col1, col2 FROM tbl (single-table) → reads.
if !joinDetectRe.MatchString(query) {
for _, m := range selectFromRe.FindAllStringSubmatch(query, -1) {
projection := strings.TrimSpace(m[1])
schema, table := splitSchemaTable(stripQuoting(m[2]))
if table == "" {
continue
}
if projection == "*" || projection == "" {
continue
}
for _, c := range splitColumnList(projection) {
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "read"})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Op != out[j].Op {
return out[i].Op < out[j].Op
}
if out[i].Schema != out[j].Schema {
return out[i].Schema < out[j].Schema
}
if out[i].Table != out[j].Table {
return out[i].Table < out[j].Table
}
return out[i].Column < out[j].Column
})
return out
}
// splitColumnList parses a comma-separated column list (used for
// INSERT and SELECT projections), returning bare column identifiers.
// Aliases (`col AS alias`) collapse to the source column. Function
// calls and expressions return "" and are dropped.
func splitColumnList(list string) []string {
out := []string{}
depth := 0
cur := strings.Builder{}
flush := func() {
s := strings.TrimSpace(cur.String())
cur.Reset()
if s == "" {
return
}
// Strip trailing AS alias.
if idx := strings.LastIndex(strings.ToUpper(s), " AS "); idx >= 0 {
s = strings.TrimSpace(s[:idx])
}
// Strip table prefix `tbl.col` → `col`.
if idx := strings.LastIndex(s, "."); idx >= 0 {
s = s[idx+1:]
}
s = strings.TrimSpace(stripQuoting(s))
// Reject bare expressions (function calls, arithmetic, *).
if s == "" || s == "*" || !isPlainSQLIdent(s) {
return
}
out = append(out, s)
}
for i := 0; i < len(list); i++ {
c := list[i]
switch c {
case '(':
depth++
case ')':
if depth > 0 {
depth--
}
case ',':
if depth == 0 {
flush()
continue
}
}
cur.WriteByte(c)
}
flush()
return out
}
// splitSetAssignments parses a SET clause body (`col = ?, col2 =
// fn(x)`) and returns the column names being written. The right-
// hand expressions are skipped; column refs deeper than `tbl.col` are
// reduced to `col`.
func splitSetAssignments(set string) []string {
out := []string{}
depth := 0
cur := strings.Builder{}
flush := func() {
seg := strings.TrimSpace(cur.String())
cur.Reset()
if seg == "" {
return
}
eq := strings.Index(seg, "=")
if eq < 0 {
return
}
name := strings.TrimSpace(seg[:eq])
if idx := strings.LastIndex(name, "."); idx >= 0 {
name = name[idx+1:]
}
name = strings.TrimSpace(stripQuoting(name))
if name != "" && isPlainSQLIdent(name) {
out = append(out, name)
}
}
for i := 0; i < len(set); i++ {
c := set[i]
switch c {
case '(':
depth++
case ')':
if depth > 0 {
depth--
}
case ',':
if depth == 0 {
flush()
continue
}
}
cur.WriteByte(c)
}
flush()
return out
}
// isPlainSQLIdent returns true when s is a single bare identifier
// (letter/underscore start, alphanum/underscore body). Filters out
// function-call shells (`fn(`), wildcards, arithmetic, and the like.
func isPlainSQLIdent(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
c := s[i]
isAlpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
isDigit := c >= '0' && c <= '9'
if i == 0 {
if !isAlpha {
return false
}
} else if !isAlpha && !isDigit {
return false
}
}
return true
}
// selectNoFromRe matches a `SELECT <projection>` that has no FROM
// clause (e.g. `SELECT 1 AS id`). Used only as the fallback when a
// query contains no FROM at all — selectFromRe takes precedence.
var selectNoFromRe = regexp.MustCompile(`(?is)\bSELECT\s+(.+?)\s*(?:;|$)`)
// ProjectionColumns returns the output column names a query produces —
// the columns of the relation the query *materialises*, as opposed to
// ExtractColumns which attributes columns to the source tables a query
// reads or writes. Used by the dbt / SQLMesh model extractor to record
// a model's own columns.
//
// The output column of a projection slot is its alias when one is
// present (`total AS order_total` → `order_total`) and the bare column
// identifier otherwise (`customers.id` → `id`). Slots that are pure
// expressions / function calls with no alias, and `*` / `tbl.*`
// wildcards, produce no name.
//
// Heuristic (mirrors the v1 regex stance of the rest of this package):
// the last `SELECT <projection> FROM` occurrence is taken as the
// query's output projection — CTEs and subqueries appear earlier in
// source order, so the final top-level SELECT is what the query
// returns. When the query has no FROM clause at all the trailing
// `SELECT <projection>` is used. Subquery-valued projection slots and
// JOIN-bearing final SELECTs degrade gracefully to whatever bare
// identifiers can still be recovered.
func ProjectionColumns(query string) []string {
if query == "" {
return nil
}
var projection string
if all := selectFromRe.FindAllStringSubmatch(query, -1); len(all) > 0 {
projection = strings.TrimSpace(all[len(all)-1][1])
} else if m := selectNoFromRe.FindStringSubmatch(query); m != nil {
projection = strings.TrimSpace(m[1])
}
if projection == "" || projection == "*" {
return nil
}
cols := splitProjectionList(projection)
seen := make(map[string]struct{}, len(cols))
out := cols[:0]
for _, c := range cols {
if _, dup := seen[c]; dup {
continue
}
seen[c] = struct{}{}
out = append(out, c)
}
return out
}
// splitProjectionList parses a SELECT projection list and returns the
// output column names — the alias when one is present, otherwise the
// bare column identifier. Distinct from splitColumnList, which collapses
// `col AS alias` to the source `col` because it attributes reads to a
// source table; here the alias *is* the produced column name.
func splitProjectionList(list string) []string {
out := []string{}
depth := 0
cur := strings.Builder{}
flush := func() {
seg := strings.TrimSpace(cur.String())
cur.Reset()
if seg == "" {
return
}
if idx := lastTopLevelAS(seg); idx >= 0 {
alias := strings.TrimSpace(stripQuoting(strings.TrimSpace(seg[idx+4:])))
if isPlainSQLIdent(alias) {
out = append(out, alias)
}
return
}
// No alias: strip a `tbl.` qualifier, require a bare identifier.
s := seg
if i := strings.LastIndex(s, "."); i >= 0 {
s = s[i+1:]
}
s = strings.TrimSpace(stripQuoting(s))
if s != "" && s != "*" && isPlainSQLIdent(s) {
out = append(out, s)
}
}
for i := 0; i < len(list); i++ {
c := list[i]
switch c {
case '(':
depth++
case ')':
if depth > 0 {
depth--
}
case ',':
if depth == 0 {
flush()
continue
}
}
cur.WriteByte(c)
}
flush()
return out
}
// lastTopLevelAS returns the byte index of the last ` AS ` keyword in
// seg that sits at paren depth zero, or -1 when there is none. Case-
// insensitive. Keeps a cast's inner ` AS ` (`CAST(x AS int)`) from
// being mistaken for a projection alias.
func lastTopLevelAS(seg string) int {
upper := strings.ToUpper(seg)
depth := 0
last := -1
for i := 0; i+4 <= len(upper); i++ {
switch upper[i] {
case '(':
depth++
case ')':
if depth > 0 {
depth--
}
}
if depth == 0 && upper[i:i+4] == " AS " {
last = i
}
}
return last
}
// ColumnNodeID returns the canonical synthetic ID for a column.
func ColumnNodeID(dialect, schema, table, column string) string {
if dialect == "" {
dialect = "generic"
}
prefix := "col::" + dialect + "::"
if schema == "" {
return prefix + table + "." + column
}
return prefix + schema + "." + table + "." + column
}
// MigrationNodeID is the canonical synthetic ID for a migration
// node. The path component lets agents reach the originating file
// in one step; the prefix matches the synthetic-ID convention
// used by db:: tables and module:: dependencies.
func MigrationNodeID(path string) string {
return "migration::" + path
}
// IsMigrationPath returns true when filePath looks like a SQL
// migration file. Recognised conventions: any .sql file under a
// directory whose name contains "migrate" or "migration"
// (case-insensitive). Matches Rails (db/migrate/), golang-migrate
// (migrations/), Alembic when wrapped (we mostly handle alembic
// via Python parsers separately), and most ORM generators.
func IsMigrationPath(filePath string) bool {
if !strings.HasSuffix(strings.ToLower(filePath), ".sql") {
return false
}
lower := strings.ToLower(filePath)
for _, segment := range []string{"/migrate/", "/migrations/", "/migrate.", "/migrations."} {
if strings.Contains(lower, segment) {
return true
}
}
return strings.HasPrefix(lower, "migrate/") ||
strings.HasPrefix(lower, "migrations/") ||
strings.HasPrefix(lower, "db/migrate/") ||
strings.HasPrefix(lower, "db/migrations/")
}
// TableNodeID returns the canonical synthetic ID for a table
// reference. Mirrors the ecosystem-prefix convention used by
// module:: / external:: / annotation:: / event:: nodes — `db::`
// keeps the table namespace distinct.
//
// dialect is the SQL dialect tag (postgres, mysql, sqlite,
// generic) — included on the ID so cross-dialect projects can
// distinguish a Postgres `users` table from a MySQL one in the
// same graph. The default dialect is "generic" when the caller
// doesn't know.
func TableNodeID(dialect, schema, table string) string {
if dialect == "" {
dialect = "generic"
}
prefix := "db::" + dialect + "::"
if schema == "" {
return prefix + table
}
return prefix + schema + "." + table
}
+385
View File
@@ -0,0 +1,385 @@
package sql
import (
"testing"
)
func TestExtractTables_BasicSelect(t *testing.T) {
refs := ExtractTables(`SELECT * FROM users WHERE id = 1`)
if len(refs) != 1 {
t.Fatalf("expected 1 ref, got %d: %+v", len(refs), refs)
}
if refs[0].Table != "users" || refs[0].Op != "select" {
t.Errorf("got %+v", refs[0])
}
}
func TestExtractTables_JoinClauses(t *testing.T) {
refs := ExtractTables(`SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id`)
if len(refs) != 2 {
t.Fatalf("expected 2 refs, got %d: %+v", len(refs), refs)
}
got := map[string]bool{}
for _, r := range refs {
got[r.Table] = true
}
if !got["users"] || !got["orders"] {
t.Errorf("missing refs: %v", got)
}
}
func TestExtractTables_InsertUpdateDelete(t *testing.T) {
cases := []struct {
query string
op string
table string
}{
{`INSERT INTO users (id, name) VALUES (1, 'a')`, "insert", "users"},
{`UPDATE accounts SET balance = 0 WHERE id = 1`, "update", "accounts"},
{`DELETE FROM sessions WHERE expired = true`, "delete", "sessions"},
{`TRUNCATE TABLE logs`, "truncate", "logs"},
{`TRUNCATE logs2`, "truncate", "logs2"},
}
for _, c := range cases {
refs := ExtractTables(c.query)
if len(refs) != 1 {
t.Fatalf("%q → %d refs, want 1", c.query, len(refs))
}
if refs[0].Op != c.op || refs[0].Table != c.table {
t.Errorf("%q → %+v, want op=%q table=%q", c.query, refs[0], c.op, c.table)
}
}
}
func TestExtractTables_QuotedIdentifiers(t *testing.T) {
cases := []string{
`SELECT * FROM "users" WHERE id = 1`, // ANSI
"SELECT * FROM `users` WHERE id = 1", // MySQL backticks
`SELECT * FROM [users] WHERE id = 1`, // T-SQL brackets
}
for _, q := range cases {
refs := ExtractTables(q)
if len(refs) != 1 || refs[0].Table != "users" {
t.Errorf("%q → %+v, want table=users", q, refs)
}
}
}
func TestExtractTables_SchemaQualified(t *testing.T) {
refs := ExtractTables(`SELECT * FROM public.users JOIN auth.sessions ON id = session_id`)
if len(refs) != 2 {
t.Fatalf("expected 2 refs, got %d", len(refs))
}
got := map[string]string{} // table → schema
for _, r := range refs {
got[r.Table] = r.Schema
}
if got["users"] != "public" {
t.Errorf("users schema = %q, want public", got["users"])
}
if got["sessions"] != "auth" {
t.Errorf("sessions schema = %q, want auth", got["sessions"])
}
}
func TestExtractTables_DeduplicatesSameOpAndTable(t *testing.T) {
refs := ExtractTables(`SELECT * FROM users JOIN users u2 ON 1=1`)
// Both users references share op=select, schema="" — should dedupe.
if len(refs) != 1 {
t.Errorf("expected dedup to single users ref, got %d", len(refs))
}
}
func TestExtractTables_MixedOps(t *testing.T) {
q := `WITH x AS (SELECT * FROM source)
INSERT INTO target SELECT * FROM x`
refs := ExtractTables(q)
// `source` (select) + `target` (insert) + `x` (select) = 3
if len(refs) != 3 {
t.Errorf("expected 3 refs, got %d: %+v", len(refs), refs)
}
}
func TestExtractTables_Empty(t *testing.T) {
if r := ExtractTables(""); len(r) != 0 {
t.Errorf("empty query should yield no refs")
}
if r := ExtractTables("SELECT 1"); len(r) != 0 {
t.Errorf("no-table query should yield no refs")
}
}
func TestStripQuoting(t *testing.T) {
cases := []struct{ in, want string }{
{`"users"`, "users"},
{"`users`", "users"},
{"[users]", "users"},
{"users", "users"},
{`"`, `"`},
}
for _, c := range cases {
if got := stripQuoting(c.in); got != c.want {
t.Errorf("stripQuoting(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestSplitSchemaTable(t *testing.T) {
cases := []struct {
in string
schema, table string
}{
{"users", "", "users"},
{"public.users", "public", "users"},
{"db.public.users", "public", "users"}, // database segment dropped
{`"public"."users"`, "public", "users"},
}
for _, c := range cases {
s, t2 := splitSchemaTable(c.in)
if s != c.schema || t2 != c.table {
t.Errorf("splitSchemaTable(%q) = (%q,%q), want (%q,%q)", c.in, s, t2, c.schema, c.table)
}
}
}
func TestExtractCreateTables_Basic(t *testing.T) {
src := `CREATE TABLE users (
id BIGINT PRIMARY KEY,
email TEXT UNIQUE
);
CREATE TABLE IF NOT EXISTS sessions (
user_id BIGINT REFERENCES users(id),
token TEXT
);
`
refs := ExtractCreateTables(src)
if len(refs) != 2 {
t.Fatalf("expected 2 refs, got %d: %+v", len(refs), refs)
}
got := map[string]bool{}
for _, r := range refs {
got[r.Table] = true
if r.Op != "create" {
t.Errorf("op = %q, want create", r.Op)
}
}
if !got["users"] || !got["sessions"] {
t.Errorf("missing tables: %v", got)
}
}
func TestExtractCreateTables_Variants(t *testing.T) {
src := `
CREATE TEMPORARY TABLE t1 (id INT);
CREATE TEMP TABLE t2 (id INT);
CREATE GLOBAL TEMPORARY TABLE t3 (id INT);
CREATE UNLOGGED TABLE t4 (id INT);
CREATE TABLE IF NOT EXISTS t5 (id INT);
CREATE TABLE "quoted_name" (id INT);
CREATE TABLE auth.tokens (id INT);
`
refs := ExtractCreateTables(src)
if len(refs) != 7 {
t.Fatalf("expected 7 refs, got %d: %+v", len(refs), refs)
}
gotSchemas := map[string]string{}
for _, r := range refs {
gotSchemas[r.Table] = r.Schema
}
if gotSchemas["tokens"] != "auth" {
t.Errorf("auth.tokens schema = %q", gotSchemas["tokens"])
}
if gotSchemas["quoted_name"] != "" {
t.Errorf("quoted-name should have no schema, got %q", gotSchemas["quoted_name"])
}
}
func TestExtractCreateTables_DedupesSameSchemaTable(t *testing.T) {
src := `
CREATE TABLE users (id INT);
CREATE TABLE users (id INT);
`
refs := ExtractCreateTables(src)
if len(refs) != 1 {
t.Errorf("expected 1 deduped ref, got %d", len(refs))
}
}
func TestExtractCreateTables_IgnoresAlterAndDrop(t *testing.T) {
src := `
ALTER TABLE users ADD COLUMN name TEXT;
DROP TABLE sessions;
`
if got := ExtractCreateTables(src); len(got) != 0 {
t.Errorf("alter/drop should not produce CREATE refs, got %+v", got)
}
}
func TestIsMigrationPath(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"db/migrate/001_create_users.sql", true},
{"db/migrations/2024_01_init.sql", true},
{"migrations/v1.sql", true},
{"internal/db/migrations/init.sql", true},
{"pkg/queries/select.sql", false},
{"main.go", false},
{"docs/migration_guide.md", false}, // not .sql
}
for _, c := range cases {
if got := IsMigrationPath(c.path); got != c.want {
t.Errorf("IsMigrationPath(%q) = %v, want %v", c.path, got, c.want)
}
}
}
func TestMigrationNodeID(t *testing.T) {
if got := MigrationNodeID("db/migrate/001_init.sql"); got != "migration::db/migrate/001_init.sql" {
t.Errorf("got %q", got)
}
}
func TestExtractColumns_InsertWrites(t *testing.T) {
got := ExtractColumns("INSERT INTO users (id, email, name) VALUES ($1, $2, $3)")
wantCols := map[string]bool{"id": false, "email": false, "name": false}
for _, c := range got {
if c.Op != "write" {
t.Errorf("expected write op, got %q for %q", c.Op, c.Column)
}
if c.Table != "users" {
t.Errorf("expected table users, got %q", c.Table)
}
if _, ok := wantCols[c.Column]; ok {
wantCols[c.Column] = true
}
}
for col, found := range wantCols {
if !found {
t.Errorf("expected column %q in result, got %v", col, got)
}
}
}
func TestExtractColumns_UpdateWrites(t *testing.T) {
got := ExtractColumns("UPDATE users SET email = $1, updated_at = NOW() WHERE id = $2")
wantCols := map[string]bool{"email": false, "updated_at": false}
for _, c := range got {
if c.Op != "write" {
continue
}
if _, ok := wantCols[c.Column]; ok {
wantCols[c.Column] = true
}
}
for col, found := range wantCols {
if !found {
t.Errorf("expected SET column %q to surface as write; got %v", col, got)
}
}
}
func TestExtractColumns_SingleTableSelectReads(t *testing.T) {
got := ExtractColumns("SELECT id, email, name FROM users WHERE id = $1")
want := map[string]bool{"id": false, "email": false, "name": false}
for _, c := range got {
if c.Op != "read" {
continue
}
if c.Table != "users" {
continue
}
if _, ok := want[c.Column]; ok {
want[c.Column] = true
}
}
for col, found := range want {
if !found {
t.Errorf("expected read column %q; got %v", col, got)
}
}
}
func TestExtractColumns_StarSelectNoColumns(t *testing.T) {
got := ExtractColumns("SELECT * FROM users")
for _, c := range got {
if c.Op == "read" {
t.Errorf("SELECT * should not produce read columns; got %v", got)
}
}
}
func TestExtractColumns_JoinSelectReturnsNoColumns(t *testing.T) {
got := ExtractColumns("SELECT u.id, o.id FROM users u JOIN orders o ON o.user_id = u.id")
for _, c := range got {
if c.Op == "read" {
t.Errorf("multi-table SELECT should not emit columns in v1; got %v", got)
}
}
}
func TestColumnNodeID(t *testing.T) {
cases := []struct {
dialect, schema, table, column, want string
}{
{"postgres", "public", "users", "email", "col::postgres::public.users.email"},
{"", "", "users", "email", "col::generic::users.email"},
}
for _, c := range cases {
if got := ColumnNodeID(c.dialect, c.schema, c.table, c.column); got != c.want {
t.Errorf("ColumnNodeID(%q,%q,%q,%q) = %q, want %q",
c.dialect, c.schema, c.table, c.column, got, c.want)
}
}
}
func TestTableNodeID(t *testing.T) {
cases := []struct {
dialect, schema, table, want string
}{
{"postgres", "public", "users", "db::postgres::public.users"},
{"", "", "users", "db::generic::users"},
{"mysql", "", "orders", "db::mysql::orders"},
}
for _, c := range cases {
if got := TableNodeID(c.dialect, c.schema, c.table); got != c.want {
t.Errorf("TableNodeID(%q,%q,%q) = %q, want %q",
c.dialect, c.schema, c.table, got, c.want)
}
}
}
func TestProjectionColumns(t *testing.T) {
cases := []struct {
name string
query string
want []string
}{
{"plain columns", "SELECT id, name FROM users", []string{"id", "name"}},
{"table-qualified", "SELECT u.id, u.name FROM users u", []string{"id", "name"}},
{"alias wins", "SELECT total AS order_total, id FROM orders", []string{"order_total", "id"}},
{"expression alias", "SELECT count(*) AS n, max(x) AS m FROM t", []string{"n", "m"}},
{"cast inner AS ignored", "SELECT cast(zip AS text) AS zip_code FROM t", []string{"zip_code"}},
{"unaliased expr dropped", "SELECT count(*), id FROM t", []string{"id"}},
{"star yields nothing", "SELECT * FROM t", nil},
{"final select of CTE", "WITH a AS (SELECT p FROM raw) SELECT x, y FROM a", []string{"x", "y"}},
{"no from clause", "SELECT 1 AS id, 2 AS qty", []string{"id", "qty"}},
{"empty", "", nil},
{"dedup", "SELECT id, id FROM t", []string{"id"}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := ProjectionColumns(c.query)
if len(got) != len(c.want) {
t.Fatalf("ProjectionColumns(%q) = %v, want %v", c.query, got, c.want)
}
for i := range got {
if got[i] != c.want[i] {
t.Fatalf("ProjectionColumns(%q) = %v, want %v", c.query, got, c.want)
}
}
})
}
}