chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
package sql
|
||||
|
||||
import "testing"
|
||||
|
||||
func colNames(cols []CreateColumn) []string {
|
||||
out := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
out[i] = c.Name
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func eqStrs(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestExtractCreateTablesWithColumns(t *testing.T) {
|
||||
ddl := `
|
||||
CREATE TABLE public.users (
|
||||
id bigint NOT NULL,
|
||||
email text NOT NULL,
|
||||
balance NUMERIC(10,2),
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "events" (
|
||||
"id" BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255),
|
||||
CONSTRAINT uq_name UNIQUE (name),
|
||||
FOREIGN KEY (id) REFERENCES users (id)
|
||||
);
|
||||
`
|
||||
tables := ExtractCreateTablesWithColumns(ddl)
|
||||
if len(tables) != 2 {
|
||||
t.Fatalf("tables = %d, want 2: %+v", len(tables), tables)
|
||||
}
|
||||
|
||||
byName := map[string]CreateTableDef{}
|
||||
for _, tb := range tables {
|
||||
byName[tb.Table] = tb
|
||||
}
|
||||
|
||||
users, ok := byName["users"]
|
||||
if !ok {
|
||||
t.Fatal("users table not extracted")
|
||||
}
|
||||
if users.Schema != "public" {
|
||||
t.Errorf("users schema = %q, want public", users.Schema)
|
||||
}
|
||||
if got := colNames(users.Columns); !eqStrs(got, []string{"id", "email", "balance"}) {
|
||||
t.Errorf("users columns = %v, want [id email balance] (PRIMARY KEY skipped)", got)
|
||||
}
|
||||
// The inner comma of NUMERIC(10,2) must not split the column entry.
|
||||
for _, c := range users.Columns {
|
||||
if c.Name == "balance" && c.Type != "NUMERIC(10,2)" {
|
||||
t.Errorf("balance type = %q, want NUMERIC(10,2)", c.Type)
|
||||
}
|
||||
}
|
||||
|
||||
events := byName["events"]
|
||||
if got := colNames(events.Columns); !eqStrs(got, []string{"id", "name"}) {
|
||||
t.Errorf("events columns = %v, want [id name] (CONSTRAINT/FOREIGN KEY skipped)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGeneratedSchema_AndDialect(t *testing.T) {
|
||||
ddl := sampleLiveSchema().ToDDL()
|
||||
if !IsGeneratedSchema([]byte(ddl)) {
|
||||
t.Error("ToDDL output must be recognised as generated schema")
|
||||
}
|
||||
if d := GeneratedSchemaDialect([]byte(ddl)); d != "postgres" {
|
||||
t.Errorf("dialect = %q, want postgres", d)
|
||||
}
|
||||
if IsGeneratedSchema([]byte("CREATE TABLE x (id int);")) {
|
||||
t.Error("plain DDL must not be flagged as generated")
|
||||
}
|
||||
if d := GeneratedSchemaDialect([]byte("CREATE TABLE x (id int);")); d != "" {
|
||||
t.Errorf("non-generated content dialect = %q, want empty", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedSchema_RoundTripsColumns(t *testing.T) {
|
||||
// The point of NEW-GFY-8: a live schema's columns survive the DDL
|
||||
// round-trip into the column extractor.
|
||||
ddl := sampleLiveSchema().ToDDL()
|
||||
tables := ExtractCreateTablesWithColumns(ddl)
|
||||
cols := map[string][]string{}
|
||||
for _, tb := range tables {
|
||||
cols[tb.Table] = colNames(tb.Columns)
|
||||
}
|
||||
if !eqStrs(cols["users"], []string{"id", "email"}) {
|
||||
t.Errorf("users columns = %v, want [id email]", cols["users"])
|
||||
}
|
||||
if !eqStrs(cols["orders"], []string{"id", "user_id"}) {
|
||||
t.Errorf("orders columns = %v, want [id user_id]", cols["orders"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// generatedSchemaMarker is the signature ToDDL writes in the leading
|
||||
// comment of every dump. IsGeneratedSchema keys on it so a
|
||||
// `gortex db schema` output is recognised — and ingested into the graph —
|
||||
// regardless of where the user saves the file.
|
||||
const generatedSchemaMarker = "Generated by `gortex db schema`"
|
||||
|
||||
// IsGeneratedSchema reports whether src is the output of `gortex db schema`.
|
||||
func IsGeneratedSchema(src []byte) bool {
|
||||
head := src
|
||||
if len(head) > 512 {
|
||||
head = head[:512]
|
||||
}
|
||||
return strings.Contains(string(head), generatedSchemaMarker)
|
||||
}
|
||||
|
||||
// GeneratedSchemaDialect extracts the dialect tag from a generated dump's
|
||||
// header ("... from a live postgres database ..."), or "" when absent.
|
||||
func GeneratedSchemaDialect(src []byte) string {
|
||||
head := src
|
||||
if len(head) > 512 {
|
||||
head = head[:512]
|
||||
}
|
||||
s := string(head)
|
||||
const pre = "from a live "
|
||||
i := strings.Index(s, pre)
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := s[i+len(pre):]
|
||||
if j := strings.Index(rest, " database"); j >= 0 {
|
||||
return strings.TrimSpace(rest[:j])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// LiveColumn is a column introspected from a live database's
|
||||
// information_schema.
|
||||
type LiveColumn struct {
|
||||
Schema string
|
||||
Table string
|
||||
Name string
|
||||
DataType string
|
||||
Nullable bool
|
||||
Ordinal int
|
||||
IsPrimaryKey bool
|
||||
}
|
||||
|
||||
// LiveForeignKey is a foreign-key relationship introspected from a live
|
||||
// database.
|
||||
type LiveForeignKey struct {
|
||||
Schema string
|
||||
Table string
|
||||
Column string
|
||||
RefSchema string
|
||||
RefTable string
|
||||
RefColumn string
|
||||
}
|
||||
|
||||
// LiveSchema is the introspected shape of a database. It is dialect-tagged
|
||||
// so the generated DDL — and the db::<dialect>:: nodes the SQL extractor
|
||||
// derives from it — carry the right dialect.
|
||||
type LiveSchema struct {
|
||||
Dialect string
|
||||
Columns []LiveColumn
|
||||
ForeignKeys []LiveForeignKey
|
||||
}
|
||||
|
||||
// IntrospectPostgres connects to a PostgreSQL DSN and reads its table /
|
||||
// column / foreign-key shape from information_schema. schema filters to a
|
||||
// single schema (default "public" when empty); pass "*" for every
|
||||
// non-system schema.
|
||||
func IntrospectPostgres(ctx context.Context, dsn, schema string) (*LiveSchema, error) {
|
||||
if schema == "" {
|
||||
schema = "public"
|
||||
}
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect postgres: %w", err)
|
||||
}
|
||||
defer func() { _ = conn.Close(ctx) }()
|
||||
|
||||
ls := &LiveSchema{Dialect: "postgres"}
|
||||
schemaFilter := func(base string, col string) (string, []any) {
|
||||
if schema == "*" {
|
||||
return base + " AND " + col + " NOT IN ('pg_catalog','information_schema')", nil
|
||||
}
|
||||
return base + " AND " + col + " = $1", []any{schema}
|
||||
}
|
||||
|
||||
// Columns.
|
||||
colSQL, colArgs := schemaFilter(
|
||||
`SELECT table_schema, table_name, column_name, data_type, is_nullable, ordinal_position
|
||||
FROM information_schema.columns WHERE true`, "table_schema")
|
||||
colSQL += " ORDER BY table_schema, table_name, ordinal_position"
|
||||
rows, err := conn.Query(ctx, colSQL, colArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query columns: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var c LiveColumn
|
||||
var nullable string
|
||||
if err := rows.Scan(&c.Schema, &c.Table, &c.Name, &c.DataType, &nullable, &c.Ordinal); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
c.Nullable = strings.EqualFold(nullable, "YES")
|
||||
ls.Columns = append(ls.Columns, c)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Primary keys — mark the matching columns.
|
||||
pkSQL, pkArgs := schemaFilter(
|
||||
`SELECT tc.table_schema, tc.table_name, kcu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY'`, "tc.table_schema")
|
||||
pkSet := map[string]bool{}
|
||||
if rows, err := conn.Query(ctx, pkSQL, pkArgs...); err == nil {
|
||||
for rows.Next() {
|
||||
var s, t, c string
|
||||
if err := rows.Scan(&s, &t, &c); err == nil {
|
||||
pkSet[s+"."+t+"."+c] = true
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
for i := range ls.Columns {
|
||||
c := &ls.Columns[i]
|
||||
if pkSet[c.Schema+"."+c.Table+"."+c.Name] {
|
||||
c.IsPrimaryKey = true
|
||||
}
|
||||
}
|
||||
|
||||
// Foreign keys.
|
||||
fkSQL, fkArgs := schemaFilter(
|
||||
`SELECT tc.table_schema, tc.table_name, kcu.column_name,
|
||||
ccu.table_schema, ccu.table_name, ccu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
|
||||
JOIN information_schema.constraint_column_usage ccu
|
||||
ON tc.constraint_name = ccu.constraint_name AND tc.table_schema = ccu.table_schema
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'`, "tc.table_schema")
|
||||
if rows, err := conn.Query(ctx, fkSQL, fkArgs...); err == nil {
|
||||
for rows.Next() {
|
||||
var fk LiveForeignKey
|
||||
if err := rows.Scan(&fk.Schema, &fk.Table, &fk.Column, &fk.RefSchema, &fk.RefTable, &fk.RefColumn); err == nil {
|
||||
ls.ForeignKeys = append(ls.ForeignKeys, fk)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
return ls, nil
|
||||
}
|
||||
|
||||
// ToDDL renders the introspected schema as standard CREATE TABLE +
|
||||
// foreign-key DDL. Feeding this through Gortex's SQL extractor produces
|
||||
// the same db::<dialect>:: table / column nodes (and reference edges) as
|
||||
// indexing hand-written migrations — so a live database becomes graph
|
||||
// nodes with no new ingestion path. The output is deterministic
|
||||
// (tables sorted; columns in ordinal order).
|
||||
func (ls *LiveSchema) ToDDL() string {
|
||||
type tableKey struct{ schema, table string }
|
||||
order := []tableKey{}
|
||||
byTable := map[tableKey][]LiveColumn{}
|
||||
for _, c := range ls.Columns {
|
||||
k := tableKey{c.Schema, c.Table}
|
||||
if _, ok := byTable[k]; !ok {
|
||||
order = append(order, k)
|
||||
}
|
||||
byTable[k] = append(byTable[k], c)
|
||||
}
|
||||
sort.Slice(order, func(i, j int) bool {
|
||||
if order[i].schema != order[j].schema {
|
||||
return order[i].schema < order[j].schema
|
||||
}
|
||||
return order[i].table < order[j].table
|
||||
})
|
||||
|
||||
qualify := func(schema, table string) string {
|
||||
if schema == "" || schema == "public" {
|
||||
return table
|
||||
}
|
||||
return schema + "." + table
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "-- %s from a live %s database. Do not edit.\n\n", generatedSchemaMarker, ls.Dialect)
|
||||
for _, k := range order {
|
||||
cols := byTable[k]
|
||||
sort.SliceStable(cols, func(i, j int) bool { return cols[i].Ordinal < cols[j].Ordinal })
|
||||
fmt.Fprintf(&b, "CREATE TABLE %s (\n", qualify(k.schema, k.table))
|
||||
var pks []string
|
||||
for i, c := range cols {
|
||||
line := " " + c.Name + " " + sqlType(c.DataType)
|
||||
if !c.Nullable {
|
||||
line += " NOT NULL"
|
||||
}
|
||||
if i < len(cols)-1 || hasPK(cols) {
|
||||
line += ","
|
||||
}
|
||||
b.WriteString(line + "\n")
|
||||
if c.IsPrimaryKey {
|
||||
pks = append(pks, c.Name)
|
||||
}
|
||||
}
|
||||
if len(pks) > 0 {
|
||||
fmt.Fprintf(&b, " PRIMARY KEY (%s)\n", strings.Join(pks, ", "))
|
||||
}
|
||||
b.WriteString(");\n\n")
|
||||
}
|
||||
for _, fk := range ls.ForeignKeys {
|
||||
fmt.Fprintf(&b, "ALTER TABLE %s ADD FOREIGN KEY (%s) REFERENCES %s (%s);\n",
|
||||
qualify(fk.Schema, fk.Table), fk.Column,
|
||||
qualify(fk.RefSchema, fk.RefTable), fk.RefColumn)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hasPK(cols []LiveColumn) bool {
|
||||
for _, c := range cols {
|
||||
if c.IsPrimaryKey {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sqlType normalises an information_schema data_type to a portable column
|
||||
// type token the SQL extractor recognises; unknown types pass through.
|
||||
func sqlType(t string) string {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" {
|
||||
return "text"
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func sampleLiveSchema() *LiveSchema {
|
||||
return &LiveSchema{
|
||||
Dialect: "postgres",
|
||||
Columns: []LiveColumn{
|
||||
{Schema: "public", Table: "users", Name: "id", DataType: "bigint", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
|
||||
{Schema: "public", Table: "users", Name: "email", DataType: "text", Nullable: false, Ordinal: 2},
|
||||
{Schema: "public", Table: "orders", Name: "id", DataType: "bigint", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
|
||||
{Schema: "public", Table: "orders", Name: "user_id", DataType: "bigint", Nullable: true, Ordinal: 2},
|
||||
},
|
||||
ForeignKeys: []LiveForeignKey{
|
||||
{Schema: "public", Table: "orders", Column: "user_id", RefSchema: "public", RefTable: "users", RefColumn: "id"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveSchema_ToDDL(t *testing.T) {
|
||||
ddl := sampleLiveSchema().ToDDL()
|
||||
|
||||
for _, want := range []string{
|
||||
"CREATE TABLE orders (",
|
||||
"CREATE TABLE users (",
|
||||
"id bigint NOT NULL",
|
||||
"email text NOT NULL",
|
||||
"PRIMARY KEY (id)",
|
||||
"ALTER TABLE orders ADD FOREIGN KEY (user_id) REFERENCES users (id);",
|
||||
} {
|
||||
if !strings.Contains(ddl, want) {
|
||||
t.Errorf("DDL missing %q\n---\n%s", want, ddl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveSchema_ToDDL_RoundTripsThroughExtractor(t *testing.T) {
|
||||
// The whole point of emitting DDL is that the existing SQL extractor
|
||||
// turns it into the same table nodes as a migration would.
|
||||
ddl := sampleLiveSchema().ToDDL()
|
||||
tables := ExtractCreateTables(ddl)
|
||||
got := map[string]bool{}
|
||||
for _, tr := range tables {
|
||||
got[tr.Table] = true
|
||||
if tr.Op != "create" {
|
||||
t.Errorf("table %s op = %q, want create", tr.Table, tr.Op)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"users", "orders"} {
|
||||
if !got[want] {
|
||||
t.Errorf("ExtractCreateTables did not recover table %q from generated DDL (got %v)", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The generated table IDs match the canonical db::postgres:: scheme.
|
||||
if id := TableNodeID("postgres", "", "users"); id != "db::postgres::users" {
|
||||
t.Errorf("unexpected table node id %q", id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveSchema_ToDDL_NonPublicSchemaQualified(t *testing.T) {
|
||||
ls := &LiveSchema{
|
||||
Dialect: "postgres",
|
||||
Columns: []LiveColumn{
|
||||
{Schema: "billing", Table: "invoices", Name: "id", DataType: "uuid", Nullable: false, Ordinal: 1, IsPrimaryKey: true},
|
||||
},
|
||||
}
|
||||
ddl := ls.ToDDL()
|
||||
if !strings.Contains(ddl, "CREATE TABLE billing.invoices (") {
|
||||
t.Errorf("non-public schema should be qualified: %s", ddl)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// RebuildStats summarises a RebuildTablesFromStringRegistry pass. All
|
||||
// counts are net (created — already-present). EmittersLinked counts the
|
||||
// unique (caller, table) pairs that produced an EdgeQueries edge.
|
||||
type RebuildStats struct {
|
||||
StringsVisited int `json:"strings_visited"`
|
||||
TablesCreated int `json:"tables_created"`
|
||||
ColumnsCreated int `json:"columns_created"`
|
||||
QueryEdges int `json:"query_edges_created"`
|
||||
ReadColEdges int `json:"reads_col_edges_created"`
|
||||
WriteColEdges int `json:"writes_col_edges_created"`
|
||||
EmittersLinked int `json:"emitters_linked"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
// RebuildTablesFromStringRegistry is the short-circuit: rederive
|
||||
// the KindTable / KindColumn / EdgeQueries / EdgeReadsCol /
|
||||
// EdgeWritesCol layer from the KindString context="sql" registry
|
||||
// already present in g, without re-parsing any source file. Idempotent
|
||||
// — nodes and edges are deduped via graph.AddNode / AddEdge semantics,
|
||||
// so running it twice produces the same graph.
|
||||
//
|
||||
// For each KindString context="sql" node:
|
||||
//
|
||||
// - Re-extract tables and columns from node.Name (the verbatim
|
||||
// query) using the canonical ExtractTables / ExtractColumns
|
||||
// parsers — the same shape the source-time extractor uses.
|
||||
// - For every emitter (EdgeEmits.From → KindString), wire
|
||||
// EdgeQueries to each derived KindTable and EdgeReadsCol /
|
||||
// EdgeWritesCol to each derived KindColumn.
|
||||
//
|
||||
// Dialect is taken from Meta["dialect"] when present (set by the Go
|
||||
// SQL extractor) and falls back to "generic". Origin on rebuilt edges
|
||||
// is text_matched (same tier the source-time extractor uses for
|
||||
// regex-derived SQL).
|
||||
//
|
||||
// Returns counts for telemetry; rebuilt edges idempotently replace
|
||||
// any existing edges with the same edgeKey, so a second call after
|
||||
// the first reports tablesCreated=0, emittersLinked=0.
|
||||
func RebuildTablesFromStringRegistry(g graph.Store) RebuildStats {
|
||||
if g == nil {
|
||||
return RebuildStats{}
|
||||
}
|
||||
var stats RebuildStats
|
||||
// Snapshot the node set to avoid mutation-during-iteration; we
|
||||
// append KindTable / KindColumn nodes below.
|
||||
nodes := g.AllNodes()
|
||||
// Track per-call counts of new node/edge insertions; we infer
|
||||
// "created" by checking presence before AddNode / AddEdge.
|
||||
preExistingTables := make(map[string]struct{})
|
||||
preExistingCols := make(map[string]struct{})
|
||||
for _, n := range nodes {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
switch n.Kind {
|
||||
case graph.KindTable:
|
||||
preExistingTables[n.ID] = struct{}{}
|
||||
case graph.KindColumn:
|
||||
preExistingCols[n.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
// EdgeQueries / EdgeReadsCol / EdgeWritesCol dedup tracker.
|
||||
// Seeded from the live graph so a re-run on an already-rebuilt
|
||||
// graph reports zero new edges (graph.AddEdge is idempotent by
|
||||
// edgeKey, but the stats counters live in this function so they
|
||||
// need to know what was already there).
|
||||
type edgeKey struct {
|
||||
from, to string
|
||||
kind graph.EdgeKind
|
||||
}
|
||||
seenEdges := make(map[edgeKey]struct{})
|
||||
for _, n := range nodes {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
if n.Kind != graph.KindTable && n.Kind != graph.KindColumn {
|
||||
continue
|
||||
}
|
||||
for _, e := range g.GetInEdges(n.ID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
switch e.Kind {
|
||||
case graph.EdgeQueries, graph.EdgeReadsCol, graph.EdgeWritesCol:
|
||||
seenEdges[edgeKey{from: e.From, to: e.To, kind: e.Kind}] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n == nil || n.Kind != graph.KindString {
|
||||
continue
|
||||
}
|
||||
if ctx, _ := n.Meta["context"].(string); ctx != "sql" {
|
||||
continue
|
||||
}
|
||||
query := n.Name
|
||||
if query == "" {
|
||||
if v, ok := n.Meta["value"].(string); ok {
|
||||
query = v
|
||||
}
|
||||
}
|
||||
if query == "" {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
stats.StringsVisited++
|
||||
dialect, _ := n.Meta["dialect"].(string)
|
||||
if dialect == "" {
|
||||
dialect = "generic"
|
||||
}
|
||||
tables := ExtractTables(query)
|
||||
columns := ExtractColumns(query)
|
||||
if len(tables) == 0 {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
// Collect emitters from EdgeEmits to this KindString.
|
||||
emitters := make([]string, 0, 4)
|
||||
seenEmitter := make(map[string]struct{}, 4)
|
||||
for _, e := range g.GetInEdges(n.ID) {
|
||||
if e == nil || e.Kind != graph.EdgeEmits {
|
||||
continue
|
||||
}
|
||||
if _, dup := seenEmitter[e.From]; dup {
|
||||
continue
|
||||
}
|
||||
seenEmitter[e.From] = struct{}{}
|
||||
emitters = append(emitters, e.From)
|
||||
}
|
||||
// Stable ordering — emitter snapshots are otherwise shard-
|
||||
// order-dependent, which makes test assertions racy.
|
||||
sort.Strings(emitters)
|
||||
// Ensure KindTable nodes.
|
||||
tableNodes := make([]struct {
|
||||
id string
|
||||
ref TableRef
|
||||
added bool
|
||||
}, 0, len(tables))
|
||||
for _, ref := range tables {
|
||||
id := TableNodeID(dialect, ref.Schema, ref.Table)
|
||||
if _, exists := preExistingTables[id]; !exists {
|
||||
meta := map[string]any{
|
||||
"table": ref.Table,
|
||||
"dialect": dialect,
|
||||
}
|
||||
if ref.Schema != "" {
|
||||
meta["schema"] = ref.Schema
|
||||
}
|
||||
g.AddNode(&graph.Node{
|
||||
ID: id,
|
||||
Kind: graph.KindTable,
|
||||
Name: ref.Table,
|
||||
FilePath: n.FilePath,
|
||||
Language: "sql",
|
||||
Meta: meta,
|
||||
})
|
||||
preExistingTables[id] = struct{}{}
|
||||
stats.TablesCreated++
|
||||
tableNodes = append(tableNodes, struct {
|
||||
id string
|
||||
ref TableRef
|
||||
added bool
|
||||
}{id, ref, true})
|
||||
} else {
|
||||
tableNodes = append(tableNodes, struct {
|
||||
id string
|
||||
ref TableRef
|
||||
added bool
|
||||
}{id, ref, false})
|
||||
}
|
||||
}
|
||||
// Ensure KindColumn nodes.
|
||||
colNodes := make([]struct {
|
||||
id string
|
||||
ref ColumnRef
|
||||
}, 0, len(columns))
|
||||
for _, col := range columns {
|
||||
id := ColumnNodeID(dialect, col.Schema, col.Table, col.Column)
|
||||
if _, exists := preExistingCols[id]; !exists {
|
||||
meta := map[string]any{
|
||||
"table": col.Table,
|
||||
"column": col.Column,
|
||||
"dialect": dialect,
|
||||
}
|
||||
if col.Schema != "" {
|
||||
meta["schema"] = col.Schema
|
||||
}
|
||||
g.AddNode(&graph.Node{
|
||||
ID: id,
|
||||
Kind: graph.KindColumn,
|
||||
Name: col.Column,
|
||||
FilePath: n.FilePath,
|
||||
Language: "sql",
|
||||
Meta: meta,
|
||||
})
|
||||
preExistingCols[id] = struct{}{}
|
||||
stats.ColumnsCreated++
|
||||
}
|
||||
colNodes = append(colNodes, struct {
|
||||
id string
|
||||
ref ColumnRef
|
||||
}{id, col})
|
||||
}
|
||||
// Wire emitters → tables / columns.
|
||||
for _, emitter := range emitters {
|
||||
for _, t := range tableNodes {
|
||||
k := edgeKey{from: emitter, to: t.id, kind: graph.EdgeQueries}
|
||||
if _, dup := seenEdges[k]; dup {
|
||||
continue
|
||||
}
|
||||
seenEdges[k] = struct{}{}
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: emitter,
|
||||
To: t.id,
|
||||
Kind: graph.EdgeQueries,
|
||||
FilePath: n.FilePath,
|
||||
Origin: graph.OriginTextMatched,
|
||||
Meta: map[string]any{
|
||||
"op": t.ref.Op,
|
||||
"source": "string_registry",
|
||||
},
|
||||
})
|
||||
stats.QueryEdges++
|
||||
stats.EmittersLinked++
|
||||
}
|
||||
for _, c := range colNodes {
|
||||
kind := graph.EdgeReadsCol
|
||||
if c.ref.Op == "write" {
|
||||
kind = graph.EdgeWritesCol
|
||||
}
|
||||
k := edgeKey{from: emitter, to: c.id, kind: kind}
|
||||
if _, dup := seenEdges[k]; dup {
|
||||
continue
|
||||
}
|
||||
seenEdges[k] = struct{}{}
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: emitter,
|
||||
To: c.id,
|
||||
Kind: kind,
|
||||
FilePath: n.FilePath,
|
||||
Origin: graph.OriginTextMatched,
|
||||
Meta: map[string]any{
|
||||
"source": "string_registry",
|
||||
},
|
||||
})
|
||||
if kind == graph.EdgeReadsCol {
|
||||
stats.ReadColEdges++
|
||||
} else {
|
||||
stats.WriteColEdges++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return stats
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_BuildsTablesAndEdges(t *testing.T) {
|
||||
g := graph.New()
|
||||
// Caller (any node kind will do — the rebuild reads emitters
|
||||
// from EdgeEmits.From regardless).
|
||||
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
|
||||
// KindString context="sql" registry node.
|
||||
strID := "string::sql::SELECT id FROM users"
|
||||
g.AddNode(&graph.Node{
|
||||
ID: strID,
|
||||
Kind: graph.KindString,
|
||||
Name: "SELECT id FROM users",
|
||||
Language: "go",
|
||||
Meta: map[string]any{
|
||||
"context": "sql",
|
||||
"value": "SELECT id FROM users",
|
||||
"dialect": "postgres",
|
||||
},
|
||||
})
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: "pkg/foo.go::Run",
|
||||
To: strID,
|
||||
Kind: graph.EdgeEmits,
|
||||
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
|
||||
})
|
||||
|
||||
stats := RebuildTablesFromStringRegistry(g)
|
||||
|
||||
if stats.StringsVisited != 1 {
|
||||
t.Errorf("StringsVisited = %d, want 1", stats.StringsVisited)
|
||||
}
|
||||
if stats.TablesCreated != 1 {
|
||||
t.Errorf("TablesCreated = %d, want 1", stats.TablesCreated)
|
||||
}
|
||||
if stats.QueryEdges != 1 {
|
||||
t.Errorf("QueryEdges = %d, want 1", stats.QueryEdges)
|
||||
}
|
||||
if stats.EmittersLinked != 1 {
|
||||
t.Errorf("EmittersLinked = %d, want 1", stats.EmittersLinked)
|
||||
}
|
||||
|
||||
tableID := TableNodeID("postgres", "", "users")
|
||||
if n := g.GetNode(tableID); n == nil {
|
||||
t.Fatalf("expected KindTable %q to be created", tableID)
|
||||
} else if n.Kind != graph.KindTable {
|
||||
t.Errorf("node kind = %s, want %s", n.Kind, graph.KindTable)
|
||||
}
|
||||
// EdgeQueries from caller to table.
|
||||
in := g.GetInEdges(tableID)
|
||||
hasQueries := false
|
||||
for _, e := range in {
|
||||
if e.Kind == graph.EdgeQueries && e.From == "pkg/foo.go::Run" {
|
||||
hasQueries = true
|
||||
if e.Origin != graph.OriginTextMatched {
|
||||
t.Errorf("Origin = %q, want text_matched", e.Origin)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !hasQueries {
|
||||
t.Errorf("missing EdgeQueries from caller to %s", tableID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_Idempotent(t *testing.T) {
|
||||
g := graph.New()
|
||||
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
|
||||
g.AddNode(&graph.Node{
|
||||
ID: "string::sql::UPDATE users SET active = true",
|
||||
Kind: graph.KindString,
|
||||
Name: "UPDATE users SET active = true",
|
||||
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
|
||||
})
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: "pkg/foo.go::Run",
|
||||
To: "string::sql::UPDATE users SET active = true",
|
||||
Kind: graph.EdgeEmits,
|
||||
})
|
||||
|
||||
first := RebuildTablesFromStringRegistry(g)
|
||||
if first.TablesCreated == 0 {
|
||||
t.Fatalf("first pass created no tables: %+v", first)
|
||||
}
|
||||
second := RebuildTablesFromStringRegistry(g)
|
||||
if second.TablesCreated != 0 {
|
||||
t.Errorf("second pass created %d tables, expected idempotent", second.TablesCreated)
|
||||
}
|
||||
if second.QueryEdges != 0 {
|
||||
t.Errorf("second pass added %d query edges, expected idempotent", second.QueryEdges)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_SkipsNonSQLStrings(t *testing.T) {
|
||||
g := graph.New()
|
||||
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
|
||||
// error_msg KindString — should be ignored.
|
||||
g.AddNode(&graph.Node{
|
||||
ID: "string::error_msg::bad token",
|
||||
Kind: graph.KindString,
|
||||
Name: "bad token",
|
||||
Meta: map[string]any{"context": "error_msg"},
|
||||
})
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: "pkg/foo.go::Run",
|
||||
To: "string::error_msg::bad token",
|
||||
Kind: graph.EdgeEmits,
|
||||
})
|
||||
|
||||
stats := RebuildTablesFromStringRegistry(g)
|
||||
if stats.StringsVisited != 0 {
|
||||
t.Errorf("StringsVisited = %d, want 0 (no sql-context strings)", stats.StringsVisited)
|
||||
}
|
||||
if stats.TablesCreated != 0 {
|
||||
t.Errorf("TablesCreated = %d, want 0", stats.TablesCreated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_SkipsUnparseableQueries(t *testing.T) {
|
||||
g := graph.New()
|
||||
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
|
||||
g.AddNode(&graph.Node{
|
||||
ID: "string::sql::not actually sql",
|
||||
Kind: graph.KindString,
|
||||
Name: "not actually sql",
|
||||
Meta: map[string]any{"context": "sql"},
|
||||
})
|
||||
|
||||
stats := RebuildTablesFromStringRegistry(g)
|
||||
if stats.StringsVisited != 1 {
|
||||
t.Errorf("StringsVisited = %d, want 1", stats.StringsVisited)
|
||||
}
|
||||
if stats.TablesCreated != 0 {
|
||||
t.Errorf("TablesCreated = %d, want 0", stats.TablesCreated)
|
||||
}
|
||||
if stats.Skipped != 1 {
|
||||
t.Errorf("Skipped = %d, want 1", stats.Skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_EmitsColumnEdges(t *testing.T) {
|
||||
g := graph.New()
|
||||
g.AddNode(&graph.Node{ID: "pkg/foo.go::Run", Kind: graph.KindFunction, Name: "Run"})
|
||||
g.AddNode(&graph.Node{
|
||||
ID: "string::sql::INSERT INTO accounts (id, balance) VALUES (1, 10)",
|
||||
Kind: graph.KindString,
|
||||
Name: "INSERT INTO accounts (id, balance) VALUES (1, 10)",
|
||||
Meta: map[string]any{"context": "sql", "dialect": "postgres"},
|
||||
})
|
||||
g.AddEdge(&graph.Edge{
|
||||
From: "pkg/foo.go::Run",
|
||||
To: "string::sql::INSERT INTO accounts (id, balance) VALUES (1, 10)",
|
||||
Kind: graph.EdgeEmits,
|
||||
})
|
||||
|
||||
stats := RebuildTablesFromStringRegistry(g)
|
||||
if stats.ColumnsCreated != 2 {
|
||||
t.Errorf("ColumnsCreated = %d, want 2 (id, balance)", stats.ColumnsCreated)
|
||||
}
|
||||
if stats.WriteColEdges != 2 {
|
||||
t.Errorf("WriteColEdges = %d, want 2 (INSERT is a write)", stats.WriteColEdges)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRebuildTablesFromStringRegistry_NilGraphSafe(t *testing.T) {
|
||||
stats := RebuildTablesFromStringRegistry(nil)
|
||||
if stats.StringsVisited != 0 {
|
||||
t.Errorf("nil graph produced StringsVisited = %d, want 0", stats.StringsVisited)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,833 @@
|
||||
// Package sql parses SQL string literals into the table references
|
||||
// they touch. Used by language extractors that detect calls into a
|
||||
// SQL exec API (db.Query, db.Exec, sqlx.NamedExec, etc.) with a
|
||||
// string-literal first arg — the literal goes through ExtractTables
|
||||
// to get the names; the caller emits KindTable nodes plus EdgeQueries
|
||||
// edges.
|
||||
//
|
||||
// Scope (v1): regex-based table extraction from FROM / JOIN /
|
||||
// INSERT INTO / UPDATE / DELETE FROM clauses. The regex picks up
|
||||
// the canonical patterns without spinning up a full SQL parser.
|
||||
// Trade-offs:
|
||||
//
|
||||
// - Dynamic SQL built by string concatenation or query builders
|
||||
// is invisible. Agents who care about that will fall back to
|
||||
// grep — same v1 stance the broader spec takes for noisy
|
||||
// extractions.
|
||||
//
|
||||
// - Quoted identifiers (`"foo"`, `[foo]`) and case-sensitive
|
||||
// schema-qualified names (`schema.table`) are handled — the
|
||||
// regex strips quoting and keeps the trailing identifier, with
|
||||
// schema preserved in the meta when present.
|
||||
//
|
||||
// - SQL keywords used as identifiers (`FROM "from"`) misclassify
|
||||
// as the keyword. A future enhancement could feed the regex
|
||||
// output through a SQL keyword list to filter them; v1 accepts
|
||||
// the noise.
|
||||
//
|
||||
// - Default-off via the `sql` coverage gate per the spec — string-
|
||||
// literal SQL is noisy enough that opt-in is the right shape.
|
||||
package sql
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// tableRefPatterns enumerates the SQL clauses that introduce a
|
||||
// table reference. Each pattern uses a single capture group on the
|
||||
// table identifier. Case-insensitive match — SQL conventionally
|
||||
// uppercases keywords but we tolerate either form.
|
||||
var tableRefPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\bFROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
regexp.MustCompile(`(?i)\bJOIN\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
regexp.MustCompile(`(?i)\bINSERT\s+INTO\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
regexp.MustCompile(`(?i)\bUPDATE\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
regexp.MustCompile(`(?i)\bDELETE\s+FROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
regexp.MustCompile(`(?i)\bTRUNCATE\s+(?:TABLE\s+)?([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`),
|
||||
}
|
||||
|
||||
// TableRef is a single resolved table reference.
|
||||
type TableRef struct {
|
||||
Table string // unquoted table name (last segment if schema.table)
|
||||
Schema string // optional schema prefix; "" when none
|
||||
Op string // canonical operation: select, insert, update, delete, truncate
|
||||
}
|
||||
|
||||
// canonicalOp maps a clause keyword to a stable operation tag for
|
||||
// downstream queries that scope by op (e.g. "find every site that
|
||||
// truncates X").
|
||||
func canonicalOp(clauseHead string) string {
|
||||
switch strings.ToUpper(strings.Fields(clauseHead)[0]) {
|
||||
case "FROM", "JOIN":
|
||||
return "select"
|
||||
case "INSERT":
|
||||
return "insert"
|
||||
case "UPDATE":
|
||||
return "update"
|
||||
case "DELETE":
|
||||
return "delete"
|
||||
case "TRUNCATE":
|
||||
return "truncate"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ExtractTables walks query and returns the de-duplicated set of
|
||||
// table references found. Order follows source-text occurrence so
|
||||
// the result is diff-able across runs of the same query string.
|
||||
func ExtractTables(query string) []TableRef {
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{})
|
||||
var refs []TableRef
|
||||
|
||||
// `DELETE FROM` matches both the DELETE FROM pattern (correct)
|
||||
// and the bare FROM pattern (wrong — we'd report the same
|
||||
// table as both a select and a delete). Process compound
|
||||
// keywords first, mask out their match ranges so the FROM
|
||||
// regex doesn't see them, then process the remaining ones.
|
||||
working := maskDeleteFromForFromPattern(query)
|
||||
|
||||
for i, re := range tableRefPatterns {
|
||||
// The FROM pattern (index 0) sees the masked text; the
|
||||
// DELETE FROM pattern (index 4) sees the original to find
|
||||
// its own matches first.
|
||||
text := query
|
||||
if i == 0 {
|
||||
text = working
|
||||
}
|
||||
matches := re.FindAllStringSubmatch(text, -1)
|
||||
for _, m := range matches {
|
||||
if len(m) < 2 {
|
||||
continue
|
||||
}
|
||||
schema, table := splitSchemaTable(stripQuoting(m[1]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
op := canonicalOp(m[0])
|
||||
key := op + "::" + schema + "::" + table
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
refs = append(refs, TableRef{
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
Op: op,
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Slice(refs, func(i, j int) bool {
|
||||
if refs[i].Op != refs[j].Op {
|
||||
return refs[i].Op < refs[j].Op
|
||||
}
|
||||
if refs[i].Schema != refs[j].Schema {
|
||||
return refs[i].Schema < refs[j].Schema
|
||||
}
|
||||
return refs[i].Table < refs[j].Table
|
||||
})
|
||||
return refs
|
||||
}
|
||||
|
||||
// maskDeleteFromForFromPattern substitutes the FROM keyword in
|
||||
// "DELETE FROM" with a non-keyword sentinel so the bare FROM
|
||||
// regex doesn't double-match the same table reference. The
|
||||
// sentinel `__GFOX_FROM__` won't appear in real SQL and is
|
||||
// valid in the regex's character class so it gets ignored
|
||||
// silently. The DELETE FROM pattern still operates on the
|
||||
// original query string and finds its own match.
|
||||
var deleteFromMaskRe = regexp.MustCompile(`(?i)\b(DELETE)\s+FROM\b`)
|
||||
|
||||
func maskDeleteFromForFromPattern(query string) string {
|
||||
return deleteFromMaskRe.ReplaceAllString(query, "$1 __GFOX_FROM__")
|
||||
}
|
||||
|
||||
// stripQuoting removes the four shapes of SQL identifier quoting:
|
||||
// double quotes (ANSI), backticks (MySQL), brackets (T-SQL). The
|
||||
// inner content is returned unchanged otherwise.
|
||||
func stripQuoting(name string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
if len(name) >= 2 {
|
||||
first, last := name[0], name[len(name)-1]
|
||||
switch {
|
||||
case first == '"' && last == '"',
|
||||
first == '`' && last == '`',
|
||||
first == '[' && last == ']':
|
||||
return name[1 : len(name)-1]
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// splitSchemaTable separates `schema.table` into its parts.
|
||||
// Multi-dot identifiers (`db.schema.table`) collapse to schema=
|
||||
// "schema", table="table" — the leftmost segment is database-
|
||||
// scoped and rarely useful for graph queries.
|
||||
func splitSchemaTable(name string) (schema, table string) {
|
||||
if i := strings.LastIndex(name, "."); i >= 0 {
|
||||
schema = name[:i]
|
||||
table = name[i+1:]
|
||||
// If the schema piece itself has a database segment, keep
|
||||
// only the immediate parent.
|
||||
if j := strings.LastIndex(schema, "."); j >= 0 {
|
||||
schema = schema[j+1:]
|
||||
}
|
||||
return strings.TrimSpace(stripQuoting(schema)), strings.TrimSpace(stripQuoting(table))
|
||||
}
|
||||
return "", strings.TrimSpace(name)
|
||||
}
|
||||
|
||||
// createTableRe matches CREATE TABLE [IF NOT EXISTS] declarations
|
||||
// across the four canonical identifier-quoting styles. Used by
|
||||
// ExtractCreateTables for migration-file extraction — distinct
|
||||
// from ExtractTables, which scans query strings rather than DDL.
|
||||
var createTableRe = regexp.MustCompile(`(?i)\bCREATE\s+(?:GLOBAL\s+TEMPORARY\s+|LOCAL\s+TEMPORARY\s+|TEMPORARY\s+|TEMP\s+|UNLOGGED\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)`)
|
||||
|
||||
// ExtractCreateTables returns the tables declared by CREATE TABLE
|
||||
// statements in a SQL source file. Schema-qualified names retain
|
||||
// their schema in TableRef.Schema; identifier quoting is stripped.
|
||||
// Op is always "create".
|
||||
//
|
||||
// Used by migration-file extraction where the SQL source is a DDL
|
||||
// script rather than an embedded query string. Drop / alter
|
||||
// statements are deliberately not extracted — a migration that
|
||||
// drops a table doesn't *provide* the table to the rest of the
|
||||
// repo, and modeling alter-as-delta would require maintaining
|
||||
// per-migration ordering that's out of scope for the v1.
|
||||
func ExtractCreateTables(source string) []TableRef {
|
||||
if source == "" {
|
||||
return nil
|
||||
}
|
||||
matches := createTableRe.FindAllStringSubmatch(source, -1)
|
||||
seen := make(map[string]struct{})
|
||||
var refs []TableRef
|
||||
for _, m := range matches {
|
||||
if len(m) < 2 {
|
||||
continue
|
||||
}
|
||||
schema, table := splitSchemaTable(stripQuoting(m[1]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
key := schema + "::" + table
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
refs = append(refs, TableRef{
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
Op: "create",
|
||||
})
|
||||
}
|
||||
sort.Slice(refs, func(i, j int) bool {
|
||||
if refs[i].Schema != refs[j].Schema {
|
||||
return refs[i].Schema < refs[j].Schema
|
||||
}
|
||||
return refs[i].Table < refs[j].Table
|
||||
})
|
||||
return refs
|
||||
}
|
||||
|
||||
// CreateColumn is a column declared inside a CREATE TABLE body. Type is a
|
||||
// best-effort leading type token ("" when not determinable).
|
||||
type CreateColumn struct {
|
||||
Name string
|
||||
Type string
|
||||
}
|
||||
|
||||
// CreateTableDef is a CREATE TABLE statement together with its declared
|
||||
// columns — the schema-ingestion counterpart to TableRef, used where
|
||||
// columns become first-class graph nodes (migration files, live-DB DDL).
|
||||
type CreateTableDef struct {
|
||||
Schema string
|
||||
Table string
|
||||
Columns []CreateColumn
|
||||
}
|
||||
|
||||
// tableConstraintHeads are the leading keywords of a table-level
|
||||
// constraint clause inside a CREATE TABLE body; an entry beginning with
|
||||
// one of these is not a column.
|
||||
var tableConstraintHeads = map[string]bool{
|
||||
"PRIMARY": true, "FOREIGN": true, "CONSTRAINT": true, "UNIQUE": true,
|
||||
"CHECK": true, "EXCLUDE": true, "LIKE": true, "PARTITION": true,
|
||||
"INDEX": true, "KEY": true,
|
||||
}
|
||||
|
||||
// ExtractCreateTablesWithColumns parses CREATE TABLE statements and their
|
||||
// column lists out of SQL DDL. The parenthesised body is scanned with
|
||||
// paren-depth + quote tracking (so VARCHAR(255) / NUMERIC(10,2) stay
|
||||
// intact), split on top-level commas, and the leading identifier of each
|
||||
// non-constraint entry is taken as a column. Best-effort and parser-free,
|
||||
// matching the rest of this package. Tables are de-duplicated by
|
||||
// schema::table and returned in a deterministic order.
|
||||
func ExtractCreateTablesWithColumns(source string) []CreateTableDef {
|
||||
if source == "" {
|
||||
return nil
|
||||
}
|
||||
locs := createTableRe.FindAllStringSubmatchIndex(source, -1)
|
||||
seen := make(map[string]struct{})
|
||||
var out []CreateTableDef
|
||||
for _, loc := range locs {
|
||||
if len(loc) < 4 || loc[2] < 0 {
|
||||
continue
|
||||
}
|
||||
schema, table := splitSchemaTable(stripQuoting(source[loc[2]:loc[3]]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
key := schema + "::" + table
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
def := CreateTableDef{Schema: schema, Table: table}
|
||||
if body := balancedParenBody(source, loc[3]); body != "" {
|
||||
def.Columns = parseColumnDefs(body)
|
||||
}
|
||||
out = append(out, def)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Schema != out[j].Schema {
|
||||
return out[i].Schema < out[j].Schema
|
||||
}
|
||||
return out[i].Table < out[j].Table
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// balancedParenBody returns the contents between the first '(' at or after
|
||||
// `from` and its matching ')', or "" when there is no balanced group
|
||||
// (e.g. CREATE TABLE ... AS SELECT, or CREATE TABLE x LIKE y).
|
||||
func balancedParenBody(s string, from int) string {
|
||||
open := strings.IndexByte(s[from:], '(')
|
||||
if open < 0 {
|
||||
return ""
|
||||
}
|
||||
start := from + open
|
||||
depth := 0
|
||||
inSingle, inDouble := false, false
|
||||
for i := start; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case inSingle:
|
||||
if c == '\'' {
|
||||
inSingle = false
|
||||
}
|
||||
case inDouble:
|
||||
if c == '"' {
|
||||
inDouble = false
|
||||
}
|
||||
case c == '\'':
|
||||
inSingle = true
|
||||
case c == '"':
|
||||
inDouble = true
|
||||
case c == '(':
|
||||
depth++
|
||||
case c == ')':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[start+1 : i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseColumnDefs splits a CREATE TABLE body on top-level commas and reads
|
||||
// the leading identifier of each non-constraint entry as a column.
|
||||
func parseColumnDefs(body string) []CreateColumn {
|
||||
var cols []CreateColumn
|
||||
seen := make(map[string]struct{})
|
||||
for _, seg := range splitTopLevel(body, ',') {
|
||||
seg = strings.TrimSpace(seg)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(seg)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
if tableConstraintHeads[strings.ToUpper(stripQuoting(fields[0]))] {
|
||||
continue
|
||||
}
|
||||
name := stripQuoting(fields[0])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
typ := ""
|
||||
if len(fields) > 1 {
|
||||
typ = fields[1]
|
||||
}
|
||||
cols = append(cols, CreateColumn{Name: name, Type: typ})
|
||||
}
|
||||
return cols
|
||||
}
|
||||
|
||||
// splitTopLevel splits s on sep at paren depth 0, ignoring sep inside
|
||||
// single/double quotes or parentheses.
|
||||
func splitTopLevel(s string, sep byte) []string {
|
||||
var parts []string
|
||||
depth := 0
|
||||
inSingle, inDouble := false, false
|
||||
last := 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case inSingle:
|
||||
if c == '\'' {
|
||||
inSingle = false
|
||||
}
|
||||
case inDouble:
|
||||
if c == '"' {
|
||||
inDouble = false
|
||||
}
|
||||
case c == '\'':
|
||||
inSingle = true
|
||||
case c == '"':
|
||||
inDouble = true
|
||||
case c == '(':
|
||||
depth++
|
||||
case c == ')':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case c == sep && depth == 0:
|
||||
parts = append(parts, s[last:i])
|
||||
last = i + 1
|
||||
}
|
||||
}
|
||||
return append(parts, s[last:])
|
||||
}
|
||||
|
||||
// ColumnRef is a single resolved column reference. Op is "read" for
|
||||
// columns appearing in SELECT projections, WHERE clauses, ORDER BY,
|
||||
// or GROUP BY; "write" for columns in INSERT INTO (col-list) and
|
||||
// UPDATE … SET col = … assignments. Table identifies the table the
|
||||
// column belongs to; multi-table queries (joins) are restricted to
|
||||
// the first table reference because column-table association would
|
||||
// otherwise require a real SQL parser.
|
||||
type ColumnRef struct {
|
||||
Schema string
|
||||
Table string
|
||||
Column string
|
||||
Op string // "read" | "write"
|
||||
}
|
||||
|
||||
// insertColsRe matches `INSERT INTO tbl (col1, col2, …)` with
|
||||
// optional schema-qualified table.
|
||||
var insertColsRe = regexp.MustCompile(`(?is)\bINSERT\s+INTO\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\s*\(([^)]*)\)`)
|
||||
|
||||
// updateSetRe matches `UPDATE tbl SET col = …, col2 = …` capturing
|
||||
// the table name and the SET clause's content (greedy until WHERE
|
||||
// or end of statement).
|
||||
var updateSetRe = regexp.MustCompile(`(?is)\bUPDATE\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\s+SET\s+(.+?)(?:\bWHERE\b|\bRETURNING\b|;|$)`)
|
||||
|
||||
// selectFromRe matches `SELECT cols FROM tbl` for single-table
|
||||
// queries (no JOIN). Multi-table SELECTs return no column edges
|
||||
// because v1 can't disambiguate which table each column lives on.
|
||||
var selectFromRe = regexp.MustCompile(`(?is)\bSELECT\s+(.+?)\s+FROM\s+([a-zA-Z_"\x60\[][a-zA-Z0-9_."\x60\]]*)\b`)
|
||||
|
||||
// joinDetectRe is used to suppress SELECT column extraction when
|
||||
// the query has any kind of JOIN — preserves correctness over
|
||||
// completeness in v1.
|
||||
var joinDetectRe = regexp.MustCompile(`(?i)\bJOIN\b`)
|
||||
|
||||
// ExtractColumns walks a query string and returns the column
|
||||
// references it touches. The "Op" field distinguishes reads from
|
||||
// writes so the caller can emit EdgeReadsCol vs EdgeWritesCol.
|
||||
//
|
||||
// Limitations (intentional for v1):
|
||||
// - SELECT * does not produce edges (wildcard).
|
||||
// - Multi-table SELECTs (with JOIN) produce no column edges.
|
||||
// - Functions, expressions, and CASE statements degrade to no edge
|
||||
// for that particular projection slot.
|
||||
// - WHERE-clause column reads are extracted only when the value-side
|
||||
// reference is a bare identifier.
|
||||
func ExtractColumns(query string) []ColumnRef {
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
out := []ColumnRef{}
|
||||
seen := map[string]struct{}{}
|
||||
add := func(c ColumnRef) {
|
||||
key := c.Op + "::" + c.Schema + "::" + c.Table + "::" + c.Column
|
||||
if _, ok := seen[key]; ok {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, c)
|
||||
}
|
||||
|
||||
// INSERT INTO tbl (col1, col2, …) → writes.
|
||||
for _, m := range insertColsRe.FindAllStringSubmatch(query, -1) {
|
||||
schema, table := splitSchemaTable(stripQuoting(m[1]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
for _, c := range splitColumnList(m[2]) {
|
||||
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "write"})
|
||||
}
|
||||
}
|
||||
|
||||
// UPDATE tbl SET col = …, col2 = … → writes.
|
||||
for _, m := range updateSetRe.FindAllStringSubmatch(query, -1) {
|
||||
schema, table := splitSchemaTable(stripQuoting(m[1]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
for _, c := range splitSetAssignments(m[2]) {
|
||||
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "write"})
|
||||
}
|
||||
}
|
||||
|
||||
// SELECT col1, col2 FROM tbl (single-table) → reads.
|
||||
if !joinDetectRe.MatchString(query) {
|
||||
for _, m := range selectFromRe.FindAllStringSubmatch(query, -1) {
|
||||
projection := strings.TrimSpace(m[1])
|
||||
schema, table := splitSchemaTable(stripQuoting(m[2]))
|
||||
if table == "" {
|
||||
continue
|
||||
}
|
||||
if projection == "*" || projection == "" {
|
||||
continue
|
||||
}
|
||||
for _, c := range splitColumnList(projection) {
|
||||
add(ColumnRef{Schema: schema, Table: table, Column: c, Op: "read"})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Op != out[j].Op {
|
||||
return out[i].Op < out[j].Op
|
||||
}
|
||||
if out[i].Schema != out[j].Schema {
|
||||
return out[i].Schema < out[j].Schema
|
||||
}
|
||||
if out[i].Table != out[j].Table {
|
||||
return out[i].Table < out[j].Table
|
||||
}
|
||||
return out[i].Column < out[j].Column
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// splitColumnList parses a comma-separated column list (used for
|
||||
// INSERT and SELECT projections), returning bare column identifiers.
|
||||
// Aliases (`col AS alias`) collapse to the source column. Function
|
||||
// calls and expressions return "" and are dropped.
|
||||
func splitColumnList(list string) []string {
|
||||
out := []string{}
|
||||
depth := 0
|
||||
cur := strings.Builder{}
|
||||
flush := func() {
|
||||
s := strings.TrimSpace(cur.String())
|
||||
cur.Reset()
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
// Strip trailing AS alias.
|
||||
if idx := strings.LastIndex(strings.ToUpper(s), " AS "); idx >= 0 {
|
||||
s = strings.TrimSpace(s[:idx])
|
||||
}
|
||||
// Strip table prefix `tbl.col` → `col`.
|
||||
if idx := strings.LastIndex(s, "."); idx >= 0 {
|
||||
s = s[idx+1:]
|
||||
}
|
||||
s = strings.TrimSpace(stripQuoting(s))
|
||||
// Reject bare expressions (function calls, arithmetic, *).
|
||||
if s == "" || s == "*" || !isPlainSQLIdent(s) {
|
||||
return
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
for i := 0; i < len(list); i++ {
|
||||
c := list[i]
|
||||
switch c {
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
}
|
||||
cur.WriteByte(c)
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// splitSetAssignments parses a SET clause body (`col = ?, col2 =
|
||||
// fn(x)`) and returns the column names being written. The right-
|
||||
// hand expressions are skipped; column refs deeper than `tbl.col` are
|
||||
// reduced to `col`.
|
||||
func splitSetAssignments(set string) []string {
|
||||
out := []string{}
|
||||
depth := 0
|
||||
cur := strings.Builder{}
|
||||
flush := func() {
|
||||
seg := strings.TrimSpace(cur.String())
|
||||
cur.Reset()
|
||||
if seg == "" {
|
||||
return
|
||||
}
|
||||
eq := strings.Index(seg, "=")
|
||||
if eq < 0 {
|
||||
return
|
||||
}
|
||||
name := strings.TrimSpace(seg[:eq])
|
||||
if idx := strings.LastIndex(name, "."); idx >= 0 {
|
||||
name = name[idx+1:]
|
||||
}
|
||||
name = strings.TrimSpace(stripQuoting(name))
|
||||
if name != "" && isPlainSQLIdent(name) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
for i := 0; i < len(set); i++ {
|
||||
c := set[i]
|
||||
switch c {
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
}
|
||||
cur.WriteByte(c)
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// isPlainSQLIdent returns true when s is a single bare identifier
|
||||
// (letter/underscore start, alphanum/underscore body). Filters out
|
||||
// function-call shells (`fn(`), wildcards, arithmetic, and the like.
|
||||
func isPlainSQLIdent(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
isAlpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
|
||||
isDigit := c >= '0' && c <= '9'
|
||||
if i == 0 {
|
||||
if !isAlpha {
|
||||
return false
|
||||
}
|
||||
} else if !isAlpha && !isDigit {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// selectNoFromRe matches a `SELECT <projection>` that has no FROM
|
||||
// clause (e.g. `SELECT 1 AS id`). Used only as the fallback when a
|
||||
// query contains no FROM at all — selectFromRe takes precedence.
|
||||
var selectNoFromRe = regexp.MustCompile(`(?is)\bSELECT\s+(.+?)\s*(?:;|$)`)
|
||||
|
||||
// ProjectionColumns returns the output column names a query produces —
|
||||
// the columns of the relation the query *materialises*, as opposed to
|
||||
// ExtractColumns which attributes columns to the source tables a query
|
||||
// reads or writes. Used by the dbt / SQLMesh model extractor to record
|
||||
// a model's own columns.
|
||||
//
|
||||
// The output column of a projection slot is its alias when one is
|
||||
// present (`total AS order_total` → `order_total`) and the bare column
|
||||
// identifier otherwise (`customers.id` → `id`). Slots that are pure
|
||||
// expressions / function calls with no alias, and `*` / `tbl.*`
|
||||
// wildcards, produce no name.
|
||||
//
|
||||
// Heuristic (mirrors the v1 regex stance of the rest of this package):
|
||||
// the last `SELECT <projection> FROM` occurrence is taken as the
|
||||
// query's output projection — CTEs and subqueries appear earlier in
|
||||
// source order, so the final top-level SELECT is what the query
|
||||
// returns. When the query has no FROM clause at all the trailing
|
||||
// `SELECT <projection>` is used. Subquery-valued projection slots and
|
||||
// JOIN-bearing final SELECTs degrade gracefully to whatever bare
|
||||
// identifiers can still be recovered.
|
||||
func ProjectionColumns(query string) []string {
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
var projection string
|
||||
if all := selectFromRe.FindAllStringSubmatch(query, -1); len(all) > 0 {
|
||||
projection = strings.TrimSpace(all[len(all)-1][1])
|
||||
} else if m := selectNoFromRe.FindStringSubmatch(query); m != nil {
|
||||
projection = strings.TrimSpace(m[1])
|
||||
}
|
||||
if projection == "" || projection == "*" {
|
||||
return nil
|
||||
}
|
||||
cols := splitProjectionList(projection)
|
||||
seen := make(map[string]struct{}, len(cols))
|
||||
out := cols[:0]
|
||||
for _, c := range cols {
|
||||
if _, dup := seen[c]; dup {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// splitProjectionList parses a SELECT projection list and returns the
|
||||
// output column names — the alias when one is present, otherwise the
|
||||
// bare column identifier. Distinct from splitColumnList, which collapses
|
||||
// `col AS alias` to the source `col` because it attributes reads to a
|
||||
// source table; here the alias *is* the produced column name.
|
||||
func splitProjectionList(list string) []string {
|
||||
out := []string{}
|
||||
depth := 0
|
||||
cur := strings.Builder{}
|
||||
flush := func() {
|
||||
seg := strings.TrimSpace(cur.String())
|
||||
cur.Reset()
|
||||
if seg == "" {
|
||||
return
|
||||
}
|
||||
if idx := lastTopLevelAS(seg); idx >= 0 {
|
||||
alias := strings.TrimSpace(stripQuoting(strings.TrimSpace(seg[idx+4:])))
|
||||
if isPlainSQLIdent(alias) {
|
||||
out = append(out, alias)
|
||||
}
|
||||
return
|
||||
}
|
||||
// No alias: strip a `tbl.` qualifier, require a bare identifier.
|
||||
s := seg
|
||||
if i := strings.LastIndex(s, "."); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
s = strings.TrimSpace(stripQuoting(s))
|
||||
if s != "" && s != "*" && isPlainSQLIdent(s) {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
for i := 0; i < len(list); i++ {
|
||||
c := list[i]
|
||||
switch c {
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
}
|
||||
cur.WriteByte(c)
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// lastTopLevelAS returns the byte index of the last ` AS ` keyword in
|
||||
// seg that sits at paren depth zero, or -1 when there is none. Case-
|
||||
// insensitive. Keeps a cast's inner ` AS ` (`CAST(x AS int)`) from
|
||||
// being mistaken for a projection alias.
|
||||
func lastTopLevelAS(seg string) int {
|
||||
upper := strings.ToUpper(seg)
|
||||
depth := 0
|
||||
last := -1
|
||||
for i := 0; i+4 <= len(upper); i++ {
|
||||
switch upper[i] {
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
}
|
||||
if depth == 0 && upper[i:i+4] == " AS " {
|
||||
last = i
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
// ColumnNodeID returns the canonical synthetic ID for a column.
|
||||
func ColumnNodeID(dialect, schema, table, column string) string {
|
||||
if dialect == "" {
|
||||
dialect = "generic"
|
||||
}
|
||||
prefix := "col::" + dialect + "::"
|
||||
if schema == "" {
|
||||
return prefix + table + "." + column
|
||||
}
|
||||
return prefix + schema + "." + table + "." + column
|
||||
}
|
||||
|
||||
// MigrationNodeID is the canonical synthetic ID for a migration
|
||||
// node. The path component lets agents reach the originating file
|
||||
// in one step; the prefix matches the synthetic-ID convention
|
||||
// used by db:: tables and module:: dependencies.
|
||||
func MigrationNodeID(path string) string {
|
||||
return "migration::" + path
|
||||
}
|
||||
|
||||
// IsMigrationPath returns true when filePath looks like a SQL
|
||||
// migration file. Recognised conventions: any .sql file under a
|
||||
// directory whose name contains "migrate" or "migration"
|
||||
// (case-insensitive). Matches Rails (db/migrate/), golang-migrate
|
||||
// (migrations/), Alembic when wrapped (we mostly handle alembic
|
||||
// via Python parsers separately), and most ORM generators.
|
||||
func IsMigrationPath(filePath string) bool {
|
||||
if !strings.HasSuffix(strings.ToLower(filePath), ".sql") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(filePath)
|
||||
for _, segment := range []string{"/migrate/", "/migrations/", "/migrate.", "/migrations."} {
|
||||
if strings.Contains(lower, segment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.HasPrefix(lower, "migrate/") ||
|
||||
strings.HasPrefix(lower, "migrations/") ||
|
||||
strings.HasPrefix(lower, "db/migrate/") ||
|
||||
strings.HasPrefix(lower, "db/migrations/")
|
||||
}
|
||||
|
||||
// TableNodeID returns the canonical synthetic ID for a table
|
||||
// reference. Mirrors the ecosystem-prefix convention used by
|
||||
// module:: / external:: / annotation:: / event:: nodes — `db::`
|
||||
// keeps the table namespace distinct.
|
||||
//
|
||||
// dialect is the SQL dialect tag (postgres, mysql, sqlite,
|
||||
// generic) — included on the ID so cross-dialect projects can
|
||||
// distinguish a Postgres `users` table from a MySQL one in the
|
||||
// same graph. The default dialect is "generic" when the caller
|
||||
// doesn't know.
|
||||
func TableNodeID(dialect, schema, table string) string {
|
||||
if dialect == "" {
|
||||
dialect = "generic"
|
||||
}
|
||||
prefix := "db::" + dialect + "::"
|
||||
if schema == "" {
|
||||
return prefix + table
|
||||
}
|
||||
return prefix + schema + "." + table
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractTables_BasicSelect(t *testing.T) {
|
||||
refs := ExtractTables(`SELECT * FROM users WHERE id = 1`)
|
||||
if len(refs) != 1 {
|
||||
t.Fatalf("expected 1 ref, got %d: %+v", len(refs), refs)
|
||||
}
|
||||
if refs[0].Table != "users" || refs[0].Op != "select" {
|
||||
t.Errorf("got %+v", refs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_JoinClauses(t *testing.T) {
|
||||
refs := ExtractTables(`SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id`)
|
||||
if len(refs) != 2 {
|
||||
t.Fatalf("expected 2 refs, got %d: %+v", len(refs), refs)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, r := range refs {
|
||||
got[r.Table] = true
|
||||
}
|
||||
if !got["users"] || !got["orders"] {
|
||||
t.Errorf("missing refs: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_InsertUpdateDelete(t *testing.T) {
|
||||
cases := []struct {
|
||||
query string
|
||||
op string
|
||||
table string
|
||||
}{
|
||||
{`INSERT INTO users (id, name) VALUES (1, 'a')`, "insert", "users"},
|
||||
{`UPDATE accounts SET balance = 0 WHERE id = 1`, "update", "accounts"},
|
||||
{`DELETE FROM sessions WHERE expired = true`, "delete", "sessions"},
|
||||
{`TRUNCATE TABLE logs`, "truncate", "logs"},
|
||||
{`TRUNCATE logs2`, "truncate", "logs2"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
refs := ExtractTables(c.query)
|
||||
if len(refs) != 1 {
|
||||
t.Fatalf("%q → %d refs, want 1", c.query, len(refs))
|
||||
}
|
||||
if refs[0].Op != c.op || refs[0].Table != c.table {
|
||||
t.Errorf("%q → %+v, want op=%q table=%q", c.query, refs[0], c.op, c.table)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_QuotedIdentifiers(t *testing.T) {
|
||||
cases := []string{
|
||||
`SELECT * FROM "users" WHERE id = 1`, // ANSI
|
||||
"SELECT * FROM `users` WHERE id = 1", // MySQL backticks
|
||||
`SELECT * FROM [users] WHERE id = 1`, // T-SQL brackets
|
||||
}
|
||||
for _, q := range cases {
|
||||
refs := ExtractTables(q)
|
||||
if len(refs) != 1 || refs[0].Table != "users" {
|
||||
t.Errorf("%q → %+v, want table=users", q, refs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_SchemaQualified(t *testing.T) {
|
||||
refs := ExtractTables(`SELECT * FROM public.users JOIN auth.sessions ON id = session_id`)
|
||||
if len(refs) != 2 {
|
||||
t.Fatalf("expected 2 refs, got %d", len(refs))
|
||||
}
|
||||
got := map[string]string{} // table → schema
|
||||
for _, r := range refs {
|
||||
got[r.Table] = r.Schema
|
||||
}
|
||||
if got["users"] != "public" {
|
||||
t.Errorf("users schema = %q, want public", got["users"])
|
||||
}
|
||||
if got["sessions"] != "auth" {
|
||||
t.Errorf("sessions schema = %q, want auth", got["sessions"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_DeduplicatesSameOpAndTable(t *testing.T) {
|
||||
refs := ExtractTables(`SELECT * FROM users JOIN users u2 ON 1=1`)
|
||||
// Both users references share op=select, schema="" — should dedupe.
|
||||
if len(refs) != 1 {
|
||||
t.Errorf("expected dedup to single users ref, got %d", len(refs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_MixedOps(t *testing.T) {
|
||||
q := `WITH x AS (SELECT * FROM source)
|
||||
INSERT INTO target SELECT * FROM x`
|
||||
refs := ExtractTables(q)
|
||||
// `source` (select) + `target` (insert) + `x` (select) = 3
|
||||
if len(refs) != 3 {
|
||||
t.Errorf("expected 3 refs, got %d: %+v", len(refs), refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTables_Empty(t *testing.T) {
|
||||
if r := ExtractTables(""); len(r) != 0 {
|
||||
t.Errorf("empty query should yield no refs")
|
||||
}
|
||||
if r := ExtractTables("SELECT 1"); len(r) != 0 {
|
||||
t.Errorf("no-table query should yield no refs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripQuoting(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{`"users"`, "users"},
|
||||
{"`users`", "users"},
|
||||
{"[users]", "users"},
|
||||
{"users", "users"},
|
||||
{`"`, `"`},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := stripQuoting(c.in); got != c.want {
|
||||
t.Errorf("stripQuoting(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSchemaTable(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
schema, table string
|
||||
}{
|
||||
{"users", "", "users"},
|
||||
{"public.users", "public", "users"},
|
||||
{"db.public.users", "public", "users"}, // database segment dropped
|
||||
{`"public"."users"`, "public", "users"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s, t2 := splitSchemaTable(c.in)
|
||||
if s != c.schema || t2 != c.table {
|
||||
t.Errorf("splitSchemaTable(%q) = (%q,%q), want (%q,%q)", c.in, s, t2, c.schema, c.table)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCreateTables_Basic(t *testing.T) {
|
||||
src := `CREATE TABLE users (
|
||||
id BIGINT PRIMARY KEY,
|
||||
email TEXT UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
user_id BIGINT REFERENCES users(id),
|
||||
token TEXT
|
||||
);
|
||||
`
|
||||
refs := ExtractCreateTables(src)
|
||||
if len(refs) != 2 {
|
||||
t.Fatalf("expected 2 refs, got %d: %+v", len(refs), refs)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, r := range refs {
|
||||
got[r.Table] = true
|
||||
if r.Op != "create" {
|
||||
t.Errorf("op = %q, want create", r.Op)
|
||||
}
|
||||
}
|
||||
if !got["users"] || !got["sessions"] {
|
||||
t.Errorf("missing tables: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCreateTables_Variants(t *testing.T) {
|
||||
src := `
|
||||
CREATE TEMPORARY TABLE t1 (id INT);
|
||||
CREATE TEMP TABLE t2 (id INT);
|
||||
CREATE GLOBAL TEMPORARY TABLE t3 (id INT);
|
||||
CREATE UNLOGGED TABLE t4 (id INT);
|
||||
CREATE TABLE IF NOT EXISTS t5 (id INT);
|
||||
CREATE TABLE "quoted_name" (id INT);
|
||||
CREATE TABLE auth.tokens (id INT);
|
||||
`
|
||||
refs := ExtractCreateTables(src)
|
||||
if len(refs) != 7 {
|
||||
t.Fatalf("expected 7 refs, got %d: %+v", len(refs), refs)
|
||||
}
|
||||
gotSchemas := map[string]string{}
|
||||
for _, r := range refs {
|
||||
gotSchemas[r.Table] = r.Schema
|
||||
}
|
||||
if gotSchemas["tokens"] != "auth" {
|
||||
t.Errorf("auth.tokens schema = %q", gotSchemas["tokens"])
|
||||
}
|
||||
if gotSchemas["quoted_name"] != "" {
|
||||
t.Errorf("quoted-name should have no schema, got %q", gotSchemas["quoted_name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCreateTables_DedupesSameSchemaTable(t *testing.T) {
|
||||
src := `
|
||||
CREATE TABLE users (id INT);
|
||||
CREATE TABLE users (id INT);
|
||||
`
|
||||
refs := ExtractCreateTables(src)
|
||||
if len(refs) != 1 {
|
||||
t.Errorf("expected 1 deduped ref, got %d", len(refs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCreateTables_IgnoresAlterAndDrop(t *testing.T) {
|
||||
src := `
|
||||
ALTER TABLE users ADD COLUMN name TEXT;
|
||||
DROP TABLE sessions;
|
||||
`
|
||||
if got := ExtractCreateTables(src); len(got) != 0 {
|
||||
t.Errorf("alter/drop should not produce CREATE refs, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMigrationPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"db/migrate/001_create_users.sql", true},
|
||||
{"db/migrations/2024_01_init.sql", true},
|
||||
{"migrations/v1.sql", true},
|
||||
{"internal/db/migrations/init.sql", true},
|
||||
{"pkg/queries/select.sql", false},
|
||||
{"main.go", false},
|
||||
{"docs/migration_guide.md", false}, // not .sql
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsMigrationPath(c.path); got != c.want {
|
||||
t.Errorf("IsMigrationPath(%q) = %v, want %v", c.path, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationNodeID(t *testing.T) {
|
||||
if got := MigrationNodeID("db/migrate/001_init.sql"); got != "migration::db/migrate/001_init.sql" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractColumns_InsertWrites(t *testing.T) {
|
||||
got := ExtractColumns("INSERT INTO users (id, email, name) VALUES ($1, $2, $3)")
|
||||
wantCols := map[string]bool{"id": false, "email": false, "name": false}
|
||||
for _, c := range got {
|
||||
if c.Op != "write" {
|
||||
t.Errorf("expected write op, got %q for %q", c.Op, c.Column)
|
||||
}
|
||||
if c.Table != "users" {
|
||||
t.Errorf("expected table users, got %q", c.Table)
|
||||
}
|
||||
if _, ok := wantCols[c.Column]; ok {
|
||||
wantCols[c.Column] = true
|
||||
}
|
||||
}
|
||||
for col, found := range wantCols {
|
||||
if !found {
|
||||
t.Errorf("expected column %q in result, got %v", col, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractColumns_UpdateWrites(t *testing.T) {
|
||||
got := ExtractColumns("UPDATE users SET email = $1, updated_at = NOW() WHERE id = $2")
|
||||
wantCols := map[string]bool{"email": false, "updated_at": false}
|
||||
for _, c := range got {
|
||||
if c.Op != "write" {
|
||||
continue
|
||||
}
|
||||
if _, ok := wantCols[c.Column]; ok {
|
||||
wantCols[c.Column] = true
|
||||
}
|
||||
}
|
||||
for col, found := range wantCols {
|
||||
if !found {
|
||||
t.Errorf("expected SET column %q to surface as write; got %v", col, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractColumns_SingleTableSelectReads(t *testing.T) {
|
||||
got := ExtractColumns("SELECT id, email, name FROM users WHERE id = $1")
|
||||
want := map[string]bool{"id": false, "email": false, "name": false}
|
||||
for _, c := range got {
|
||||
if c.Op != "read" {
|
||||
continue
|
||||
}
|
||||
if c.Table != "users" {
|
||||
continue
|
||||
}
|
||||
if _, ok := want[c.Column]; ok {
|
||||
want[c.Column] = true
|
||||
}
|
||||
}
|
||||
for col, found := range want {
|
||||
if !found {
|
||||
t.Errorf("expected read column %q; got %v", col, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractColumns_StarSelectNoColumns(t *testing.T) {
|
||||
got := ExtractColumns("SELECT * FROM users")
|
||||
for _, c := range got {
|
||||
if c.Op == "read" {
|
||||
t.Errorf("SELECT * should not produce read columns; got %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractColumns_JoinSelectReturnsNoColumns(t *testing.T) {
|
||||
got := ExtractColumns("SELECT u.id, o.id FROM users u JOIN orders o ON o.user_id = u.id")
|
||||
for _, c := range got {
|
||||
if c.Op == "read" {
|
||||
t.Errorf("multi-table SELECT should not emit columns in v1; got %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestColumnNodeID(t *testing.T) {
|
||||
cases := []struct {
|
||||
dialect, schema, table, column, want string
|
||||
}{
|
||||
{"postgres", "public", "users", "email", "col::postgres::public.users.email"},
|
||||
{"", "", "users", "email", "col::generic::users.email"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := ColumnNodeID(c.dialect, c.schema, c.table, c.column); got != c.want {
|
||||
t.Errorf("ColumnNodeID(%q,%q,%q,%q) = %q, want %q",
|
||||
c.dialect, c.schema, c.table, c.column, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableNodeID(t *testing.T) {
|
||||
cases := []struct {
|
||||
dialect, schema, table, want string
|
||||
}{
|
||||
{"postgres", "public", "users", "db::postgres::public.users"},
|
||||
{"", "", "users", "db::generic::users"},
|
||||
{"mysql", "", "orders", "db::mysql::orders"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := TableNodeID(c.dialect, c.schema, c.table); got != c.want {
|
||||
t.Errorf("TableNodeID(%q,%q,%q) = %q, want %q",
|
||||
c.dialect, c.schema, c.table, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectionColumns(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
want []string
|
||||
}{
|
||||
{"plain columns", "SELECT id, name FROM users", []string{"id", "name"}},
|
||||
{"table-qualified", "SELECT u.id, u.name FROM users u", []string{"id", "name"}},
|
||||
{"alias wins", "SELECT total AS order_total, id FROM orders", []string{"order_total", "id"}},
|
||||
{"expression alias", "SELECT count(*) AS n, max(x) AS m FROM t", []string{"n", "m"}},
|
||||
{"cast inner AS ignored", "SELECT cast(zip AS text) AS zip_code FROM t", []string{"zip_code"}},
|
||||
{"unaliased expr dropped", "SELECT count(*), id FROM t", []string{"id"}},
|
||||
{"star yields nothing", "SELECT * FROM t", nil},
|
||||
{"final select of CTE", "WITH a AS (SELECT p FROM raw) SELECT x, y FROM a", []string{"x", "y"}},
|
||||
{"no from clause", "SELECT 1 AS id, 2 AS qty", []string{"id", "qty"}},
|
||||
{"empty", "", nil},
|
||||
{"dedup", "SELECT id, id FROM t", []string{"id"}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := ProjectionColumns(c.query)
|
||||
if len(got) != len(c.want) {
|
||||
t.Fatalf("ProjectionColumns(%q) = %v, want %v", c.query, got, c.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != c.want[i] {
|
||||
t.Fatalf("ProjectionColumns(%q) = %v, want %v", c.query, got, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user