package resolver import ( "strings" "github.com/zzet/gortex/internal/graph" ) // attributeNonGoModuleImports runs as a serial post-pass at the // end of ResolveAll. It walks every EdgeImports edge that ended up // pointing at an `external::*` stub (i.e. resolveImport found no // matching package in the graph and no go.mod-derived dep contract // — typical for Python and Dart imports today), and for those whose // caller file lives in an ecosystem we know how to attribute, // rewrites the edge target onto a materialised KindModule node and // emits a matching EdgeDependsOnModule. // // This mirrors the goanalysis externals pass for Go without // requiring CGO type-checker output: text-based attribution maps // each import URI to a top-level module ID per ecosystem // (`module::pypi:` for Python, `module::pub:` for Dart; // stdlib variants for the curated builtins). False negatives // (modules left as `external::*`) are tolerated; false positives // would corrupt the module graph and are guarded against by the // per-ecosystem mapping refusing relative paths and ambiguous // shapes. // // Idempotent: every (file, module) pair is short-circuited via a // per-pass set so a second invocation in the same ResolveAll burst // emits no duplicate EdgeDependsOnModule edges. func (r *Resolver) attributeNonGoModuleImports() { // Python/Dart-only attribution (nonGoImportToModuleID handles exactly // those two ecosystems). Skip the EdgeImports scan when the graph has // neither language. if !r.graphHasLanguage("python") && !r.graphHasLanguage("dart") { return } fileLang := r.collectFileLanguages() type pendingEdge struct { edge *graph.Edge oldTo string moduleID string } var rewrites []pendingEdge moduleSeeds := map[string]moduleSeed{} dependsSeen := map[string]map[string]struct{}{} // fileID → set of moduleIDs for e := range r.graph.EdgesByKind(graph.EdgeImports) { if !strings.HasPrefix(e.To, "external::") { continue } // Scoped warm pass: an unchanged repo's external imports were already // attributed by a prior full pass, so only reconsider the changed repos. if !r.edgeFromInScope(e.From) { continue } lang, ok := fileLang[e.From] if !ok { continue } importPath := strings.TrimPrefix(e.To, "external::") moduleID, mlang, ok := nonGoImportToModuleID(lang, importPath) if !ok { continue } rewrites = append(rewrites, pendingEdge{ edge: e, oldTo: e.To, moduleID: moduleID, }) moduleSeeds[moduleID] = moduleSeed{ id: moduleID, language: mlang, path: importPath, } } if len(rewrites) == 0 { return } // Materialise module nodes first; later loops assume the // node exists when we add EdgeDependsOnModule. Batch the // presence check via GetNodesByIDs so disk backends do one // indexed SELECT IN (...) instead of one per-seed GetNode. seedIDs := make([]string, 0, len(moduleSeeds)) for id := range moduleSeeds { seedIDs = append(seedIDs, id) } existing := r.graph.GetNodesByIDs(seedIDs) for _, seed := range moduleSeeds { if _, ok := existing[seed.id]; ok { continue } r.graph.AddNode(buildNonGoModuleNode(seed)) } // Pre-build a set of every (fileID, moduleID) pair the graph // already has an EdgeDependsOnModule edge for. The old code // called hasDependsOnModule per rewrite, which on a disk backend // fans out to N per-file GetOutEdges queries (50k+ on a // gortex-scale pass). One EdgesByKind scan is an indexed range // read on every backend, plus a Go-side map build that turns // the per-rewrite check into a constant-time lookup. existingDepends := make(map[string]map[string]struct{}) for e := range r.graph.EdgesByKind(graph.EdgeDependsOnModule) { // Only the rewrites' own callers (in scope) are queried below, so a // scoped pass needs the dedupe index for just those repos. if !r.edgeFromInScope(e.From) { continue } set := existingDepends[e.From] if set == nil { set = make(map[string]struct{}) existingDepends[e.From] = set } set[e.To] = struct{}{} } // Rewrite each EdgeImports target and collect the re-bucket // jobs into one batch so disk backends commit in chunks rather // than once per import rewrite. reindexBatch := make([]graph.EdgeReindex, 0, len(rewrites)) for _, p := range rewrites { p.edge.To = p.moduleID p.edge.Origin = graph.OriginASTResolved reindexBatch = append(reindexBatch, graph.EdgeReindex{Edge: p.edge, OldTo: p.oldTo}) set, ok := dependsSeen[p.edge.From] if !ok { set = map[string]struct{}{} dependsSeen[p.edge.From] = set } if _, dup := set[p.moduleID]; dup { continue } set[p.moduleID] = struct{}{} // Avoid emitting a duplicate EdgeDependsOnModule when an // earlier pass already wired one (e.g. cold + warm // indexing of the same file). Constant-time map lookup // against the pre-built existingDepends index. if existing, ok := existingDepends[p.edge.From]; ok { if _, dup := existing[p.moduleID]; dup { continue } } r.graph.AddEdge(&graph.Edge{ From: p.edge.From, To: p.moduleID, Kind: graph.EdgeDependsOnModule, FilePath: p.edge.FilePath, Line: p.edge.Line, Confidence: 1.0, ConfidenceLabel: "EXTRACTED", Origin: graph.OriginASTResolved, }) } if len(reindexBatch) > 0 { r.graph.ReindexEdges(reindexBatch) } } // collectFileLanguages walks KindFile nodes once and returns // (file ID → language) for the per-edge dispatch above. func (r *Resolver) collectFileLanguages() map[string]string { out := map[string]string{} for n := range r.graph.NodesByKind(graph.KindFile) { out[n.ID] = n.Language } return out } // nonGoImportToModuleID maps a (language, importPath) pair to its // canonical KindModule ID. The second return value is the module's // own language tag (used at materialisation time so a stdlib module // claims the same language as the consuming file). // // Returns ok=false for ecosystems we don't yet attribute (everything // outside Python/Dart) and for shapes we can't disambiguate (Dart // relative imports, Python relative imports). These flow through // unchanged as `external::*` — a future pass can attribute them // once we wire pyproject/pubspec parsing. func nonGoImportToModuleID(language, importPath string) (string, string, bool) { importPath = strings.TrimSpace(importPath) if importPath == "" { return "", "", false } switch language { case "python": return pythonImportToModuleID(importPath) case "dart": return dartImportToModuleID(importPath) } return "", "", false } func pythonImportToModuleID(path string) (string, string, bool) { if strings.HasPrefix(path, ".") { // Relative import — attributable only with knowledge of // the package layout, which we don't have here. return "", "", false } if strings.Contains(path, "/") { // File-path stem from a project-rooted relative import // (emitted by emitImportFromRelative). The relative-import // resolver pass owns these; landing them on a pypi module // would invent a phantom package named after the repo's // directory layout. return "", "", false } top := path if i := strings.Index(path, "."); i > 0 { top = path[:i] } if top == "" { return "", "", false } if isPythonStdlibTop(top) { return "module::python:stdlib::" + top, "python", true } return "module::pypi:" + top, "python", true } func dartImportToModuleID(path string) (string, string, bool) { switch { case strings.HasPrefix(path, "dart:"): name := strings.TrimPrefix(path, "dart:") if name == "" { return "", "", false } return "module::dart:stdlib::" + name, "dart", true case strings.HasPrefix(path, "package:"): rest := strings.TrimPrefix(path, "package:") pkg := rest if i := strings.Index(rest, "/"); i > 0 { pkg = rest[:i] } if pkg == "" { return "", "", false } return "module::pub:" + pkg, "dart", true } return "", "", false } // buildNonGoModuleNode shapes the KindModule for the attribution // pass. The Meta block carries the original import path so // downstream consumers can recover it without re-parsing the ID. func buildNonGoModuleNode(seed moduleSeed) *graph.Node { ecosystem := "pypi" role := "external" switch { case strings.HasPrefix(seed.id, "module::python:stdlib"): ecosystem = "python:stdlib" role = "stdlib" case strings.HasPrefix(seed.id, "module::pypi:"): ecosystem = "pypi" case strings.HasPrefix(seed.id, "module::dart:stdlib"): ecosystem = "dart:stdlib" role = "stdlib" case strings.HasPrefix(seed.id, "module::pub:"): ecosystem = "pub" } return &graph.Node{ ID: seed.id, Kind: graph.KindModule, Name: seed.path, Language: seed.language, Meta: map[string]any{ "ecosystem": ecosystem, "role": role, "import_path": seed.path, }, } } // moduleSeed is exported only at the package scope for the // pendingEdge / moduleSeeds maps above. Unique per moduleID; later // edges with the same module reuse the seed. type moduleSeed struct { id string language string path string } // pythonStdlibTops is a curated set of the most-imported Python // stdlib top-level modules. This is the same trade-off // `isExternalPyImport` makes in the parser: false negatives push a // stdlib module into pypi: (where it'll show up as a phantom // external dependency in audits), false positives make a real pypi // dependency look like stdlib. We pick the conservative direction — // the list covers everything the typical app reaches into, and // false negatives at most degrade the audit's separation of concerns. var pythonStdlibTops = map[string]struct{}{ "abc": {}, "argparse": {}, "array": {}, "ast": {}, "asyncio": {}, "base64": {}, "binascii": {}, "bisect": {}, "builtins": {}, "calendar": {}, "cmath": {}, "collections": {}, "concurrent": {}, "configparser": {}, "contextlib": {}, "contextvars": {}, "copy": {}, "csv": {}, "ctypes": {}, "dataclasses": {}, "datetime": {}, "decimal": {}, "difflib": {}, "dis": {}, "email": {}, "enum": {}, "errno": {}, "fnmatch": {}, "fractions": {}, "functools": {}, "gc": {}, "getopt": {}, "gettext": {}, "glob": {}, "gzip": {}, "hashlib": {}, "heapq": {}, "hmac": {}, "html": {}, "http": {}, "imaplib": {}, "importlib": {}, "inspect": {}, "io": {}, "ipaddress": {}, "itertools": {}, "json": {}, "keyword": {}, "linecache": {}, "locale": {}, "logging": {}, "math": {}, "mimetypes": {}, "multiprocessing": {}, "numbers": {}, "operator": {}, "os": {}, "pathlib": {}, "pickle": {}, "platform": {}, "posixpath": {}, "pprint": {}, "queue": {}, "random": {}, "re": {}, "secrets": {}, "shutil": {}, "signal": {}, "smtplib": {}, "socket": {}, "sqlite3": {}, "ssl": {}, "stat": {}, "statistics": {}, "string": {}, "struct": {}, "subprocess": {}, "sys": {}, "sysconfig": {}, "tarfile": {}, "tempfile": {}, "textwrap": {}, "threading": {}, "time": {}, "timeit": {}, "tokenize": {}, "traceback": {}, "types": {}, "typing": {}, "unicodedata": {}, "unittest": {}, "urllib": {}, "uuid": {}, "warnings": {}, "weakref": {}, "xml": {}, "xmlrpc": {}, "zipfile": {}, "zlib": {}, "zoneinfo": {}, } func isPythonStdlibTop(name string) bool { _, ok := pythonStdlibTops[name] return ok }