chore: import upstream snapshot with attribution

internal/parser/languages/markdown.go

@@ -0,0 +1,316 @@
+package languages
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/parser"
+	sitter "github.com/zzet/gortex/internal/parser/tsitter"
+	tree_sitter_markdown "github.com/zzet/gortex/internal/parser/tsitter/markdown"
+)
+
+// linkPattern matches inline markdown links: [text](target)
+var linkPattern = regexp.MustCompile(`\[([^\]]+)\]\(([^)]+)\)`)
+
+// wikiLinkPattern matches wiki-style links: [[target]], [[target|label]],
+// [[target#heading]].
+var wikiLinkPattern = regexp.MustCompile(`\[\[([^\[\]]+)\]\]`)
+
+// MarkdownExtractor extracts Markdown document structure.
+type MarkdownExtractor struct {
+	lang *sitter.Language
+}
+
+func NewMarkdownExtractor() *MarkdownExtractor {
+	return &MarkdownExtractor{lang: tree_sitter_markdown.GetLanguage()}
+}
+
+func (e *MarkdownExtractor) Language() string     { return "markdown" }
+func (e *MarkdownExtractor) Extensions() []string { return []string{".md", ".mdx"} }
+
+func (e *MarkdownExtractor) Extract(filePath string, src []byte) (*parser.ExtractionResult, error) {
+	tree, err := parser.ParseFile(src, e.lang)
+	if err != nil {
+		return nil, err
+	}
+	defer tree.Close()
+
+	root := tree.RootNode()
+	result := &parser.ExtractionResult{}
+
+	fileNode := &graph.Node{
+		ID: filePath, Kind: graph.KindFile, Name: filePath,
+		FilePath: filePath, StartLine: 1, EndLine: int(root.EndPoint().Row) + 1,
+		Language: "markdown",
+	}
+	result.Nodes = append(result.Nodes, fileNode)
+
+	seen := make(map[string]bool)
+	seenLinks := make(map[string]bool)
+
+	// YAML frontmatter relations — wiki-links, document paths, and tags
+	// declared in the `---` fenced block at the top of the file.
+	e.extractFrontmatterRelations(src, filePath, fileNode.ID, seenLinks, result)
+
+	// Walk the AST for headings, code blocks, and inline content.
+	var walk func(node *sitter.Node)
+	walk = func(node *sitter.Node) {
+		switch node.Type() {
+		case "atx_heading":
+			e.extractHeading(node, src, filePath, fileNode.ID, seen, result)
+		case "fenced_code_block":
+			e.extractCodeBlock(node, src, filePath, fileNode.ID, seen, result)
+		case "paragraph", "inline":
+			// Extract inline and wiki links from inline text.
+			text := node.Content(src)
+			line := int(node.StartPoint().Row) + 1
+			e.extractLinks(text, filePath, fileNode.ID, seenLinks, result, line)
+			e.extractWikiLinks(text, filePath, fileNode.ID, seenLinks, result, line)
+		}
+
+		for i, _nc := 0, int(node.NamedChildCount()); i < _nc; i++ {
+			walk(node.NamedChild(i))
+		}
+	}
+	walk(root)
+
+	// Second pass: emit first-class KindDoc prose-section nodes --
+	// one per heading-delimited region, carrying the section body so
+	// it is searchable, not just the heading text.
+	e.extractProseSections(root, src, filePath, fileNode.ID, result)
+
+	return result, nil
+}
+
+func (e *MarkdownExtractor) extractHeading(node *sitter.Node, src []byte, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult) {
+	// Get heading level from marker type.
+	level := 0
+	var headingText string
+	for i, _nc := 0, int(node.NamedChildCount()); i < _nc; i++ {
+		child := node.NamedChild(i)
+		switch child.Type() {
+		case "atx_h1_marker":
+			level = 1
+		case "atx_h2_marker":
+			level = 2
+		case "atx_h3_marker":
+			level = 3
+		case "atx_h4_marker":
+			level = 4
+		case "atx_h5_marker":
+			level = 5
+		case "atx_h6_marker":
+			level = 6
+		case "inline":
+			headingText = strings.TrimSpace(child.Content(src))
+		}
+	}
+
+	if headingText == "" || level == 0 {
+		return
+	}
+
+	id := filePath + "::h" + string(rune('0'+level)) + ":" + headingText
+	if seen[id] {
+		return
+	}
+	seen[id] = true
+
+	result.Nodes = append(result.Nodes, &graph.Node{
+		ID: id, Kind: graph.KindVariable, Name: headingText,
+		FilePath: filePath, StartLine: int(node.StartPoint().Row) + 1, EndLine: int(node.EndPoint().Row) + 1,
+		Language: "markdown", Meta: map[string]any{"heading_level": level},
+	})
+	result.Edges = append(result.Edges, &graph.Edge{
+		From: fileID, To: id, Kind: graph.EdgeDefines,
+		FilePath: filePath, Line: int(node.StartPoint().Row) + 1,
+	})
+}
+
+func (e *MarkdownExtractor) extractCodeBlock(node *sitter.Node, src []byte, filePath, _ string, seen map[string]bool, result *parser.ExtractionResult) {
+	// Extract language from info_string.
+	for i, _nc := 0, int(node.NamedChildCount()); i < _nc; i++ {
+		child := node.NamedChild(i)
+		if child.Type() == "info_string" {
+			for j, _nc := 0, int(child.NamedChildCount()); j < _nc; j++ {
+				langNode := child.NamedChild(j)
+				if langNode.Type() == "language" {
+					lang := strings.TrimSpace(langNode.Content(src))
+					if lang != "" {
+						id := filePath + "::codeblock:" + lang + ":" + string(rune('0'+int(node.StartPoint().Row)))
+						if !seen[id] {
+							seen[id] = true
+							result.Nodes = append(result.Nodes, &graph.Node{
+								ID: id, Kind: graph.KindVariable, Name: lang + " code block",
+								FilePath: filePath, StartLine: int(node.StartPoint().Row) + 1, EndLine: int(node.EndPoint().Row) + 1,
+								Language: "markdown", Meta: map[string]any{"code_language": lang},
+							})
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (e *MarkdownExtractor) extractLinks(text, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult, line int) {
+	matches := linkPattern.FindAllStringSubmatch(text, -1)
+	for _, m := range matches {
+		target := m[2]
+		// Skip external URLs and anchors.
+		if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") || strings.HasPrefix(target, "#") {
+			continue
+		}
+		// Strip anchor from local links: "file.md#section" → "file.md"
+		if idx := strings.Index(target, "#"); idx > 0 {
+			target = target[:idx]
+		}
+		if target == "" {
+			continue
+		}
+		emitMarkdownLink(fileID, filePath, target, line, seen, result)
+	}
+}
+
+// extractWikiLinks emits a link edge for every wiki-style [[target]]
+// reference in text — the Obsidian / wiki convention the inline
+// [text](path) form does not cover.
+func (e *MarkdownExtractor) extractWikiLinks(text, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult, line int) {
+	for _, m := range wikiLinkPattern.FindAllStringSubmatch(text, -1) {
+		if target := normalizeWikiTarget(m[1]); target != "" {
+			emitMarkdownLink(fileID, filePath, target, line, seen, result)
+		}
+	}
+}
+
+// normalizeWikiTarget reduces a wiki-link body to its target page:
+// [[target|label]] → target, [[target#heading]] → target. A pure
+// same-page anchor ([[#heading]]) yields "".
+func normalizeWikiTarget(inner string) string {
+	t := strings.TrimSpace(inner)
+	if i := strings.IndexByte(t, '|'); i >= 0 {
+		t = strings.TrimSpace(t[:i])
+	}
+	if i := strings.IndexByte(t, '#'); i >= 0 {
+		t = strings.TrimSpace(t[:i])
+	}
+	return t
+}
+
+// emitMarkdownLink appends a document-to-document link edge, deduped
+// by target so the same relation is recorded once per file.
+func emitMarkdownLink(fileID, filePath, target string, line int, seen map[string]bool, result *parser.ExtractionResult) {
+	to := "unresolved::import::" + target
+	if seen[to] {
+		return
+	}
+	seen[to] = true
+	result.Edges = append(result.Edges, &graph.Edge{
+		From: fileID, To: to, Kind: graph.EdgeImports, FilePath: filePath, Line: line,
+	})
+}
+
+// extractFrontmatterRelations parses a leading YAML frontmatter block
+// and emits a relation edge for every wiki-link, markdown-document
+// path, and tag it declares.
+func (e *MarkdownExtractor) extractFrontmatterRelations(src []byte, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult) {
+	fm := frontmatterBlock(src)
+	if len(fm) == 0 {
+		return
+	}
+	var doc map[string]any
+	if err := yaml.Unmarshal(fm, &doc); err != nil || doc == nil {
+		return
+	}
+	for key, val := range doc {
+		e.emitFrontmatterValue(key, val, filePath, fileID, seen, result)
+	}
+}
+
+// frontmatterBlock returns the YAML text between a leading `---` fence
+// and its closing `---` / `...` fence, or nil when the file carries no
+// frontmatter.
+func frontmatterBlock(src []byte) []byte {
+	s := bytes.TrimPrefix(src, []byte{0xEF, 0xBB, 0xBF})
+	lines := bytes.SplitAfter(s, []byte("\n"))
+	if len(lines) == 0 || strings.TrimRight(string(lines[0]), "\r\n") != "---" {
+		return nil
+	}
+	var body [][]byte
+	for _, ln := range lines[1:] {
+		switch strings.TrimRight(string(ln), "\r\n") {
+		case "---", "...":
+			return bytes.Join(body, nil)
+		}
+		body = append(body, ln)
+	}
+	return nil
+}
+
+// emitFrontmatterValue walks a decoded frontmatter value — scalar,
+// sequence, or nested mapping — emitting a relation edge for each
+// wiki-link, markdown path, or tag string it finds.
+func (e *MarkdownExtractor) emitFrontmatterValue(key string, val any, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult) {
+	switch v := val.(type) {
+	case string:
+		e.emitFrontmatterString(key, v, filePath, fileID, seen, result)
+	case []any:
+		for _, item := range v {
+			e.emitFrontmatterValue(key, item, filePath, fileID, seen, result)
+		}
+	case map[string]any:
+		for k, item := range v {
+			e.emitFrontmatterValue(k, item, filePath, fileID, seen, result)
+		}
+	}
+}
+
+// emitFrontmatterString classifies one frontmatter scalar: a `tags`
+// entry becomes a topic relation, a value carrying a [[wiki-link]] or
+// a bare markdown-document path becomes a document relation.
+func (e *MarkdownExtractor) emitFrontmatterString(key, val, filePath, fileID string, seen map[string]bool, result *parser.ExtractionResult) {
+	val = strings.TrimSpace(val)
+	if val == "" {
+		return
+	}
+	// `tags:` / `tag:` — each value is a topic relation.
+	if key == "tags" || key == "tag" {
+		to := "unresolved::tag::" + val
+		if !seen[to] {
+			seen[to] = true
+			result.Edges = append(result.Edges, &graph.Edge{
+				From: fileID, To: to, Kind: graph.EdgeReferences, FilePath: filePath, Line: 1,
+			})
+		}
+		return
+	}
+	// A wiki-link relation expressed in frontmatter (related: "[[Foo]]").
+	if matches := wikiLinkPattern.FindAllStringSubmatch(val, -1); matches != nil {
+		for _, m := range matches {
+			if t := normalizeWikiTarget(m[1]); t != "" {
+				emitMarkdownLink(fileID, filePath, t, 1, seen, result)
+			}
+		}
+		return
+	}
+	// A bare markdown-document path (parent: ../index.md).
+	if isMarkdownDocPath(val) {
+		emitMarkdownLink(fileID, filePath, val, 1, seen, result)
+	}
+}
+
+// isMarkdownDocPath reports whether val is a local path to a markdown
+// document — used to treat a frontmatter scalar as a document relation.
+func isMarkdownDocPath(val string) bool {
+	if strings.Contains(val, "://") {
+		return false
+	}
+	lower := strings.ToLower(val)
+	return strings.HasSuffix(lower, ".md") || strings.HasSuffix(lower, ".mdx")
+}
+
+var _ parser.Extractor = (*MarkdownExtractor)(nil)