zzet--gortex/internal/parser/languages/markdown_prose.go

package languages

import (
	"regexp"
	"strings"

	"github.com/zzet/gortex/internal/graph"
	"github.com/zzet/gortex/internal/parser"
	sitter "github.com/zzet/gortex/internal/parser/tsitter"
)

// Prose extraction turns a Markdown document into first-class,
// searchable KindDoc section nodes. Each heading opens a section that
// runs until the next heading of equal-or-shallower level; the
// section's paragraph text (markdown syntax stripped) becomes the
// node body, and the heading path forms a breadcrumb name. Section
// node IDs are derived from the heading path, never line numbers, so
// an incremental reindex of an edited file keeps stable identity.

// proseSection accumulates one heading-delimited region while the
// extractor walks the document in source order.
type proseSection struct {
	level     int      // heading depth (1..6); 0 is the synthetic preamble
	crumbs    []string // heading-path breadcrumb, root-first
	startLine int      // 1-based line of the opening heading
	endLine   int      // 1-based line of the last content line seen
	body      strings.Builder
}

// mdInlineLink / mdImage / mdEmphasis strip the common inline
// markdown constructs so the section body indexed for search is
// plain prose, not syntax.
var (
	mdInlineLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
	mdImage      = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
	mdInlineCode = regexp.MustCompile("`+([^`]*)`+")
)

// extractProseSections walks the Markdown AST and appends one KindDoc
// node per heading-delimited section, plus an EdgeDefines edge from
// the file node. It is called by MarkdownExtractor.Extract after the
// structural (heading / code-block / link) pass.
func (e *MarkdownExtractor) extractProseSections(root *sitter.Node, src []byte, filePath, fileID string, result *parser.ExtractionResult) {
	// A stack of open sections, deepest last. A new heading closes
	// every open section of equal-or-greater level before opening
	// its own.
	var stack []*proseSection
	var finished []*proseSection

	closeTo := func(level int) {
		for len(stack) > 0 && stack[len(stack)-1].level >= level {
			finished = append(finished, stack[len(stack)-1])
			stack = stack[:len(stack)-1]
		}
	}
	appendText := func(line int, text string) {
		text = strings.TrimSpace(stripMarkdownInline(text))
		if text == "" {
			return
		}
		// Text before the first heading belongs to a synthetic
		// preamble section so a doc with leading prose is not lost.
		if len(stack) == 0 {
			stack = append(stack, &proseSection{level: 0, crumbs: nil, startLine: line})
		}
		top := stack[len(stack)-1]
		if top.body.Len() > 0 {
			top.body.WriteByte(' ')
		}
		top.body.WriteString(text)
		if line > top.endLine {
			top.endLine = line
		}
	}

	var walk func(node *sitter.Node)
	walk = func(node *sitter.Node) {
		switch node.Type() {
		case "atx_heading":
			level, text := headingLevelText(node, src)
			if level == 0 || text == "" {
				return // malformed heading; children carry nothing useful
			}
			line := int(node.StartPoint().Row) + 1
			closeTo(level)
			var crumbs []string
			if len(stack) > 0 {
				crumbs = append(crumbs, stack[len(stack)-1].crumbs...)
			}
			crumbs = append(crumbs, text)
			stack = append(stack, &proseSection{
				level: level, crumbs: crumbs, startLine: line, endLine: line,
			})
			return
		case "paragraph":
			appendText(int(node.StartPoint().Row)+1, node.Content(src))
			return
		}
		for i, _nc := 0, int(node.NamedChildCount()); i < _nc; i++ {
			walk(node.NamedChild(i))
		}
	}
	walk(root)
	closeTo(1) // flush every still-open section
	finished = append(finished, stack...)

	base := proseFileBase(filePath)
	seen := make(map[string]bool, len(finished))
	for _, sec := range finished {
		// A section with no captured prose carries no search signal.
		if strings.TrimSpace(sec.body.String()) == "" {
			continue
		}
		id := proseSectionID(filePath, base, sec.crumbs)
		// A duplicated heading path would collide; disambiguate by
		// appending an occurrence index so identity stays stable per
		// (heading-path, occurrence).
		if seen[id] {
			id = proseSectionID(filePath, base, append(sec.crumbs, "")) + idSlug(itoaSmall(len(seen)))
		}
		seen[id] = true
		result.Nodes = append(result.Nodes, &graph.Node{
			ID:        id,
			Kind:      graph.KindDoc,
			Name:      proseBreadcrumb(base, sec.crumbs),
			FilePath:  filePath,
			StartLine: sec.startLine,
			EndLine:   sec.endLine,
			Language:  "markdown",
			Meta: map[string]any{
				"section_text":  sec.body.String(),
				"heading_path":  sec.crumbs,
				"heading_level": sec.level,
			},
		})
		result.Edges = append(result.Edges, &graph.Edge{
			From: fileID, To: id, Kind: graph.EdgeDefines,
			FilePath: filePath, Line: sec.startLine,
		})
	}
}

// headingLevelText returns the depth (1..6) and trimmed text of an
// atx_heading node, or (0, "") when either cannot be determined.
func headingLevelText(node *sitter.Node, src []byte) (int, string) {
	level := 0
	var text string
	for i, _nc := 0, int(node.NamedChildCount()); i < _nc; i++ {
		child := node.NamedChild(i)
		switch child.Type() {
		case "atx_h1_marker":
			level = 1
		case "atx_h2_marker":
			level = 2
		case "atx_h3_marker":
			level = 3
		case "atx_h4_marker":
			level = 4
		case "atx_h5_marker":
			level = 5
		case "atx_h6_marker":
			level = 6
		case "inline":
			text = strings.TrimSpace(child.Content(src))
		}
	}
	return level, text
}

// stripMarkdownInline removes the common inline markdown constructs
// (images, links, inline code, emphasis markers) so the indexed
// section body is plain prose. Link/image text is kept; the URL is
// dropped.
func stripMarkdownInline(s string) string {
	s = mdImage.ReplaceAllString(s, "$1")
	s = mdInlineLink.ReplaceAllString(s, "$1")
	s = mdInlineCode.ReplaceAllString(s, "$1")
	// Drop emphasis / strong markers and stray heading hashes.
	s = strings.NewReplacer("**", "", "__", "", "*", "", "_", "", "`", "").Replace(s)
	// Collapse internal whitespace runs.
	return strings.Join(strings.Fields(s), " ")
}

// proseFileBase returns the file's base name -- the breadcrumb root
// and the seed of the stable section ID.
func proseFileBase(filePath string) string {
	p := strings.ReplaceAll(filePath, "\\", "/")
	if i := strings.LastIndexByte(p, '/'); i >= 0 {
		p = p[i+1:]
	}
	return p
}

// proseBreadcrumb renders the human-facing section name -- the file
// base followed by the heading path, joined with " > ".
func proseBreadcrumb(base string, crumbs []string) string {
	parts := make([]string, 0, len(crumbs)+1)
	parts = append(parts, base)
	parts = append(parts, crumbs...)
	return strings.Join(parts, " > ")
}

// proseSectionID builds the stable section node ID. The ID is keyed
// on the slugified heading path -- NOT line numbers -- so editing the
// prose of a section, or inserting text above it, leaves its
// identity unchanged across an incremental reindex.
func proseSectionID(filePath, base string, crumbs []string) string {
	slug := idSlug(base)
	for _, c := range crumbs {
		if s := idSlug(c); s != "" {
			slug += "-" + s
		}
	}
	return filePath + "::doc:" + slug
}

// idSlug lowercases an identifier fragment and collapses every run of
// non-alphanumeric characters to a single hyphen -- a filesystem- and
// ID-safe slug.
func idSlug(s string) string {
	var b strings.Builder
	lastHyphen := true // suppress a leading hyphen
	for _, r := range strings.ToLower(s) {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
			lastHyphen = false
			continue
		}
		if !lastHyphen {
			b.WriteByte('-')
			lastHyphen = true
		}
	}
	return strings.Trim(b.String(), "-")
}

// itoaSmall renders a small non-negative int without importing
// strconv into this file.
func itoaSmall(n int) string {
	if n == 0 {
		return "0"
	}
	var buf [12]byte
	i := len(buf)
	for n > 0 {
		i--
		buf[i] = byte('0' + n%10)
		n /= 10
	}
	return string(buf[i:])
}