package rerank import ( "strings" "unicode" ) // QueryClass is the detected shape of a search query. The class tunes // the bm25 ↔ semantic blend in two places: the per-signal weight // scaling inside Pipeline.Rerank, and the α value of the hybrid // alpha-fusion path in internal/search/hybrid.go. Identifier and path // queries lean on exact-token (BM25) evidence; natural-language // queries give the semantic channel its full weight. type QueryClass int const ( // QueryClassUnknown is the zero value — "not yet classified". A // caller that leaves it unset lets Pipeline.Rerank auto-detect. QueryClassUnknown QueryClass = iota // QueryClassSymbol is a single identifier-shaped token: a symbol // or API name (validateToken, HTTPServer, pkg.Type). Exact-token // BM25 evidence dominates. QueryClassSymbol // QueryClassConcept is a natural-language description of intent // ("how does auth refresh", "validate user token"). The semantic // channel earns its keep here. This is the neutral baseline class. QueryClassConcept // QueryClassPath is a file-path-shaped query (internal/auth/token.go, // auth/handler). Path components are exact tokens and the semantic // channel is near-useless, so BM25 leans hardest of all classes. QueryClassPath // QueryClassSignature is a type- or function-signature fragment // ("func(ctx) error", "(string) bool"). Structural keywords carry // the signal: BM25-leaning but less extreme than a bare path. QueryClassSignature // QueryClassKeywordSoup is a degenerate boolean / OR-soup query // ("A OR B OR 'no access'") -- a list of disjuncts rather than a // description of intent. It defeats ordinary retrieval, so the // search handler treats it specially: LLM expansion is suppressed // and the soup is split into per-disjunct BM25 fetches. Detected // by LooksLikeKeywordSoup; a caller may also pin it via the // search_symbols query_class argument. QueryClassKeywordSoup ) // String returns the lowercase class name used by the search_symbols // query_class argument and surfaced back on the response. func (q QueryClass) String() string { switch q { case QueryClassSymbol: return "symbol" case QueryClassConcept: return "concept" case QueryClassPath: return "path" case QueryClassSignature: return "signature" case QueryClassKeywordSoup: return "keyword_soup" default: return "unknown" } } // ParseQueryClass maps the search_symbols query_class argument to a // QueryClass. "" and "auto" map to QueryClassUnknown — the signal to // auto-detect. The bool is false for an unrecognised value so the // caller can reject it with a clear error. func ParseQueryClass(s string) (QueryClass, bool) { switch strings.ToLower(strings.TrimSpace(s)) { case "", "auto": return QueryClassUnknown, true case "symbol": return QueryClassSymbol, true case "concept": return QueryClassConcept, true case "path": return QueryClassPath, true case "signature": return QueryClassSignature, true case "keyword_soup", "soup": return QueryClassKeywordSoup, true default: return QueryClassUnknown, false } } // Hybrid retrieval blend weights for the BM25 ↔ semantic mix used by // the alpha-fusion path in internal/search/hybrid.go. Lower α leans // toward BM25 (identifier-style queries where exact tokens dominate); // higher α gives semantic search more weight (natural-language // concept queries where wording varies). // // The values mirror the empirical sweet spot found by hybrid-search // benchmarks (HyDE / RAGAS papers): symbol queries benefit from a // strong BM25 prior; NL queries benefit from a near-balanced blend. const ( // AlphaSymbol weights BM25 vs semantic for identifier-shaped // queries (CamelCase, snake_case, namespaced, all-caps). The // blend favors BM25 because exact-token matches are the most // reliable signal for code identifiers. AlphaSymbol = 0.3 // AlphaNL weights BM25 vs semantic for natural-language queries // ("validate user token", "auth middleware"). Both channels // contribute roughly equally — semantic catches synonymous // wording, BM25 catches literal keywords. AlphaNL = 0.5 // AlphaPath weights BM25 vs semantic for file-path queries. The // most BM25-heavy blend: path components are exact tokens and the // semantic channel mostly contributes noise. AlphaPath = 0.15 // AlphaSignature weights BM25 vs semantic for type/function- // signature fragments. BM25-leaning — structural keywords are // literal — but less extreme than a bare path. AlphaSignature = 0.35 ) // IsSymbolQuery returns true when the query looks like a code // identifier rather than a natural-language description. The // classification drives: // // - the definition-keyword bias signal (only fires for symbol // queries — boosting `class Foo` for the query "Foo") // - the auto-adaptive α blend in hybrid retrieval (symbol queries // get AlphaSymbol, NL queries get AlphaNL) // // Heuristic: the query is a single token (no whitespace) that carries // at least one structural marker — CamelCase, snake_case, dotted / // double-colon / slash namespace qualifier, or an all-uppercase // shape. A multi-word query (containing spaces) is always treated as // NL even if individual tokens look identifier-shaped — the user is // describing intent, not naming a symbol. Empty or whitespace-only // queries return false. func IsSymbolQuery(query string) bool { q := strings.TrimSpace(query) if q == "" { return false } // Multi-token queries are natural-language. for _, r := range q { if unicode.IsSpace(r) { return false } } // Namespace qualifiers: pkg.Type, Module::Symbol, dir/file::Sym, // path/to/foo.go. Any of these is a strong symbol indicator. for _, sep := range []string{"::", ".", "/", "\\"} { if strings.Contains(q, sep) && hasIdentifierChar(q) { return true } } // Snake-case identifier: at least one underscore between letters // or digits and no whitespace already established above. if strings.Contains(q, "_") && hasIdentifierChar(q) { return true } // CamelCase / PascalCase: lowercase→uppercase transition or // uppercase→lowercase after another uppercase (e.g. HTTPServer). var prev rune for i, r := range q { if i > 0 { if unicode.IsUpper(r) && unicode.IsLower(prev) { return true } if unicode.IsLower(r) && unicode.IsUpper(prev) { return true } } prev = r } // All-uppercase token of length >= 2 (e.g. URL, JWT, API). Single // uppercase letter is too ambiguous to flag. if len(q) >= 2 { allUpper := true hasLetter := false for _, r := range q { if unicode.IsLetter(r) { hasLetter = true if !unicode.IsUpper(r) { allUpper = false break } } } if hasLetter && allUpper { return true } } return false } // hasIdentifierChar reports whether the string contains at least one // letter or digit. Used by IsSymbolQuery to avoid classifying punct- // only strings ("::", ".", "/") as symbol queries. func hasIdentifierChar(s string) bool { for _, r := range s { if unicode.IsLetter(r) || unicode.IsDigit(r) { return true } } return false } // ClassifyQuery detects the QueryClass of a query with cheap // structural heuristics — no LLM, no graph walk. Checks run in // precedence order: signature markers (parentheses, arrows) are the // most distinctive, then a path-separated single token, then the // identifier-shape rubric IsSymbolQuery uses, and finally the // natural-language default. An empty query classifies as concept. func ClassifyQuery(query string) QueryClass { q := strings.TrimSpace(query) if q == "" { return QueryClassConcept } // Keyword-soup detection runs first: a degenerate boolean OR-list // is its own class regardless of what its disjuncts look like. if soup, _ := LooksLikeKeywordSoup(q); soup { return QueryClassKeywordSoup } if looksLikeSignature(q) { return QueryClassSignature } if looksLikePath(q) { return QueryClassPath } if IsSymbolQuery(q) { return QueryClassSymbol } return QueryClassConcept } // looksLikeSignature reports whether the query carries an unambiguous // type/function-signature marker — a parenthesis or a Go/JS-style // return or lambda arrow. Natural-language queries virtually never do. func looksLikeSignature(q string) bool { if strings.ContainsAny(q, "()") { return true } return strings.Contains(q, "->") || strings.Contains(q, "=>") } // looksLikePath reports whether the query is a single whitespace-free // token carrying a directory separator — "internal/auth/token.go", // "auth/handler". A "::" or bare "." qualifier is NOT a path: those // stay in the symbol class. func looksLikePath(q string) bool { if strings.ContainsAny(q, " \t\n") { return false } if !strings.ContainsAny(q, "/\\") { return false } return hasIdentifierChar(q) } // AlphaFor returns the recommended α blend value for the query. It // scores the query shape on a continuous identifier↔natural-language // axis (see AlphaForContinuous) rather than snapping to a discrete // class bucket, so a half-identifier query like "validateToken // handler" lands between AlphaSymbol and AlphaNL instead of jumping a // whole tier. The discrete AlphaForClass remains for callers that // have already classified. func AlphaFor(query string) float64 { return AlphaForContinuous(query) } // queryStopwords are the relationship / question words that mark a // query as natural-language intent rather than an identifier list. // Deliberately excludes code-ish verbs (get/set/find/parse) which are // real symbol-name fragments. Kept local to rerank so the package // stays free of an internal/search import cycle. var queryStopwords = map[string]struct{}{ "how": {}, "does": {}, "do": {}, "the": {}, "a": {}, "an": {}, "of": {}, "to": {}, "is": {}, "are": {}, "in": {}, "on": {}, "for": {}, "and": {}, "or": {}, "with": {}, "where": {}, "what": {}, "when": {}, "why": {}, "which": {}, "this": {}, "that": {}, "from": {}, "by": {}, "as": {}, "about": {}, "into": {}, "via": {}, } // AlphaForContinuous returns a continuous α blend in [AlphaPath, // AlphaNL] derived from the query's shape rather than a discrete // class. Hard structural shapes (signature, path, keyword-soup) still // pin to their class α — they are unambiguous and the discrete value // is already correct, and keyword-soup carries non-α side effects // (LLM suppression, per-disjunct fetch) that must stay class-based. // Everything else interpolates between AlphaSymbol (a single // identifier-shaped token) and AlphaNL (multi-word prose) by how // identifier-like the query reads. // // The extreme points match the discrete table — AlphaForContinuous of // a bare CamelCase token equals AlphaSymbol and of a prose phrase // equals AlphaNL — so it is a strict refinement, not a re-tuning. func AlphaForContinuous(query string) float64 { q := strings.TrimSpace(query) if q == "" { return AlphaNL } if soup, _ := LooksLikeKeywordSoup(q); soup { return AlphaForClass(QueryClassKeywordSoup) } if looksLikeSignature(q) { return AlphaSignature } if looksLikePath(q) { return AlphaPath } // score in [0,1]: 1 == fully identifier-shaped, 0 == prose. score := identifierScore(q) // AlphaSymbol < AlphaNL, so a higher identifier score pulls α down // toward the BM25-leaning end. return AlphaNL + score*(AlphaSymbol-AlphaNL) } // identifierScore rates how identifier-like a (non-path, non-signature) // query is, in [0,1]. A single CamelCase/snake/dotted token scores 1; // a single plain word or any stopword-heavy phrase scores 0; a mixed // multi-word query interpolates by the fraction of identifier-shaped // tokens, discounted by stopword density and a gentle length penalty // (longer queries read more like prose). func identifierScore(q string) float64 { tokens := strings.Fields(q) n := len(tokens) if n == 0 { return 0 } if n == 1 { if IsSymbolQuery(tokens[0]) { return 1.0 } return 0 } idCount, stopCount := 0, 0 for _, t := range tokens { if IsSymbolQuery(t) { idCount++ } if _, ok := queryStopwords[strings.ToLower(t)]; ok { stopCount++ } } idFrac := float64(idCount) / float64(n) stopFrac := float64(stopCount) / float64(n) lengthPenalty := 0.12 * float64(n-2) score := idFrac - stopFrac - lengthPenalty if score < 0 { return 0 } if score > 1 { return 1 } return score } // AlphaForClass returns the α blend for a known class. The α-fusion // formula in hybrid.go is `final = (1-α)·text_rrf + α·vector_rrf`, so // a smaller α leans toward BM25. QueryClassUnknown falls back to the // natural-language blend. func AlphaForClass(c QueryClass) float64 { switch c { case QueryClassSymbol: return AlphaSymbol case QueryClassPath: return AlphaPath case QueryClassSignature: return AlphaSignature case QueryClassKeywordSoup: // Soup is split into per-disjunct BM25 fetches and the LLM // channel is suppressed -- lean hard on exact-token evidence. return AlphaSymbol default: // QueryClassConcept, QueryClassUnknown return AlphaNL } } // classWeights holds the per-class scaling applied to the bm25 and // semantic rerank signals. Every other signal scales by 1.0 — the // per-class lever tunes only the text-vs-semantic balance and leaves // the structural and session signals untouched. type classWeights struct { bm25 float64 semantic float64 // proximity scales the RWR/PPR centrality signal per class. // Concept (natural-language) queries lean hardest on graph // proximity — the user is describing intent, so "what is this code // structurally about" is the strongest discriminator. Exact // identifier / path queries dampen it so a central-but-wrong symbol // can't unseat the literal match the user named. proximity float64 } // classWeightTable is the tuned per-class multiplier set. Concept is // the neutral 1.0/1.0 baseline, so natural-language queries score // exactly as they did before per-class weighting existed; the // identifier, path, and signature classes push BM25 up and the // semantic channel down by amounts that grow with how literal the // query is. var classWeightTable = map[QueryClass]classWeights{ QueryClassConcept: {bm25: 1.00, semantic: 1.00, proximity: 1.30}, QueryClassSymbol: {bm25: 1.20, semantic: 0.65, proximity: 0.55}, QueryClassPath: {bm25: 1.25, semantic: 0.45, proximity: 0.40}, QueryClassSignature: {bm25: 1.10, semantic: 0.80, proximity: 0.75}, QueryClassKeywordSoup: {bm25: 1.20, semantic: 0.50, proximity: 0.60}, } // ClassWeightMultiplier returns the factor applied to a signal's // configured weight for a given query class. Only the bm25 and // semantic signals are class-sensitive; every other signal — and an // unknown class — returns 1.0. func ClassWeightMultiplier(c QueryClass, signal string) float64 { cw, ok := classWeightTable[c] if !ok { return 1.0 } switch signal { case SignalBM25: return cw.bm25 case SignalSemantic, SignalSemanticCosine: return cw.semantic case SignalProximity: if cw.proximity == 0 { return 1.0 } return cw.proximity default: return 1.0 } } // proseWeightTable is the per-signal multiplier applied on top of the // class / α weighting when Context.ProseMode is set — a documentation // (prose-section) query. It mirrors the shape of classWeightTable / // ClassWeightMultiplier but lives on its own lever, independent of // Alpha, so a docs query gets BOTH its query-shape blend and this // prose profile. // // The profile lifts the two signals that actually discriminate prose // (bm25 and semantic — a prose section has body text and an embedding, // little else) and zeroes the code-structural signals that are // meaningless for a KindDoc node: api_signature and type_signature // (a prose section has no signature) and definition_bias (no // definition keyword to match). A signal absent from the table keeps // its weight unchanged (multiplier 1.0), so the structural / session // signals that still make sense for prose (recency, feedback, // file_coherence) are untouched. var proseWeightTable = map[string]float64{ SignalBM25: 1.25, SignalSemantic: 1.30, SignalSemanticCosine: 1.30, SignalAPISignature: 0.0, SignalTypeSignature: 0.0, SignalDefinitionBias: 0.0, } // proseWeightMultiplier returns the prose-profile multiplier for a // signal. Signals not in proseWeightTable return 1.0 (unchanged). // Applied by Pipeline.Rerank only when Context.ProseMode is set, and // composed multiplicatively with the class / α multiplier so the two // levers stay independent. func proseWeightMultiplier(signal string) float64 { if m, ok := proseWeightTable[signal]; ok { return m } return 1.0 } // continuousClassMultiplier is the continuous analogue of // ClassWeightMultiplier: it maps a continuous α (as produced by // AlphaForContinuous) onto the bm25 / semantic weight multipliers, // interpolating smoothly between the natural-language anchor // (α=AlphaNL → 1.0/1.0, the neutral baseline) and the most // BM25-leaning anchor (α=AlphaPath → the path class's 1.25/0.45). // The endpoints reproduce the discrete classWeightTable exactly, so a // query that snaps to a class scores as it always did and only the // in-between queries see the interpolation. Pipeline.Rerank uses this // in place of ClassWeightMultiplier when Context.Alpha is set. func continuousClassMultiplier(alpha float64, signal string) float64 { a := alpha if a < AlphaPath { a = AlphaPath } if a > AlphaNL { a = AlphaNL } // frac: 0 at the NL anchor, 1 at the most BM25-leaning anchor. frac := (AlphaNL - a) / (AlphaNL - AlphaPath) // The BM25-leaning anchor's multipliers come from the path class, // the most exact-token-reliant of the discrete buckets. maxBM25 := classWeightTable[QueryClassPath].bm25 minSem := classWeightTable[QueryClassPath].semantic switch signal { case SignalBM25: return 1.0 + frac*(maxBM25-1.0) case SignalSemantic, SignalSemanticCosine: return 1.0 - frac*(1.0-minSem) case SignalProximity: // Interpolate proximity from the concept (NL) anchor down to // the path anchor, the mirror of how bm25/semantic move: a // natural-language query leans hardest on graph centrality, // an exact path/identifier query least. nlProx := classWeightTable[QueryClassConcept].proximity pathProx := classWeightTable[QueryClassPath].proximity return nlProx + frac*(pathProx-nlProx) default: return 1.0 } }