wehub-resource-sync
111 lines
2.9 KiB
Go
111 lines
2.9 KiB
Go
package config
|
|
|
|
import (
|
|
_ "embed"
|
|
"encoding/json"
|
|
"strings"
|
|
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
//go:embed parser_defaults.json
|
|
var parserDefaultsJSON []byte
|
|
|
|
type parserDefaultsData struct {
|
|
Families map[string]map[string]string `json:"families"`
|
|
Patterns []string `json:"patterns"`
|
|
}
|
|
|
|
var parsersData *parserDefaultsData
|
|
|
|
func init() {
|
|
parsersData = &parserDefaultsData{}
|
|
if err := json.Unmarshal(parserDefaultsJSON, parsersData); err != nil {
|
|
xlog.Warn("failed to parse parser_defaults.json", "error", err)
|
|
}
|
|
|
|
RegisterBackendHook("vllm", vllmDefaults)
|
|
RegisterBackendHook("vllm-omni", vllmDefaults)
|
|
}
|
|
|
|
// MatchParserDefaults returns parser defaults for the best-matching model family.
|
|
// Returns nil if no family matches. Used both at load time (via hook) and at import time.
|
|
func MatchParserDefaults(modelID string) map[string]string {
|
|
if parsersData == nil || len(parsersData.Patterns) == 0 {
|
|
return nil
|
|
}
|
|
normalized := normalizeModelID(modelID)
|
|
for _, pattern := range parsersData.Patterns {
|
|
if strings.Contains(normalized, pattern) {
|
|
if family, ok := parsersData.Families[pattern]; ok {
|
|
return family
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// productionEngineArgsDefaults are vLLM ≥ 0.6 features that production deployments
|
|
// almost always want. Applied at load time when the user hasn't set the key in
|
|
// engine_args. Anything user-supplied wins; we never silently override.
|
|
var productionEngineArgsDefaults = map[string]any{
|
|
"enable_prefix_caching": true,
|
|
"enable_chunked_prefill": true,
|
|
}
|
|
|
|
func vllmDefaults(cfg *ModelConfig, modelPath string) {
|
|
applyEngineArgDefaults(cfg)
|
|
applyParserDefaults(cfg)
|
|
}
|
|
|
|
// applyEngineArgDefaults seeds production-friendly engine_args without overwriting
|
|
// anything the user already set.
|
|
func applyEngineArgDefaults(cfg *ModelConfig) {
|
|
if cfg.EngineArgs == nil {
|
|
cfg.EngineArgs = map[string]any{}
|
|
}
|
|
for k, v := range productionEngineArgsDefaults {
|
|
if _, set := cfg.EngineArgs[k]; set {
|
|
continue
|
|
}
|
|
cfg.EngineArgs[k] = v
|
|
}
|
|
}
|
|
|
|
func applyParserDefaults(cfg *ModelConfig) {
|
|
hasToolParser := false
|
|
hasReasoningParser := false
|
|
for _, opt := range cfg.Options {
|
|
if strings.HasPrefix(opt, "tool_parser:") {
|
|
hasToolParser = true
|
|
}
|
|
if strings.HasPrefix(opt, "reasoning_parser:") {
|
|
hasReasoningParser = true
|
|
}
|
|
}
|
|
if hasToolParser && hasReasoningParser {
|
|
return
|
|
}
|
|
|
|
parsers := MatchParserDefaults(cfg.Model)
|
|
if parsers == nil {
|
|
parsers = MatchParserDefaults(cfg.Name)
|
|
}
|
|
if parsers == nil {
|
|
return
|
|
}
|
|
|
|
if !hasToolParser {
|
|
if tp, ok := parsers["tool_parser"]; ok {
|
|
cfg.Options = append(cfg.Options, "tool_parser:"+tp)
|
|
xlog.Debug("[parser_defaults] auto-set tool_parser", "parser", tp, "model", cfg.Model)
|
|
}
|
|
}
|
|
if !hasReasoningParser {
|
|
if rp, ok := parsers["reasoning_parser"]; ok {
|
|
cfg.Options = append(cfg.Options, "reasoning_parser:"+rp)
|
|
xlog.Debug("[parser_defaults] auto-set reasoning_parser", "parser", rp, "model", cfg.Model)
|
|
}
|
|
}
|
|
}
|