chore: import upstream snapshot with attribution
Deploy Site / deploy-vercel (push) Has been skipped
Deploy Site / deploy-docs (push) Has been skipped
Build Skills Index / build-index (push) Has been skipped
CI / Deny unrelated histories (push) Has been skipped
CI / Detect affected areas (push) Successful in 27m35s
CI / OSV scan (push) Failing after 4s
CI / Build&Test Docker image (push) Successful in 9s
CI / Supply-chain scan (push) Has been skipped
CI / Lint Docker scripts (push) Failing after 5m13s
CI / Check contributors (push) Failing after 12m8s
CI / Docs Site (push) Failing after 12m8s
CI / TypeScript (push) Failing after 12m8s
CI / Python lints (push) Failing after 12m9s
CI / Python tests (push) Failing after 12m9s
CI / Check uv.lock (push) Failing after 23m22s
CI / CI timing report (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
CI / All required checks pass (push) Has been cancelled
Deploy Site / deploy-vercel (push) Has been skipped
Deploy Site / deploy-docs (push) Has been skipped
Build Skills Index / build-index (push) Has been skipped
CI / Deny unrelated histories (push) Has been skipped
CI / Detect affected areas (push) Successful in 27m35s
CI / OSV scan (push) Failing after 4s
CI / Build&Test Docker image (push) Successful in 9s
CI / Supply-chain scan (push) Has been skipped
CI / Lint Docker scripts (push) Failing after 5m13s
CI / Check contributors (push) Failing after 12m8s
CI / Docs Site (push) Failing after 12m8s
CI / TypeScript (push) Failing after 12m8s
CI / Python lints (push) Failing after 12m9s
CI / Python tests (push) Failing after 12m9s
CI / Check uv.lock (push) Failing after 23m22s
CI / CI timing report (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
CI / All required checks pass (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,450 @@
|
||||
"""Memory provider plugin discovery.
|
||||
|
||||
Scans two directories for memory provider plugins:
|
||||
|
||||
1. Bundled providers: ``plugins/memory/<name>/`` (shipped with hermes-agent)
|
||||
2. User-installed providers: ``$HERMES_HOME/plugins/<name>/``
|
||||
|
||||
Each subdirectory must contain ``__init__.py`` with a class implementing
|
||||
the MemoryProvider ABC. On name collisions, bundled providers take
|
||||
precedence.
|
||||
|
||||
Only ONE provider can be active at a time, selected via
|
||||
``memory.provider`` in config.yaml.
|
||||
|
||||
Usage:
|
||||
from plugins.memory import discover_memory_providers, load_memory_provider
|
||||
|
||||
available = discover_memory_providers() # [(name, desc, available), ...]
|
||||
provider = load_memory_provider("mnemosyne") # MemoryProvider instance
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
from hermes_cli.config import cfg_get
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MEMORY_PLUGINS_DIR = Path(__file__).parent
|
||||
|
||||
# Synthetic parent package for user-installed providers, so they don't
|
||||
# collide with bundled providers in sys.modules.
|
||||
_USER_NAMESPACE = "_hermes_user_memory"
|
||||
|
||||
|
||||
def _register_synthetic_package(name: str, search_locations: List[str]) -> None:
|
||||
"""Register an empty package shell in sys.modules.
|
||||
|
||||
User-installed providers import as ``_hermes_user_memory.<name>``, a
|
||||
dotted name whose parents exist nowhere on disk. Unless those parents
|
||||
are present in ``sys.modules``, any relative import inside the plugin
|
||||
(``from . import config``) fails with
|
||||
``ModuleNotFoundError: No module named '_hermes_user_memory'`` — the
|
||||
same reason the loader already registers ``plugins`` and
|
||||
``plugins.memory`` for bundled providers.
|
||||
"""
|
||||
if name in sys.modules:
|
||||
return
|
||||
spec = importlib.machinery.ModuleSpec(name, None, is_package=True)
|
||||
spec.submodule_search_locations = search_locations
|
||||
sys.modules[name] = importlib.util.module_from_spec(spec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Directory helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_user_plugins_dir() -> Optional[Path]:
|
||||
"""Return ``$HERMES_HOME/plugins/`` or None if unavailable."""
|
||||
try:
|
||||
from hermes_constants import get_hermes_home
|
||||
d = get_hermes_home() / "plugins"
|
||||
return d if d.is_dir() else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_memory_provider_dir(path: Path) -> bool:
|
||||
"""Heuristic: does *path* look like a memory provider plugin?
|
||||
|
||||
Checks for ``register_memory_provider`` or ``MemoryProvider`` in the
|
||||
``__init__.py`` source. Cheap text scan — no import needed.
|
||||
"""
|
||||
init_file = path / "__init__.py"
|
||||
if not init_file.exists():
|
||||
return False
|
||||
try:
|
||||
source = init_file.read_text(errors="replace")[:8192]
|
||||
return "register_memory_provider" in source or "MemoryProvider" in source
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _iter_provider_dirs() -> List[Tuple[str, Path]]:
|
||||
"""Yield ``(name, path)`` for all discovered provider directories.
|
||||
|
||||
Scans bundled first, then user-installed. Bundled takes precedence
|
||||
on name collisions (first-seen wins via ``seen`` set).
|
||||
"""
|
||||
seen: set = set()
|
||||
dirs: List[Tuple[str, Path]] = []
|
||||
|
||||
# 1. Bundled providers (plugins/memory/<name>/)
|
||||
if _MEMORY_PLUGINS_DIR.is_dir():
|
||||
for child in sorted(_MEMORY_PLUGINS_DIR.iterdir()):
|
||||
if not child.is_dir() or child.name.startswith(("_", ".")):
|
||||
continue
|
||||
if not (child / "__init__.py").exists():
|
||||
continue
|
||||
seen.add(child.name)
|
||||
dirs.append((child.name, child))
|
||||
|
||||
# 2. User-installed providers ($HERMES_HOME/plugins/<name>/)
|
||||
user_dir = _get_user_plugins_dir()
|
||||
if user_dir:
|
||||
for child in sorted(user_dir.iterdir()):
|
||||
if not child.is_dir() or child.name.startswith(("_", ".")):
|
||||
continue
|
||||
if child.name in seen:
|
||||
continue # bundled takes precedence
|
||||
if not _is_memory_provider_dir(child):
|
||||
continue # skip non-memory plugins
|
||||
dirs.append((child.name, child))
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def find_provider_dir(name: str) -> Optional[Path]:
|
||||
"""Resolve a provider name to its directory.
|
||||
|
||||
Checks bundled first, then user-installed.
|
||||
"""
|
||||
# Bundled
|
||||
bundled = _MEMORY_PLUGINS_DIR / name
|
||||
if bundled.is_dir() and (bundled / "__init__.py").exists():
|
||||
return bundled
|
||||
# User-installed
|
||||
user_dir = _get_user_plugins_dir()
|
||||
if user_dir:
|
||||
user = user_dir / name
|
||||
if user.is_dir() and _is_memory_provider_dir(user):
|
||||
return user
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_memory_providers() -> List[Tuple[str, str, bool]]:
|
||||
"""Scan bundled and user-installed directories for available providers.
|
||||
|
||||
Returns list of (name, description, is_available) tuples.
|
||||
Bundled providers take precedence on name collisions.
|
||||
"""
|
||||
results = []
|
||||
|
||||
for name, child in _iter_provider_dirs():
|
||||
# Read description from plugin.yaml if available
|
||||
desc = ""
|
||||
yaml_file = child / "plugin.yaml"
|
||||
if yaml_file.exists():
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_file, encoding="utf-8-sig") as f:
|
||||
meta = yaml.safe_load(f) or {}
|
||||
desc = meta.get("description", "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Quick availability check — try loading and calling is_available()
|
||||
available = True
|
||||
try:
|
||||
provider = _load_provider_from_dir(child)
|
||||
if provider:
|
||||
available = provider.is_available()
|
||||
else:
|
||||
available = False
|
||||
except Exception:
|
||||
available = False
|
||||
|
||||
results.append((name, desc, available))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def load_memory_provider(name: str) -> Optional["MemoryProvider"]:
|
||||
"""Load and return a MemoryProvider instance by name.
|
||||
|
||||
Checks both bundled (``plugins/memory/<name>/``) and user-installed
|
||||
(``$HERMES_HOME/plugins/<name>/``) directories. Bundled takes
|
||||
precedence on name collisions.
|
||||
|
||||
Returns None if the provider is not found or fails to load.
|
||||
"""
|
||||
provider_dir = find_provider_dir(name)
|
||||
if not provider_dir:
|
||||
logger.debug("Memory provider '%s' not found in bundled or user plugins", name)
|
||||
return None
|
||||
|
||||
try:
|
||||
provider = _load_provider_from_dir(provider_dir)
|
||||
if provider:
|
||||
return provider
|
||||
logger.warning("Memory provider '%s' loaded but no provider instance found", name)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load memory provider '%s': %s", name, e)
|
||||
return None
|
||||
|
||||
|
||||
def _load_provider_from_dir(provider_dir: Path) -> Optional["MemoryProvider"]:
|
||||
"""Import a provider module and extract the MemoryProvider instance.
|
||||
|
||||
The module must have either:
|
||||
- A register(ctx) function (plugin-style) — we simulate a ctx
|
||||
- A top-level class that extends MemoryProvider — we instantiate it
|
||||
"""
|
||||
name = provider_dir.name
|
||||
# Use a separate namespace for user-installed plugins so they don't
|
||||
# collide with bundled providers in sys.modules.
|
||||
_is_bundled = _MEMORY_PLUGINS_DIR in provider_dir.parents or provider_dir.parent == _MEMORY_PLUGINS_DIR
|
||||
module_name = f"plugins.memory.{name}" if _is_bundled else f"{_USER_NAMESPACE}.{name}"
|
||||
init_file = provider_dir / "__init__.py"
|
||||
|
||||
if not init_file.exists():
|
||||
return None
|
||||
|
||||
# Check if already loaded. A synthetic package shell registered by
|
||||
# discover_plugin_cli_commands() for relative-import support has no
|
||||
# __file__; only reuse modules that were actually loaded from disk.
|
||||
cached = sys.modules.get(module_name)
|
||||
if cached is not None and getattr(cached, "__file__", None):
|
||||
mod = cached
|
||||
else:
|
||||
# Handle relative imports within the plugin
|
||||
# First ensure the parent packages are registered
|
||||
for parent in ("plugins", "plugins.memory"):
|
||||
if parent not in sys.modules:
|
||||
parent_path = Path(__file__).parent
|
||||
if parent == "plugins":
|
||||
parent_path = parent_path.parent
|
||||
parent_init = parent_path / "__init__.py"
|
||||
if parent_init.exists():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
parent, str(parent_init),
|
||||
submodule_search_locations=[str(parent_path)]
|
||||
)
|
||||
if spec:
|
||||
parent_mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[parent] = parent_mod
|
||||
try:
|
||||
spec.loader.exec_module(parent_mod)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# User-installed plugins need their synthetic parent registered the
|
||||
# same way, or relative imports inside the plugin cannot resolve.
|
||||
if not _is_bundled:
|
||||
_register_synthetic_package(_USER_NAMESPACE, [])
|
||||
|
||||
# Now load the provider module
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, str(init_file),
|
||||
submodule_search_locations=[str(provider_dir)]
|
||||
)
|
||||
if not spec:
|
||||
return None
|
||||
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = mod
|
||||
|
||||
# Register submodules so relative imports work
|
||||
# e.g., "from .store import MemoryStore" in holographic plugin
|
||||
for sub_file in provider_dir.glob("*.py"):
|
||||
if sub_file.name == "__init__.py":
|
||||
continue
|
||||
sub_name = sub_file.stem
|
||||
full_sub_name = f"{module_name}.{sub_name}"
|
||||
if full_sub_name not in sys.modules:
|
||||
sub_spec = importlib.util.spec_from_file_location(
|
||||
full_sub_name, str(sub_file)
|
||||
)
|
||||
if sub_spec:
|
||||
sub_mod = importlib.util.module_from_spec(sub_spec)
|
||||
sys.modules[full_sub_name] = sub_mod
|
||||
try:
|
||||
sub_spec.loader.exec_module(sub_mod)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to load submodule %s: %s", full_sub_name, e)
|
||||
|
||||
try:
|
||||
spec.loader.exec_module(mod)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to exec_module %s: %s", module_name, e)
|
||||
sys.modules.pop(module_name, None)
|
||||
return None
|
||||
|
||||
# Try register(ctx) pattern first (how our plugins are written)
|
||||
if hasattr(mod, "register"):
|
||||
collector = _ProviderCollector()
|
||||
try:
|
||||
mod.register(collector)
|
||||
if collector.provider:
|
||||
return collector.provider
|
||||
except Exception as e:
|
||||
logger.debug("register() failed for %s: %s", name, e)
|
||||
|
||||
# Fallback: find a MemoryProvider subclass and instantiate it
|
||||
from agent.memory_provider import MemoryProvider
|
||||
for attr_name in dir(mod):
|
||||
attr = getattr(mod, attr_name, None)
|
||||
if (isinstance(attr, type) and issubclass(attr, MemoryProvider)
|
||||
and attr is not MemoryProvider):
|
||||
try:
|
||||
return attr()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class _ProviderCollector:
|
||||
"""Fake plugin context that captures register_memory_provider calls."""
|
||||
|
||||
def __init__(self):
|
||||
self.provider = None
|
||||
|
||||
def register_memory_provider(self, provider):
|
||||
self.provider = provider
|
||||
|
||||
# No-op for other registration methods
|
||||
def register_tool(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def register_hook(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def register_cli_command(self, *args, **kwargs):
|
||||
pass # CLI registration happens via discover_plugin_cli_commands()
|
||||
|
||||
|
||||
def _get_active_memory_provider() -> Optional[str]:
|
||||
"""Read the active memory provider name from config.yaml.
|
||||
|
||||
Returns the provider name (e.g. ``"honcho"``) or None if no
|
||||
external provider is configured. Lightweight — only reads config,
|
||||
no plugin loading.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
config = load_config()
|
||||
return cfg_get(config, "memory", "provider") or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def discover_plugin_cli_commands() -> List[dict]:
|
||||
"""Return CLI commands for the **active** memory plugin only.
|
||||
|
||||
Only one memory provider can be active at a time (set via
|
||||
``memory.provider`` in config.yaml). This function reads that
|
||||
value and only loads CLI registration for the matching plugin.
|
||||
If no provider is active, no commands are registered.
|
||||
|
||||
Looks for a ``register_cli(subparser)`` function in the active
|
||||
plugin's ``cli.py``. Returns a list of at most one dict with
|
||||
keys: ``name``, ``help``, ``description``, ``setup_fn``,
|
||||
``handler_fn``.
|
||||
|
||||
This is a lightweight scan — it only imports ``cli.py``, not the
|
||||
full plugin module. Safe to call during argparse setup before
|
||||
any provider is loaded.
|
||||
"""
|
||||
results: List[dict] = []
|
||||
if not _MEMORY_PLUGINS_DIR.is_dir():
|
||||
return results
|
||||
|
||||
active_provider = _get_active_memory_provider()
|
||||
if not active_provider:
|
||||
return results
|
||||
|
||||
# Only look at the active provider's directory
|
||||
plugin_dir = find_provider_dir(active_provider)
|
||||
if not plugin_dir:
|
||||
return results
|
||||
|
||||
cli_file = plugin_dir / "cli.py"
|
||||
if not cli_file.exists():
|
||||
return results
|
||||
|
||||
_is_bundled = _MEMORY_PLUGINS_DIR in plugin_dir.parents or plugin_dir.parent == _MEMORY_PLUGINS_DIR
|
||||
module_name = f"plugins.memory.{active_provider}.cli" if _is_bundled else f"{_USER_NAMESPACE}.{active_provider}.cli"
|
||||
try:
|
||||
# Import the CLI module (lightweight — no SDK needed)
|
||||
if module_name in sys.modules:
|
||||
cli_mod = sys.modules[module_name]
|
||||
else:
|
||||
if not _is_bundled:
|
||||
# cli.py imports as _hermes_user_memory.<name>.cli, usually
|
||||
# before the provider itself is loaded. Register its parent
|
||||
# packages so relative imports inside cli.py
|
||||
# ("from . import config") resolve without executing the
|
||||
# plugin's __init__.py. The package shell has no __file__,
|
||||
# so _load_provider_from_dir() will still load the real
|
||||
# module later instead of reusing the shell.
|
||||
_register_synthetic_package(_USER_NAMESPACE, [])
|
||||
_register_synthetic_package(
|
||||
f"{_USER_NAMESPACE}.{active_provider}", [str(plugin_dir)]
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, str(cli_file)
|
||||
)
|
||||
if not spec or not spec.loader:
|
||||
return results
|
||||
cli_mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = cli_mod
|
||||
spec.loader.exec_module(cli_mod)
|
||||
|
||||
register_cli = getattr(cli_mod, "register_cli", None)
|
||||
if not callable(register_cli):
|
||||
return results
|
||||
|
||||
# Read metadata from plugin.yaml if available
|
||||
help_text = f"Manage {active_provider} memory plugin"
|
||||
description = ""
|
||||
yaml_file = plugin_dir / "plugin.yaml"
|
||||
if yaml_file.exists():
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_file, encoding="utf-8-sig") as f:
|
||||
meta = yaml.safe_load(f) or {}
|
||||
desc = meta.get("description", "")
|
||||
if desc:
|
||||
help_text = desc
|
||||
description = desc
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
handler_fn = getattr(cli_mod, f"{active_provider}_command", None) or \
|
||||
getattr(cli_mod, "honcho_command", None)
|
||||
|
||||
results.append({
|
||||
"name": active_provider,
|
||||
"help": help_text,
|
||||
"description": description,
|
||||
"setup_fn": register_cli,
|
||||
"handler_fn": handler_fn,
|
||||
"plugin": active_provider,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.debug("Failed to scan CLI for memory plugin '%s': %s", active_provider, e)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,41 @@
|
||||
# ByteRover Memory Provider
|
||||
|
||||
Persistent memory via the `brv` CLI — hierarchical knowledge tree with tiered retrieval (fuzzy text → LLM-driven search).
|
||||
|
||||
## Requirements
|
||||
|
||||
Install the ByteRover CLI:
|
||||
```bash
|
||||
curl -fsSL https://byterover.dev/install.sh | sh
|
||||
# or
|
||||
npm install -g byterover-cli
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "byterover"
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider byterover
|
||||
# Optional cloud sync:
|
||||
echo "BRV_API_KEY=your-key" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
| Env Var | Required | Description |
|
||||
|---------|----------|-------------|
|
||||
| `BRV_API_KEY` | No | Cloud sync key (optional, local-first by default) |
|
||||
|
||||
Working directory: `$HERMES_HOME/byterover/` (profile-scoped).
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `brv_query` | Search the knowledge tree |
|
||||
| `brv_curate` | Store facts, decisions, patterns |
|
||||
| `brv_status` | CLI version, tree stats, sync state |
|
||||
@@ -0,0 +1,449 @@
|
||||
"""ByteRover memory plugin — MemoryProvider interface.
|
||||
|
||||
Persistent memory via the ByteRover CLI (``brv``). Organizes knowledge into
|
||||
a hierarchical context tree with tiered retrieval (fuzzy text → LLM-driven
|
||||
search). Local-first with optional cloud sync.
|
||||
|
||||
Original PR #3499 by hieuntg81, adapted to MemoryProvider ABC.
|
||||
|
||||
Requires: ``brv`` CLI installed (npm install -g byterover-cli or
|
||||
curl -fsSL https://byterover.dev/install.sh | sh).
|
||||
|
||||
Config via environment variables (profile-scoped via each profile's .env):
|
||||
BRV_API_KEY — ByteRover API key (for cloud features, optional for local)
|
||||
|
||||
Config via config.yaml:
|
||||
memory:
|
||||
byterover:
|
||||
auto_extract: false # disable automatic brv curate hooks
|
||||
|
||||
Working directory: $HERMES_HOME/byterover/ (profile-scoped context tree)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.memory_provider import MemoryProvider
|
||||
from tools.registry import tool_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Timeouts
|
||||
_QUERY_TIMEOUT = 10 # brv query — should be fast
|
||||
_CURATE_TIMEOUT = 120 # brv curate — may involve LLM processing
|
||||
|
||||
# Minimum lengths to filter noise
|
||||
_MIN_QUERY_LEN = 10
|
||||
_MIN_OUTPUT_LEN = 20
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
if isinstance(value, str):
|
||||
text = value.strip().lower()
|
||||
if text in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _load_plugin_config() -> Dict[str, Any]:
|
||||
"""Read ByteRover's profile-scoped memory config.
|
||||
|
||||
New memory-provider setup stores non-secret provider settings under
|
||||
``memory.<provider>``. Some users also set ``memory.provider_config`` from
|
||||
early docs/issues, so accept it as a compatibility fallback.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
config = load_config()
|
||||
memory_config = config.get("memory", {})
|
||||
if not isinstance(memory_config, dict):
|
||||
return {}
|
||||
|
||||
provider_config = memory_config.get("byterover", {})
|
||||
if isinstance(provider_config, dict) and provider_config:
|
||||
return dict(provider_config)
|
||||
|
||||
legacy_config = memory_config.get("provider_config", {})
|
||||
if isinstance(legacy_config, dict):
|
||||
return dict(legacy_config)
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# brv binary resolution (cached, thread-safe)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_brv_path_lock = threading.Lock()
|
||||
_cached_brv_path: Optional[str] = None
|
||||
|
||||
|
||||
def _resolve_brv_path() -> Optional[str]:
|
||||
"""Find the brv binary on PATH or well-known install locations."""
|
||||
global _cached_brv_path
|
||||
with _brv_path_lock:
|
||||
if _cached_brv_path is not None:
|
||||
return _cached_brv_path if _cached_brv_path != "" else None
|
||||
|
||||
found = shutil.which("brv")
|
||||
if not found:
|
||||
home = Path.home()
|
||||
candidates = [
|
||||
home / ".brv-cli" / "bin" / "brv",
|
||||
Path("/usr/local/bin/brv"),
|
||||
home / ".npm-global" / "bin" / "brv",
|
||||
]
|
||||
for c in candidates:
|
||||
if c.exists():
|
||||
found = str(c)
|
||||
break
|
||||
|
||||
with _brv_path_lock:
|
||||
if _cached_brv_path is not None:
|
||||
return _cached_brv_path if _cached_brv_path != "" else None
|
||||
_cached_brv_path = found or ""
|
||||
return found
|
||||
|
||||
|
||||
def _run_brv(args: List[str], timeout: int = _QUERY_TIMEOUT,
|
||||
cwd: str = None) -> dict:
|
||||
"""Run a brv CLI command. Returns {success, output, error}."""
|
||||
brv_path = _resolve_brv_path()
|
||||
if not brv_path:
|
||||
return {"success": False, "error": "brv CLI not found. Install: npm install -g byterover-cli"}
|
||||
|
||||
cmd = [brv_path] + args
|
||||
effective_cwd = cwd or str(_get_brv_cwd())
|
||||
Path(effective_cwd).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
brv_bin_dir = str(Path(brv_path).parent)
|
||||
env["PATH"] = brv_bin_dir + os.pathsep + env.get("PATH", "")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True,
|
||||
timeout=timeout, cwd=effective_cwd, env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
stdout = result.stdout.strip()
|
||||
stderr = result.stderr.strip()
|
||||
|
||||
if result.returncode == 0:
|
||||
return {"success": True, "output": stdout}
|
||||
return {"success": False, "error": stderr or stdout or f"brv exited {result.returncode}"}
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"success": False, "error": f"brv timed out after {timeout}s"}
|
||||
except FileNotFoundError:
|
||||
global _cached_brv_path
|
||||
with _brv_path_lock:
|
||||
_cached_brv_path = None
|
||||
return {"success": False, "error": "brv CLI not found"}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
def _get_brv_cwd() -> Path:
|
||||
"""Profile-scoped working directory for the brv context tree."""
|
||||
from hermes_constants import get_hermes_home
|
||||
return get_hermes_home() / "byterover"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
QUERY_SCHEMA = {
|
||||
"name": "brv_query",
|
||||
"description": (
|
||||
"Search ByteRover's persistent knowledge tree for relevant context. "
|
||||
"Returns memories, project knowledge, architectural decisions, and "
|
||||
"patterns from previous sessions. Use for any question where past "
|
||||
"context would help."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "What to search for."},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
}
|
||||
|
||||
CURATE_SCHEMA = {
|
||||
"name": "brv_curate",
|
||||
"description": (
|
||||
"Store important information in ByteRover's persistent knowledge tree. "
|
||||
"Use for architectural decisions, bug fixes, user preferences, project "
|
||||
"patterns — anything worth remembering across sessions. ByteRover's LLM "
|
||||
"automatically categorizes and organizes the memory."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string", "description": "The information to remember."},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
}
|
||||
|
||||
STATUS_SCHEMA = {
|
||||
"name": "brv_status",
|
||||
"description": "Check ByteRover status — CLI version, context tree stats, cloud sync state.",
|
||||
"parameters": {"type": "object", "properties": {}, "required": []},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryProvider implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ByteRoverMemoryProvider(MemoryProvider):
|
||||
"""ByteRover persistent memory via the brv CLI."""
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self._config = dict(config) if config is not None else _load_plugin_config()
|
||||
self._auto_extract = _coerce_bool(self._config.get("auto_extract"), True)
|
||||
self._cwd = ""
|
||||
self._session_id = ""
|
||||
self._turn_count = 0
|
||||
self._sync_thread: Optional[threading.Thread] = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "byterover"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if brv CLI is installed. No network calls."""
|
||||
return _resolve_brv_path() is not None
|
||||
|
||||
def get_config_schema(self):
|
||||
return [
|
||||
{
|
||||
"key": "api_key",
|
||||
"description": "ByteRover API key (optional, for cloud sync)",
|
||||
"secret": True,
|
||||
"env_var": "BRV_API_KEY",
|
||||
"url": "https://app.byterover.dev",
|
||||
},
|
||||
{
|
||||
"key": "auto_extract",
|
||||
"description": "Automatically curate completed turns and compression/memory hooks",
|
||||
"default": "true",
|
||||
"choices": ["true", "false"],
|
||||
},
|
||||
]
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
self._cwd = str(_get_brv_cwd())
|
||||
self._session_id = session_id
|
||||
self._turn_count = 0
|
||||
Path(self._cwd).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
if not _resolve_brv_path():
|
||||
return ""
|
||||
return (
|
||||
"# ByteRover Memory\n"
|
||||
"Active. Persistent knowledge tree with hierarchical context.\n"
|
||||
"Use brv_query to search past knowledge, brv_curate to store "
|
||||
"important facts, brv_status to check state."
|
||||
)
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
"""Run brv query synchronously before the agent's first LLM call.
|
||||
|
||||
Blocks until the query completes (up to _QUERY_TIMEOUT seconds), ensuring
|
||||
the result is available as context before the model is called.
|
||||
"""
|
||||
if not query or len(query.strip()) < _MIN_QUERY_LEN:
|
||||
return ""
|
||||
result = _run_brv(
|
||||
["query", "--", query.strip()[:5000]],
|
||||
timeout=_QUERY_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
if result["success"] and result.get("output"):
|
||||
output = result["output"].strip()
|
||||
if len(output) > _MIN_OUTPUT_LEN:
|
||||
return f"## ByteRover Context\n{output}"
|
||||
return ""
|
||||
|
||||
def queue_prefetch(self, query: str, *, session_id: str = "") -> None:
|
||||
"""No-op: prefetch() now runs synchronously at turn start."""
|
||||
pass
|
||||
|
||||
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
||||
"""Curate the conversation turn in background (non-blocking)."""
|
||||
self._turn_count += 1
|
||||
if not self._auto_extract:
|
||||
logger.debug("ByteRover sync_turn skipped (auto_extract disabled)")
|
||||
return
|
||||
|
||||
# Only curate substantive turns
|
||||
if len(user_content.strip()) < _MIN_QUERY_LEN:
|
||||
return
|
||||
|
||||
def _sync():
|
||||
try:
|
||||
combined = f"User: {user_content[:2000]}\nAssistant: {assistant_content[:2000]}"
|
||||
_run_brv(
|
||||
["curate", "--", combined],
|
||||
timeout=_CURATE_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("ByteRover sync failed: %s", e)
|
||||
|
||||
# Wait for previous sync
|
||||
if self._sync_thread and self._sync_thread.is_alive():
|
||||
self._sync_thread.join(timeout=5.0)
|
||||
|
||||
self._sync_thread = threading.Thread(
|
||||
target=_sync, daemon=True, name="brv-sync"
|
||||
)
|
||||
self._sync_thread.start()
|
||||
|
||||
def on_memory_write(self, action: str, target: str, content: str) -> None:
|
||||
"""Mirror built-in memory writes to ByteRover."""
|
||||
if not self._auto_extract:
|
||||
logger.debug("ByteRover memory mirror skipped (auto_extract disabled)")
|
||||
return
|
||||
if action not in {"add", "replace"} or not content:
|
||||
return
|
||||
|
||||
def _write():
|
||||
try:
|
||||
label = "User profile" if target == "user" else "Agent memory"
|
||||
_run_brv(
|
||||
["curate", "--", f"[{label}] {content}"],
|
||||
timeout=_CURATE_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("ByteRover memory mirror failed: %s", e)
|
||||
|
||||
t = threading.Thread(target=_write, daemon=True, name="brv-memwrite")
|
||||
t.start()
|
||||
|
||||
def on_pre_compress(self, messages: List[Dict[str, Any]]) -> str:
|
||||
"""Extract insights before context compression discards turns."""
|
||||
if not self._auto_extract:
|
||||
logger.debug("ByteRover pre-compression flush skipped (auto_extract disabled)")
|
||||
return ""
|
||||
if not messages:
|
||||
return ""
|
||||
|
||||
# Build a summary of messages about to be compressed
|
||||
parts = []
|
||||
for msg in messages[-10:]: # last 10 messages
|
||||
role = msg.get("role", "")
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str) and content.strip() and role in {"user", "assistant"}:
|
||||
parts.append(f"{role}: {content[:500]}")
|
||||
|
||||
if not parts:
|
||||
return ""
|
||||
|
||||
combined = "\n".join(parts)
|
||||
|
||||
def _flush():
|
||||
try:
|
||||
_run_brv(
|
||||
["curate", "--", f"[Pre-compression context]\n{combined}"],
|
||||
timeout=_CURATE_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
logger.info("ByteRover pre-compression flush: %d messages", len(parts))
|
||||
except Exception as e:
|
||||
logger.debug("ByteRover pre-compression flush failed: %s", e)
|
||||
|
||||
t = threading.Thread(target=_flush, daemon=True, name="brv-flush")
|
||||
t.start()
|
||||
return ""
|
||||
|
||||
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
||||
return [QUERY_SCHEMA, CURATE_SCHEMA, STATUS_SCHEMA]
|
||||
|
||||
def handle_tool_call(self, tool_name: str, args: dict, **kwargs) -> str:
|
||||
if tool_name == "brv_query":
|
||||
return self._tool_query(args)
|
||||
elif tool_name == "brv_curate":
|
||||
return self._tool_curate(args)
|
||||
elif tool_name == "brv_status":
|
||||
return self._tool_status()
|
||||
return tool_error(f"Unknown tool: {tool_name}")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self._sync_thread and self._sync_thread.is_alive():
|
||||
self._sync_thread.join(timeout=10.0)
|
||||
|
||||
# -- Tool implementations ------------------------------------------------
|
||||
|
||||
def _tool_query(self, args: dict) -> str:
|
||||
query = args.get("query", "")
|
||||
if not query:
|
||||
return tool_error("query is required")
|
||||
|
||||
result = _run_brv(
|
||||
["query", "--", query.strip()[:5000]],
|
||||
timeout=_QUERY_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
|
||||
if not result["success"]:
|
||||
return tool_error(result.get("error", "Query failed"))
|
||||
|
||||
output = result.get("output", "").strip()
|
||||
if not output or len(output) < _MIN_OUTPUT_LEN:
|
||||
return json.dumps({"result": "No relevant memories found."})
|
||||
|
||||
# Truncate very long results
|
||||
if len(output) > 8000:
|
||||
output = output[:8000] + "\n\n[... truncated]"
|
||||
|
||||
return json.dumps({"result": output})
|
||||
|
||||
def _tool_curate(self, args: dict) -> str:
|
||||
content = args.get("content", "")
|
||||
if not content:
|
||||
return tool_error("content is required")
|
||||
|
||||
result = _run_brv(
|
||||
["curate", "--", content],
|
||||
timeout=_CURATE_TIMEOUT, cwd=self._cwd,
|
||||
)
|
||||
|
||||
if not result["success"]:
|
||||
return tool_error(result.get("error", "Curate failed"))
|
||||
|
||||
return json.dumps({"result": "Memory curated successfully."})
|
||||
|
||||
def _tool_status(self) -> str:
|
||||
result = _run_brv(["status"], timeout=15, cwd=self._cwd)
|
||||
if not result["success"]:
|
||||
return tool_error(result.get("error", "Status check failed"))
|
||||
return json.dumps({"status": result.get("output", "")})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register ByteRover as a memory provider plugin."""
|
||||
ctx.register_memory_provider(ByteRoverMemoryProvider())
|
||||
@@ -0,0 +1,9 @@
|
||||
name: byterover
|
||||
version: 1.0.0
|
||||
description: "ByteRover — persistent knowledge tree with tiered retrieval via the brv CLI."
|
||||
external_dependencies:
|
||||
- name: brv
|
||||
install: "curl -fsSL https://byterover.dev/install.sh | sh"
|
||||
check: "brv --version"
|
||||
hooks:
|
||||
- on_pre_compress
|
||||
@@ -0,0 +1,147 @@
|
||||
# Hindsight Memory Provider
|
||||
|
||||
Long-term memory with knowledge graph, entity resolution, and multi-strategy retrieval. Supports cloud, local embedded, and local external modes.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Cloud:** API key from [ui.hindsight.vectorize.io](https://ui.hindsight.vectorize.io)
|
||||
- **Local Embedded:** API key for a supported LLM provider (OpenAI, Anthropic, Gemini, Groq, OpenRouter, MiniMax, Ollama, or any OpenAI-compatible endpoint). Embeddings and reranking run locally — no additional API keys needed.
|
||||
- **Local External:** A running Hindsight instance (Docker or self-hosted) reachable over HTTP.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "hindsight"
|
||||
```
|
||||
|
||||
The setup wizard will install dependencies automatically via `uv` and walk you through configuration.
|
||||
|
||||
Or manually (cloud mode with defaults):
|
||||
```bash
|
||||
hermes config set memory.provider hindsight
|
||||
echo "HINDSIGHT_API_KEY=your-key" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
### Cloud
|
||||
|
||||
Connects to the Hindsight Cloud API. Requires an API key from [ui.hindsight.vectorize.io](https://ui.hindsight.vectorize.io).
|
||||
|
||||
### Local Embedded
|
||||
|
||||
Hermes spins up a local Hindsight daemon with built-in PostgreSQL. Requires an LLM API key for memory extraction and synthesis. The daemon starts automatically in the background on first use and stops after 5 minutes of inactivity.
|
||||
|
||||
Supports any OpenAI-compatible LLM endpoint (llama.cpp, vLLM, LM Studio, etc.) — pick `openai_compatible` as the provider and enter the base URL.
|
||||
|
||||
Daemon startup logs: `~/.hermes/logs/hindsight-embed.log`
|
||||
Daemon runtime logs: `~/.hindsight/profiles/<profile>.log`
|
||||
|
||||
To open the Hindsight web UI (local embedded mode only):
|
||||
```bash
|
||||
hindsight-embed -p hermes ui start
|
||||
```
|
||||
|
||||
### Local External
|
||||
|
||||
Points the plugin at an existing Hindsight instance you're already running (Docker, self-hosted, etc.). No daemon management — just a URL and an optional API key.
|
||||
|
||||
## Config
|
||||
|
||||
Config file: `~/.hermes/hindsight/config.json`
|
||||
|
||||
### Connection
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `mode` | `cloud` | `cloud`, `local_embedded`, or `local_external` |
|
||||
| `api_url` | `https://api.hindsight.vectorize.io` | API URL (cloud and local_external modes) |
|
||||
|
||||
### Memory Bank
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `bank_id` | `hermes` | Memory bank name (static fallback used when `bank_id_template` is unset or resolves empty) |
|
||||
| `bank_id_template` | — | Optional template to derive the bank name dynamically. Placeholders: `{profile}`, `{workspace}`, `{platform}`, `{user}`, `{session}`. Example: `hermes-{profile}` isolates memory per active Hermes profile. Empty placeholders collapse cleanly (e.g. `hermes-{user}` with no user becomes `hermes`). |
|
||||
| `bank_mission` | — | Reflect mission (identity/framing for reflect reasoning). Applied via Banks API. |
|
||||
| `bank_retain_mission` | — | Retain mission (steers what gets extracted). Applied via Banks API. |
|
||||
|
||||
### Recall
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `recall_budget` | `mid` | Recall thoroughness: `low` / `mid` / `high` |
|
||||
| `recall_prefetch_method` | `recall` | Auto-recall method: `recall` (raw facts) or `reflect` (LLM synthesis) |
|
||||
| `recall_max_tokens` | `4096` | Maximum tokens for recall results |
|
||||
| `recall_max_input_chars` | `800` | Maximum input query length for auto-recall |
|
||||
| `recall_prompt_preamble` | — | Custom preamble for recalled memories in context |
|
||||
| `recall_tags` | — | Tags to filter when searching memories |
|
||||
| `recall_tags_match` | `any` | Tag matching mode: `any` / `all` / `any_strict` / `all_strict` |
|
||||
| `recall_types` | `observation` | Fact types surfaced by recall (both auto-recall and the `hindsight_recall` tool). Comma-separated string or JSON list. **Default narrowed to `observation` only** (see "Behavior change" below). Set to `observation,world,experience` to also include raw facts. |
|
||||
| `auto_recall` | `true` | Automatically recall memories before each turn |
|
||||
|
||||
> **Behavior change — `recall_types` defaults to `observation` only.**
|
||||
>
|
||||
> Previously recall returned all three fact types. It now returns only observations.
|
||||
>
|
||||
> Per [Hindsight's docs](https://hindsight.vectorize.io/developer/observations), observations are the **consolidated** knowledge layer Hindsight builds on top of raw facts: deduplicated beliefs grounded in evidence, refined as new facts arrive, with proof counts and freshness signals. Raw `world` / `experience` facts are the individual supporting evidence that feeds them. For per-turn context injection, observations are denser per token and avoid feeding the model multiple raw facts that one observation already summarizes.
|
||||
>
|
||||
> Restore the broad recall with `"recall_types": "observation,world,experience"` (string or JSON list) in `~/.hermes/hindsight/config.json`. This applies to **both** auto-recall and the `hindsight_recall` tool — both read the same `recall_types` setting (the tool schema has no per-call `types` argument), so narrowing the default narrows both paths.
|
||||
|
||||
### Retain
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `auto_retain` | `true` | Automatically retain conversation turns |
|
||||
| `retain_async` | `true` | Process retain asynchronously on the Hindsight server |
|
||||
| `retain_every_n_turns` | `1` | Retain every N turns (1 = every turn) |
|
||||
| `retain_context` | `conversation between Hermes Agent and the User` | Context label for retained memories |
|
||||
| `retain_tags` | — | Default tags applied to retained memories; merged with per-call tool tags |
|
||||
| `retain_source` | — | Optional `metadata.source` attached to retained memories |
|
||||
| `retain_user_prefix` | `User` | Label used before user turns in auto-retained transcripts |
|
||||
| `retain_assistant_prefix` | `Assistant` | Label used before assistant turns in auto-retained transcripts |
|
||||
|
||||
### Integration
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `memory_mode` | `hybrid` | How memories are integrated into the agent |
|
||||
|
||||
**memory_mode:**
|
||||
- `hybrid` — automatic context injection + tools available to the LLM
|
||||
- `context` — automatic injection only, no tools exposed
|
||||
- `tools` — tools only, no automatic injection
|
||||
|
||||
### Local Embedded LLM
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `llm_provider` | `openai` | `openai`, `anthropic`, `gemini`, `groq`, `openrouter`, `minimax`, `ollama`, `lmstudio`, `openai_compatible` |
|
||||
| `llm_model` | per-provider | Model name (e.g. `gpt-4o-mini`, `qwen/qwen3.5-9b`) |
|
||||
| `llm_base_url` | — | Endpoint URL for `openai_compatible` (e.g. `http://192.168.1.10:8080/v1`) |
|
||||
|
||||
The LLM API key is stored in `~/.hermes/.env` as `HINDSIGHT_LLM_API_KEY`.
|
||||
|
||||
## Tools
|
||||
|
||||
Available in `hybrid` and `tools` memory modes:
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `hindsight_retain` | Store information with auto entity extraction; supports optional per-call `tags` |
|
||||
| `hindsight_recall` | Multi-strategy search (semantic + entity graph) |
|
||||
| `hindsight_reflect` | Cross-memory synthesis (LLM-powered) |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `HINDSIGHT_API_KEY` | API key for Hindsight Cloud |
|
||||
| `HINDSIGHT_LLM_API_KEY` | LLM API key for local mode |
|
||||
| `HINDSIGHT_API_LLM_BASE_URL` | LLM Base URL for local mode (e.g. OpenRouter) |
|
||||
| `HINDSIGHT_API_URL` | Override API endpoint |
|
||||
| `HINDSIGHT_BANK_ID` | Override bank name |
|
||||
| `HINDSIGHT_BUDGET` | Override recall budget |
|
||||
| `HINDSIGHT_MODE` | Override mode (`cloud`, `local_embedded`, `local_external`) |
|
||||
|
||||
## Client Version
|
||||
|
||||
Requires `hindsight-client >= 0.6.1`. The plugin auto-upgrades on session start if an older version is detected.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
name: hindsight
|
||||
version: 1.0.0
|
||||
description: "Hindsight — long-term memory with knowledge graph, entity resolution, and multi-strategy retrieval."
|
||||
pip_dependencies:
|
||||
- "hindsight-client>=0.6.1"
|
||||
requires_env: []
|
||||
hooks:
|
||||
- on_session_end
|
||||
@@ -0,0 +1,36 @@
|
||||
# Holographic Memory Provider
|
||||
|
||||
Local SQLite fact store with FTS5 search, trust scoring, entity resolution, and HRR-based compositional retrieval.
|
||||
|
||||
## Requirements
|
||||
|
||||
None — uses SQLite (always available). NumPy optional for HRR algebra.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "holographic"
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider holographic
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
Config in `config.yaml` under `plugins.hermes-memory-store`:
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `db_path` | `$HERMES_HOME/memory_store.db` | SQLite database path |
|
||||
| `auto_extract` | `false` | Auto-extract facts at session end |
|
||||
| `default_trust` | `0.5` | Default trust score for new facts |
|
||||
| `hrr_dim` | `1024` | HRR vector dimensions |
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `fact_store` | 9 actions: add, search, probe, related, reason, contradict, update, remove, list |
|
||||
| `fact_feedback` | Rate facts as helpful/unhelpful (trains trust scores) |
|
||||
@@ -0,0 +1,419 @@
|
||||
"""hermes-memory-store — holographic memory plugin using MemoryProvider interface.
|
||||
|
||||
Registers as a MemoryProvider plugin, giving the agent structured fact storage
|
||||
with entity resolution, trust scoring, and HRR-based compositional retrieval.
|
||||
|
||||
Original plugin by dusterbloom (PR #2351), adapted to the MemoryProvider ABC.
|
||||
|
||||
Config in $HERMES_HOME/config.yaml (profile-scoped):
|
||||
plugins:
|
||||
hermes-memory-store:
|
||||
db_path: $HERMES_HOME/memory_store.db # omit to use the default
|
||||
auto_extract: false
|
||||
default_trust: 0.5
|
||||
min_trust_threshold: 0.3
|
||||
temporal_decay_half_life: 0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from agent.memory_provider import MemoryProvider
|
||||
from tools.registry import tool_error
|
||||
from .store import MemoryStore
|
||||
from .retrieval import FactRetriever
|
||||
from hermes_cli.config import cfg_get
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas (unchanged from original PR)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FACT_STORE_SCHEMA = {
|
||||
"name": "fact_store",
|
||||
"description": (
|
||||
"Deep structured memory with algebraic reasoning. "
|
||||
"Use alongside the memory tool — memory for always-on context, "
|
||||
"fact_store for deep recall and compositional queries.\n\n"
|
||||
"ACTIONS (simple → powerful):\n"
|
||||
"• add — Store a fact the user would expect you to remember.\n"
|
||||
"• search — Keyword lookup ('editor config', 'deploy process').\n"
|
||||
"• probe — Entity recall: ALL facts about a person/thing.\n"
|
||||
"• related — What connects to an entity? Structural adjacency.\n"
|
||||
"• reason — Compositional: facts connected to MULTIPLE entities simultaneously.\n"
|
||||
"• contradict — Memory hygiene: find facts making conflicting claims.\n"
|
||||
"• update/remove/list — CRUD operations.\n\n"
|
||||
"IMPORTANT: Before answering questions about the user, ALWAYS probe or reason first."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"],
|
||||
},
|
||||
"content": {"type": "string", "description": "Fact content (required for 'add')."},
|
||||
"query": {"type": "string", "description": "Search query (required for 'search')."},
|
||||
"entity": {"type": "string", "description": "Entity name for 'probe'/'related'."},
|
||||
"entities": {"type": "array", "items": {"type": "string"}, "description": "Entity names for 'reason'."},
|
||||
"fact_id": {"type": "integer", "description": "Fact ID for 'update'/'remove'."},
|
||||
"category": {"type": "string", "enum": ["user_pref", "project", "tool", "general"]},
|
||||
"tags": {"type": "string", "description": "Comma-separated tags."},
|
||||
"trust_delta": {"type": "number", "description": "Trust adjustment for 'update'."},
|
||||
"min_trust": {"type": "number", "description": "Minimum trust filter (default: 0.3)."},
|
||||
"limit": {"type": "integer", "description": "Max results (default: 10)."},
|
||||
},
|
||||
"required": ["action"],
|
||||
},
|
||||
}
|
||||
|
||||
FACT_FEEDBACK_SCHEMA = {
|
||||
"name": "fact_feedback",
|
||||
"description": (
|
||||
"Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
|
||||
"This trains the memory — good facts rise, bad facts sink."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
|
||||
"fact_id": {"type": "integer", "description": "The fact ID to rate."},
|
||||
},
|
||||
"required": ["action", "fact_id"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_plugin_config() -> dict:
|
||||
from hermes_constants import get_hermes_home
|
||||
config_path = get_hermes_home() / "config.yaml"
|
||||
if not config_path.exists():
|
||||
return {}
|
||||
try:
|
||||
import yaml
|
||||
with open(config_path, encoding="utf-8-sig") as f:
|
||||
all_config = yaml.safe_load(f) or {}
|
||||
return cfg_get(all_config, "plugins", "hermes-memory-store", default={}) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryProvider implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class HolographicMemoryProvider(MemoryProvider):
|
||||
"""Holographic memory with structured facts, entity resolution, and HRR retrieval."""
|
||||
|
||||
def __init__(self, config: dict | None = None):
|
||||
self._config = config or _load_plugin_config()
|
||||
self._store = None
|
||||
self._retriever = None
|
||||
self._min_trust = float(self._config.get("min_trust_threshold", 0.3))
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "holographic"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True # SQLite is always available, numpy is optional
|
||||
|
||||
def save_config(self, values, hermes_home):
|
||||
"""Write config to config.yaml under plugins.hermes-memory-store."""
|
||||
from pathlib import Path
|
||||
config_path = Path(hermes_home) / "config.yaml"
|
||||
try:
|
||||
import yaml
|
||||
existing = {}
|
||||
if config_path.exists():
|
||||
with open(config_path, encoding="utf-8-sig") as f:
|
||||
existing = yaml.safe_load(f) or {}
|
||||
existing.setdefault("plugins", {})
|
||||
existing["plugins"]["hermes-memory-store"] = values
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(existing, f, default_flow_style=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_config_schema(self):
|
||||
from hermes_constants import display_hermes_home
|
||||
_default_db = f"{display_hermes_home()}/memory_store.db"
|
||||
return [
|
||||
{"key": "db_path", "description": "SQLite database path", "default": _default_db},
|
||||
{"key": "auto_extract", "description": "Auto-extract facts at session end", "default": "false", "choices": ["true", "false"]},
|
||||
{"key": "default_trust", "description": "Default trust score for new facts", "default": "0.5"},
|
||||
{"key": "hrr_dim", "description": "HRR vector dimensions", "default": "1024"},
|
||||
]
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
from hermes_constants import get_hermes_home
|
||||
_hermes_home = str(get_hermes_home())
|
||||
_default_db = _hermes_home + "/memory_store.db"
|
||||
db_path = self._config.get("db_path", _default_db)
|
||||
# Expand $HERMES_HOME in user-supplied paths so config values like
|
||||
# "$HERMES_HOME/memory_store.db" or "~/.hermes/memory_store.db" both
|
||||
# resolve to the active profile's directory.
|
||||
if isinstance(db_path, str):
|
||||
db_path = db_path.replace("$HERMES_HOME", _hermes_home)
|
||||
db_path = db_path.replace("${HERMES_HOME}", _hermes_home)
|
||||
default_trust = float(self._config.get("default_trust", 0.5))
|
||||
hrr_dim = int(self._config.get("hrr_dim", 1024))
|
||||
hrr_weight = float(self._config.get("hrr_weight", 0.3))
|
||||
temporal_decay = int(self._config.get("temporal_decay_half_life", 0))
|
||||
|
||||
self._store = MemoryStore(db_path=db_path, default_trust=default_trust, hrr_dim=hrr_dim)
|
||||
self._retriever = FactRetriever(
|
||||
store=self._store,
|
||||
temporal_decay_half_life=temporal_decay,
|
||||
hrr_weight=hrr_weight,
|
||||
hrr_dim=hrr_dim,
|
||||
)
|
||||
self._session_id = session_id
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
if not self._store:
|
||||
return ""
|
||||
try:
|
||||
total = self._store._conn.execute(
|
||||
"SELECT COUNT(*) FROM facts"
|
||||
).fetchone()[0]
|
||||
except Exception:
|
||||
total = 0
|
||||
if total == 0:
|
||||
return (
|
||||
"# Holographic Memory\n"
|
||||
"Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
|
||||
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
|
||||
"Use fact_feedback to rate facts after using them (trains trust scores)."
|
||||
)
|
||||
return (
|
||||
f"# Holographic Memory\n"
|
||||
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
|
||||
f"Use fact_store to search, probe entities, reason across entities, or add facts.\n"
|
||||
f"Use fact_feedback to rate facts after using them (trains trust scores)."
|
||||
)
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
if not self._retriever or not query:
|
||||
return ""
|
||||
try:
|
||||
results = self._retriever.search(query, min_trust=self._min_trust, limit=5)
|
||||
if not results:
|
||||
return ""
|
||||
lines = []
|
||||
for r in results:
|
||||
trust = r.get("trust_score", r.get("trust", 0))
|
||||
lines.append(f"- [{trust:.1f}] {r.get('content', '')}")
|
||||
return "## Holographic Memory\n" + "\n".join(lines)
|
||||
except Exception as e:
|
||||
logger.debug("Holographic prefetch failed: %s", e)
|
||||
return ""
|
||||
|
||||
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
||||
# Holographic memory stores explicit facts via tools, not auto-sync.
|
||||
# The on_session_end hook handles auto-extraction if configured.
|
||||
pass
|
||||
|
||||
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
||||
return [FACT_STORE_SCHEMA, FACT_FEEDBACK_SCHEMA]
|
||||
|
||||
def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str:
|
||||
if tool_name == "fact_store":
|
||||
return self._handle_fact_store(args)
|
||||
elif tool_name == "fact_feedback":
|
||||
return self._handle_fact_feedback(args)
|
||||
return tool_error(f"Unknown tool: {tool_name}")
|
||||
|
||||
def on_session_end(self, messages: List[Dict[str, Any]]) -> None:
|
||||
if not self._config.get("auto_extract", False):
|
||||
return
|
||||
if not self._store or not messages:
|
||||
return
|
||||
self._auto_extract_facts(messages)
|
||||
|
||||
def on_memory_write(self, action: str, target: str, content: str) -> None:
|
||||
"""Mirror built-in memory writes as facts."""
|
||||
if action == "add" and self._store and content:
|
||||
try:
|
||||
category = "user_pref" if target == "user" else "general"
|
||||
self._store.add_fact(content, category=category)
|
||||
except Exception as e:
|
||||
logger.debug("Holographic memory_write mirror failed: %s", e)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Release the shared SQLite connection deterministically on the
|
||||
# caller's thread. Dropping the reference alone leaves fd finalization
|
||||
# to GC, which keeps the connection (and its write lock) alive on a
|
||||
# long-running gateway and prolongs the "database is locked" contention
|
||||
# this store's shared-connection refcounting is meant to eliminate.
|
||||
# close() is idempotent and refcount-guarded, so siblings stay safe.
|
||||
if self._store is not None:
|
||||
try:
|
||||
self._store.close()
|
||||
except Exception as e:
|
||||
logger.debug("Holographic shutdown close() failed: %s", e)
|
||||
self._store = None
|
||||
self._retriever = None
|
||||
|
||||
# -- Tool handlers -------------------------------------------------------
|
||||
|
||||
def _handle_fact_store(self, args: dict) -> str:
|
||||
try:
|
||||
action = args["action"]
|
||||
store = self._store
|
||||
retriever = self._retriever
|
||||
|
||||
if action == "add":
|
||||
fact_id = store.add_fact(
|
||||
args["content"],
|
||||
category=args.get("category", "general"),
|
||||
tags=args.get("tags", ""),
|
||||
)
|
||||
return json.dumps({"fact_id": fact_id, "status": "added"})
|
||||
|
||||
elif action == "search":
|
||||
results = retriever.search(
|
||||
args["query"],
|
||||
category=args.get("category"),
|
||||
min_trust=float(args.get("min_trust", self._min_trust)),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)})
|
||||
|
||||
elif action == "probe":
|
||||
results = retriever.probe(
|
||||
args["entity"],
|
||||
category=args.get("category"),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)})
|
||||
|
||||
elif action == "related":
|
||||
results = retriever.related(
|
||||
args["entity"],
|
||||
category=args.get("category"),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)})
|
||||
|
||||
elif action == "reason":
|
||||
entities = args.get("entities", [])
|
||||
if not entities:
|
||||
return tool_error("reason requires 'entities' list")
|
||||
results = retriever.reason(
|
||||
entities,
|
||||
category=args.get("category"),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)})
|
||||
|
||||
elif action == "contradict":
|
||||
results = retriever.contradict(
|
||||
category=args.get("category"),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)})
|
||||
|
||||
elif action == "update":
|
||||
updated = store.update_fact(
|
||||
int(args["fact_id"]),
|
||||
content=args.get("content"),
|
||||
trust_delta=float(args["trust_delta"]) if "trust_delta" in args else None,
|
||||
tags=args.get("tags"),
|
||||
category=args.get("category"),
|
||||
)
|
||||
return json.dumps({"updated": updated})
|
||||
|
||||
elif action == "remove":
|
||||
removed = store.remove_fact(int(args["fact_id"]))
|
||||
return json.dumps({"removed": removed})
|
||||
|
||||
elif action == "list":
|
||||
facts = store.list_facts(
|
||||
category=args.get("category"),
|
||||
min_trust=float(args.get("min_trust", 0.0)),
|
||||
limit=int(args.get("limit", 10)),
|
||||
)
|
||||
return json.dumps({"facts": facts, "count": len(facts)})
|
||||
|
||||
else:
|
||||
return tool_error(f"Unknown action: {action}")
|
||||
|
||||
except KeyError as exc:
|
||||
return tool_error(f"Missing required argument: {exc}")
|
||||
except Exception as exc:
|
||||
return tool_error(str(exc))
|
||||
|
||||
def _handle_fact_feedback(self, args: dict) -> str:
|
||||
try:
|
||||
fact_id = int(args["fact_id"])
|
||||
helpful = args["action"] == "helpful"
|
||||
result = self._store.record_feedback(fact_id, helpful=helpful)
|
||||
return json.dumps(result)
|
||||
except KeyError as exc:
|
||||
return tool_error(f"Missing required argument: {exc}")
|
||||
except Exception as exc:
|
||||
return tool_error(str(exc))
|
||||
|
||||
# -- Auto-extraction (on_session_end) ------------------------------------
|
||||
|
||||
def _auto_extract_facts(self, messages: list) -> None:
|
||||
_PREF_PATTERNS = [
|
||||
re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE),
|
||||
]
|
||||
_DECISION_PATTERNS = [
|
||||
re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE),
|
||||
]
|
||||
|
||||
extracted = 0
|
||||
for msg in messages:
|
||||
if msg.get("role") != "user":
|
||||
continue
|
||||
content = msg.get("content", "")
|
||||
if not isinstance(content, str) or len(content) < 10:
|
||||
continue
|
||||
|
||||
for pattern in _PREF_PATTERNS:
|
||||
if pattern.search(content):
|
||||
try:
|
||||
self._store.add_fact(content[:400], category="user_pref")
|
||||
extracted += 1
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
for pattern in _DECISION_PATTERNS:
|
||||
if pattern.search(content):
|
||||
try:
|
||||
self._store.add_fact(content[:400], category="project")
|
||||
extracted += 1
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
if extracted:
|
||||
logger.info("Auto-extracted %d facts from conversation", extracted)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register the holographic memory provider with the plugin system."""
|
||||
config = _load_plugin_config()
|
||||
provider = HolographicMemoryProvider(config=config)
|
||||
ctx.register_memory_provider(provider)
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Holographic Reduced Representations (HRR) with phase encoding.
|
||||
|
||||
HRRs are a vector symbolic architecture for encoding compositional structure
|
||||
into fixed-width distributed representations. This module uses *phase vectors*:
|
||||
each concept is a vector of angles in [0, 2π). The algebraic operations are:
|
||||
|
||||
bind — circular convolution (phase addition) — associates two concepts
|
||||
unbind — circular correlation (phase subtraction) — retrieves a bound value
|
||||
bundle — superposition (circular mean) — merges multiple concepts
|
||||
|
||||
Phase encoding is numerically stable, avoids the magnitude collapse of
|
||||
traditional complex-number HRRs, and maps cleanly to cosine similarity.
|
||||
|
||||
Atoms are generated deterministically from SHA-256 so representations are
|
||||
identical across processes, machines, and language versions.
|
||||
|
||||
References:
|
||||
Plate (1995) — Holographic Reduced Representations
|
||||
Gayler (2004) — Vector Symbolic Architectures answer Jackendoff's challenges
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import struct
|
||||
import math
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
_HAS_NUMPY = True
|
||||
except ImportError:
|
||||
_HAS_NUMPY = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TWO_PI = 2.0 * math.pi
|
||||
|
||||
|
||||
def _require_numpy() -> None:
|
||||
if not _HAS_NUMPY:
|
||||
raise RuntimeError("numpy is required for holographic operations")
|
||||
|
||||
|
||||
def encode_atom(word: str, dim: int = 1024) -> "np.ndarray":
|
||||
"""Deterministic phase vector via SHA-256 counter blocks.
|
||||
|
||||
Uses hashlib (not numpy RNG) for cross-platform reproducibility.
|
||||
|
||||
Algorithm:
|
||||
- Generate enough SHA-256 blocks by hashing f"{word}:{i}" for i=0,1,2,...
|
||||
- Concatenate digests, interpret as uint16 values via struct.unpack
|
||||
- Scale to [0, 2π): phases = values * (2π / 65536)
|
||||
- Truncate to dim elements
|
||||
- Returns np.float64 array of shape (dim,)
|
||||
"""
|
||||
_require_numpy()
|
||||
|
||||
# Each SHA-256 digest is 32 bytes = 16 uint16 values.
|
||||
values_per_block = 16
|
||||
blocks_needed = math.ceil(dim / values_per_block)
|
||||
|
||||
uint16_values: list[int] = []
|
||||
for i in range(blocks_needed):
|
||||
digest = hashlib.sha256(f"{word}:{i}".encode()).digest()
|
||||
uint16_values.extend(struct.unpack("<16H", digest))
|
||||
|
||||
phases = np.array(uint16_values[:dim], dtype=np.float64) * (_TWO_PI / 65536.0)
|
||||
return phases
|
||||
|
||||
|
||||
def bind(a: "np.ndarray", b: "np.ndarray") -> "np.ndarray":
|
||||
"""Circular convolution = element-wise phase addition.
|
||||
|
||||
Binding associates two concepts into a single composite vector.
|
||||
The result is dissimilar to both inputs (quasi-orthogonal).
|
||||
"""
|
||||
_require_numpy()
|
||||
return (a + b) % _TWO_PI
|
||||
|
||||
|
||||
def unbind(memory: "np.ndarray", key: "np.ndarray") -> "np.ndarray":
|
||||
"""Circular correlation = element-wise phase subtraction.
|
||||
|
||||
Unbinding retrieves the value associated with a key from a memory vector.
|
||||
unbind(bind(a, b), a) ≈ b (up to superposition noise)
|
||||
"""
|
||||
_require_numpy()
|
||||
return (memory - key) % _TWO_PI
|
||||
|
||||
|
||||
def bundle(*vectors: "np.ndarray") -> "np.ndarray":
|
||||
"""Superposition via circular mean of complex exponentials.
|
||||
|
||||
Bundling merges multiple vectors into one that is similar to each input.
|
||||
The result can hold O(sqrt(dim)) items before similarity degrades.
|
||||
"""
|
||||
_require_numpy()
|
||||
complex_sum = np.sum([np.exp(1j * v) for v in vectors], axis=0)
|
||||
return np.angle(complex_sum) % _TWO_PI
|
||||
|
||||
|
||||
def similarity(a: "np.ndarray", b: "np.ndarray") -> float:
|
||||
"""Phase cosine similarity. Range [-1, 1].
|
||||
|
||||
Returns 1.0 for identical vectors, near 0.0 for random (unrelated) vectors,
|
||||
and -1.0 for perfectly anti-correlated vectors.
|
||||
"""
|
||||
_require_numpy()
|
||||
return float(np.mean(np.cos(a - b)))
|
||||
|
||||
|
||||
def encode_text(text: str, dim: int = 1024) -> "np.ndarray":
|
||||
"""Bag-of-words: bundle of atom vectors for each token.
|
||||
|
||||
Tokenizes by lowercasing, splitting on whitespace, and stripping
|
||||
leading/trailing punctuation from each token.
|
||||
|
||||
Returns bundle of all token atom vectors.
|
||||
If text is empty or produces no tokens, returns encode_atom("__hrr_empty__", dim).
|
||||
"""
|
||||
_require_numpy()
|
||||
|
||||
tokens = [
|
||||
token.strip(".,!?;:\"'()[]{}")
|
||||
for token in text.lower().split()
|
||||
]
|
||||
tokens = [t for t in tokens if t]
|
||||
|
||||
if not tokens:
|
||||
return encode_atom("__hrr_empty__", dim)
|
||||
|
||||
atom_vectors = [encode_atom(token, dim) for token in tokens]
|
||||
return bundle(*atom_vectors)
|
||||
|
||||
|
||||
def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarray":
|
||||
"""Structured encoding: content bound to ROLE_CONTENT, each entity bound to ROLE_ENTITY, all bundled.
|
||||
|
||||
Role vectors are reserved atoms: "__hrr_role_content__", "__hrr_role_entity__"
|
||||
|
||||
Components:
|
||||
1. bind(encode_text(content, dim), encode_atom("__hrr_role_content__", dim))
|
||||
2. For each entity: bind(encode_atom(entity.lower(), dim), encode_atom("__hrr_role_entity__", dim))
|
||||
3. bundle all components together
|
||||
|
||||
This enables algebraic extraction:
|
||||
unbind(fact, bind(entity, ROLE_ENTITY)) ≈ content_vector
|
||||
"""
|
||||
_require_numpy()
|
||||
|
||||
role_content = encode_atom("__hrr_role_content__", dim)
|
||||
role_entity = encode_atom("__hrr_role_entity__", dim)
|
||||
|
||||
components: list[np.ndarray] = [
|
||||
bind(encode_text(content, dim), role_content)
|
||||
]
|
||||
|
||||
for entity in entities:
|
||||
components.append(bind(encode_atom(entity.lower(), dim), role_entity))
|
||||
|
||||
return bundle(*components)
|
||||
|
||||
|
||||
def phases_to_bytes(phases: "np.ndarray") -> bytes:
|
||||
"""Serialize phase vector to bytes. float64 tobytes — 8 KB at dim=1024."""
|
||||
_require_numpy()
|
||||
return phases.tobytes()
|
||||
|
||||
|
||||
def bytes_to_phases(data: bytes) -> "np.ndarray":
|
||||
"""Deserialize bytes back to phase vector. Inverse of phases_to_bytes.
|
||||
|
||||
The .copy() call is required because frombuffer returns a read-only view
|
||||
backed by the bytes object; callers expect a mutable array.
|
||||
"""
|
||||
_require_numpy()
|
||||
return np.frombuffer(data, dtype=np.float64).copy()
|
||||
|
||||
|
||||
def snr_estimate(dim: int, n_items: int) -> float:
|
||||
"""Signal-to-noise ratio estimate for holographic storage.
|
||||
|
||||
SNR = sqrt(dim / n_items) when n_items > 0, else inf.
|
||||
|
||||
The SNR falls below 2.0 when n_items > dim / 4, meaning retrieval
|
||||
errors become likely. Logs a warning when this threshold is crossed.
|
||||
"""
|
||||
_require_numpy()
|
||||
|
||||
if n_items <= 0:
|
||||
return float("inf")
|
||||
|
||||
snr = math.sqrt(dim / n_items)
|
||||
|
||||
if snr < 2.0:
|
||||
logger.warning(
|
||||
"HRR storage near capacity: SNR=%.2f (dim=%d, n_items=%d). "
|
||||
"Retrieval accuracy may degrade. Consider increasing dim or reducing stored items.",
|
||||
snr,
|
||||
dim,
|
||||
n_items,
|
||||
)
|
||||
|
||||
return snr
|
||||
@@ -0,0 +1,5 @@
|
||||
name: holographic
|
||||
version: 0.1.0
|
||||
description: "Holographic memory — local SQLite fact store with FTS5 search, trust scoring, and HRR-based compositional retrieval."
|
||||
hooks:
|
||||
- on_session_end
|
||||
@@ -0,0 +1,654 @@
|
||||
"""Hybrid keyword/BM25 retrieval for the memory store.
|
||||
|
||||
Ported from KIK memory_agent.py — combines FTS5 full-text search with
|
||||
Jaccard similarity reranking and trust-weighted scoring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .store import MemoryStore
|
||||
|
||||
try:
|
||||
from . import holographic as hrr
|
||||
except ImportError:
|
||||
import holographic as hrr # type: ignore[no-redef]
|
||||
|
||||
|
||||
class FactRetriever:
|
||||
"""Multi-strategy fact retrieval with trust-weighted scoring."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: MemoryStore,
|
||||
temporal_decay_half_life: int = 0, # days, 0 = disabled
|
||||
fts_weight: float = 0.4,
|
||||
jaccard_weight: float = 0.3,
|
||||
hrr_weight: float = 0.3,
|
||||
hrr_dim: int = 1024,
|
||||
):
|
||||
self.store = store
|
||||
self.half_life = temporal_decay_half_life
|
||||
self.hrr_dim = hrr_dim
|
||||
|
||||
# Auto-redistribute weights if numpy unavailable
|
||||
if hrr_weight > 0 and not hrr._HAS_NUMPY:
|
||||
fts_weight = 0.6
|
||||
jaccard_weight = 0.4
|
||||
hrr_weight = 0.0
|
||||
|
||||
self.fts_weight = fts_weight
|
||||
self.jaccard_weight = jaccard_weight
|
||||
self.hrr_weight = hrr_weight
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
category: str | None = None,
|
||||
min_trust: float = 0.3,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Hybrid search: FTS5 candidates → Jaccard rerank → trust weighting.
|
||||
|
||||
Pipeline:
|
||||
1. FTS5 search: Get limit*3 candidates from SQLite full-text search
|
||||
2. Jaccard boost: Token overlap between query and fact content
|
||||
3. Trust weighting: final_score = relevance * trust_score
|
||||
4. Temporal decay (optional): decay = 0.5^(age_days / half_life)
|
||||
|
||||
Returns list of dicts with fact data + 'score' field, sorted by score desc.
|
||||
"""
|
||||
# Stage 1: Get FTS5 candidates (more than limit for reranking headroom)
|
||||
candidates = self._fts_candidates(query, category, min_trust, limit * 3)
|
||||
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
# Stage 2: Rerank with Jaccard + trust + optional decay
|
||||
query_tokens = self._tokenize(query)
|
||||
scored = []
|
||||
|
||||
for fact in candidates:
|
||||
content_tokens = self._tokenize(fact["content"])
|
||||
tag_tokens = self._tokenize(fact.get("tags", ""))
|
||||
all_tokens = content_tokens | tag_tokens
|
||||
|
||||
jaccard = self._jaccard_similarity(query_tokens, all_tokens)
|
||||
fts_score = fact.get("fts_rank", 0.0)
|
||||
|
||||
# HRR similarity
|
||||
if self.hrr_weight > 0 and fact.get("hrr_vector"):
|
||||
fact_vec = hrr.bytes_to_phases(fact["hrr_vector"])
|
||||
query_vec = hrr.encode_text(query, self.hrr_dim)
|
||||
hrr_sim = (hrr.similarity(query_vec, fact_vec) + 1.0) / 2.0 # shift to [0,1]
|
||||
else:
|
||||
hrr_sim = 0.5 # neutral
|
||||
|
||||
# Combine FTS5 + Jaccard + HRR
|
||||
relevance = (self.fts_weight * fts_score
|
||||
+ self.jaccard_weight * jaccard
|
||||
+ self.hrr_weight * hrr_sim)
|
||||
|
||||
# Trust weighting
|
||||
score = relevance * fact["trust_score"]
|
||||
|
||||
# Optional temporal decay
|
||||
if self.half_life > 0:
|
||||
score *= self._temporal_decay(fact.get("updated_at") or fact.get("created_at"))
|
||||
|
||||
fact["score"] = score
|
||||
scored.append(fact)
|
||||
|
||||
# Sort by score descending, return top limit
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
results = scored[:limit]
|
||||
# Strip raw HRR bytes — callers expect JSON-serializable dicts
|
||||
for fact in results:
|
||||
fact.pop("hrr_vector", None)
|
||||
return results
|
||||
|
||||
def probe(
|
||||
self,
|
||||
entity: str,
|
||||
category: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Compositional entity query using HRR algebra.
|
||||
|
||||
Unbinds entity from memory bank to extract associated content.
|
||||
This is NOT keyword search — it uses algebraic structure to find facts
|
||||
where the entity plays a structural role.
|
||||
|
||||
Falls back to FTS5 search if numpy unavailable.
|
||||
"""
|
||||
if not hrr._HAS_NUMPY:
|
||||
# Fallback to keyword search on entity name
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
|
||||
conn = self.store._conn
|
||||
|
||||
# Encode entity as role-bound vector
|
||||
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
|
||||
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
|
||||
probe_key = hrr.bind(entity_vec, role_entity)
|
||||
|
||||
# Try category-specific bank first, then all facts
|
||||
if category:
|
||||
bank_name = f"cat:{category}"
|
||||
bank_row = conn.execute(
|
||||
"SELECT vector FROM memory_banks WHERE bank_name = ?",
|
||||
(bank_name,),
|
||||
).fetchone()
|
||||
if bank_row:
|
||||
bank_vec = hrr.bytes_to_phases(bank_row["vector"])
|
||||
extracted = hrr.unbind(bank_vec, probe_key)
|
||||
# Use extracted signal to score individual facts
|
||||
return self._score_facts_by_vector(
|
||||
extracted, category=category, limit=limit
|
||||
)
|
||||
|
||||
# Score against individual fact vectors directly
|
||||
where = "WHERE hrr_vector IS NOT NULL"
|
||||
params: list = []
|
||||
if category:
|
||||
where += " AND category = ?"
|
||||
params.append(category)
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT fact_id, content, category, tags, trust_score,
|
||||
retrieval_count, helpful_count, created_at, updated_at,
|
||||
hrr_vector
|
||||
FROM facts
|
||||
{where}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
# Final fallback: keyword search
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
|
||||
scored = []
|
||||
for row in rows:
|
||||
fact = dict(row)
|
||||
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"))
|
||||
# Unbind probe key from fact to see if entity is structurally present
|
||||
residual = hrr.unbind(fact_vec, probe_key)
|
||||
# Compare residual against content signal
|
||||
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
|
||||
content_vec = hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content)
|
||||
sim = hrr.similarity(residual, content_vec)
|
||||
fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"]
|
||||
scored.append(fact)
|
||||
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def related(
|
||||
self,
|
||||
entity: str,
|
||||
category: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Discover facts that share structural connections with an entity.
|
||||
|
||||
Unlike probe (which finds facts *about* an entity), related finds
|
||||
facts that are connected through shared context — e.g., other entities
|
||||
mentioned alongside this one, or content that overlaps structurally.
|
||||
|
||||
Falls back to FTS5 search if numpy unavailable.
|
||||
"""
|
||||
if not hrr._HAS_NUMPY:
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
|
||||
conn = self.store._conn
|
||||
|
||||
# Encode entity as a bare atom (not role-bound — we want ANY structural match)
|
||||
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
|
||||
|
||||
# Get all facts with vectors
|
||||
where = "WHERE hrr_vector IS NOT NULL"
|
||||
params: list = []
|
||||
if category:
|
||||
where += " AND category = ?"
|
||||
params.append(category)
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT fact_id, content, category, tags, trust_score,
|
||||
retrieval_count, helpful_count, created_at, updated_at,
|
||||
hrr_vector
|
||||
FROM facts
|
||||
{where}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
|
||||
# Score each fact by how much the entity's atom appears in its vector
|
||||
# This catches both role-bound entity matches AND content word matches
|
||||
scored = []
|
||||
for row in rows:
|
||||
fact = dict(row)
|
||||
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"))
|
||||
|
||||
# Check structural similarity: unbind entity from fact
|
||||
residual = hrr.unbind(fact_vec, entity_vec)
|
||||
# A high-similarity residual to ANY known role vector means this entity
|
||||
# plays a structural role in the fact
|
||||
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
|
||||
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
|
||||
|
||||
entity_role_sim = hrr.similarity(residual, role_entity)
|
||||
content_role_sim = hrr.similarity(residual, role_content)
|
||||
# Take the max — entity could appear in either role
|
||||
best_sim = max(entity_role_sim, content_role_sim)
|
||||
|
||||
fact["score"] = (best_sim + 1.0) / 2.0 * fact["trust_score"]
|
||||
scored.append(fact)
|
||||
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def reason(
|
||||
self,
|
||||
entities: list[str],
|
||||
category: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Multi-entity compositional query — vector-space JOIN.
|
||||
|
||||
Given multiple entities, algebraically intersects their structural
|
||||
connections to find facts related to ALL of them simultaneously.
|
||||
This is compositional reasoning that no embedding DB can do.
|
||||
|
||||
Example: reason(["peppi", "backend"]) finds facts where peppi AND
|
||||
backend both play structural roles — without keyword matching.
|
||||
|
||||
Falls back to FTS5 search if numpy unavailable.
|
||||
"""
|
||||
if not hrr._HAS_NUMPY or not entities:
|
||||
# Fallback: search with all entities as keywords
|
||||
query = " ".join(entities)
|
||||
return self.search(query, category=category, limit=limit)
|
||||
|
||||
conn = self.store._conn
|
||||
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
|
||||
|
||||
# For each entity, compute what the bank "remembers" about it
|
||||
# by unbinding entity+role from each fact vector
|
||||
entity_residuals = []
|
||||
for entity in entities:
|
||||
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
|
||||
probe_key = hrr.bind(entity_vec, role_entity)
|
||||
entity_residuals.append(probe_key)
|
||||
|
||||
# Get all facts with vectors
|
||||
where = "WHERE hrr_vector IS NOT NULL"
|
||||
params: list = []
|
||||
if category:
|
||||
where += " AND category = ?"
|
||||
params.append(category)
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT fact_id, content, category, tags, trust_score,
|
||||
retrieval_count, helpful_count, created_at, updated_at,
|
||||
hrr_vector
|
||||
FROM facts
|
||||
{where}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
query = " ".join(entities)
|
||||
return self.search(query, category=category, limit=limit)
|
||||
|
||||
# Score each fact by how much EACH entity is structurally present.
|
||||
# A fact scores high only if ALL entities have structural presence
|
||||
# (AND semantics via min, vs OR which would use mean/max).
|
||||
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
|
||||
|
||||
scored = []
|
||||
for row in rows:
|
||||
fact = dict(row)
|
||||
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"))
|
||||
|
||||
entity_scores = []
|
||||
for probe_key in entity_residuals:
|
||||
residual = hrr.unbind(fact_vec, probe_key)
|
||||
sim = hrr.similarity(residual, role_content)
|
||||
entity_scores.append(sim)
|
||||
|
||||
min_sim = min(entity_scores)
|
||||
fact["score"] = (min_sim + 1.0) / 2.0 * fact["trust_score"]
|
||||
scored.append(fact)
|
||||
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def contradict(
|
||||
self,
|
||||
category: str | None = None,
|
||||
threshold: float = 0.3,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Find potentially contradictory facts via entity overlap + content divergence.
|
||||
|
||||
Two facts contradict when they share entities (same subject) but have
|
||||
low content-vector similarity (different claims). This is automated
|
||||
memory hygiene — no other memory system does this.
|
||||
|
||||
Returns pairs of facts with a contradiction score.
|
||||
Falls back to empty list if numpy unavailable.
|
||||
"""
|
||||
if not hrr._HAS_NUMPY:
|
||||
return []
|
||||
|
||||
conn = self.store._conn
|
||||
|
||||
# Get all facts with vectors and their linked entities
|
||||
where = "WHERE f.hrr_vector IS NOT NULL"
|
||||
params: list = []
|
||||
if category:
|
||||
where += " AND f.category = ?"
|
||||
params.append(category)
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT f.fact_id, f.content, f.category, f.tags, f.trust_score,
|
||||
f.created_at, f.updated_at, f.hrr_vector
|
||||
FROM facts f
|
||||
{where}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
if len(rows) < 2:
|
||||
return []
|
||||
|
||||
# Guard against O(n²) explosion on large fact stores.
|
||||
# At 500 facts, that's ~125K comparisons — acceptable.
|
||||
# Above that, only check the most recently updated facts.
|
||||
_MAX_CONTRADICT_FACTS = 500
|
||||
if len(rows) > _MAX_CONTRADICT_FACTS:
|
||||
rows = sorted(rows, key=lambda r: r["updated_at"] or r["created_at"], reverse=True)
|
||||
rows = rows[:_MAX_CONTRADICT_FACTS]
|
||||
|
||||
# Build entity sets per fact
|
||||
fact_entities: dict[int, set[str]] = {}
|
||||
for row in rows:
|
||||
fid = row["fact_id"]
|
||||
entity_rows = conn.execute(
|
||||
"""
|
||||
SELECT e.name FROM entities e
|
||||
JOIN fact_entities fe ON fe.entity_id = e.entity_id
|
||||
WHERE fe.fact_id = ?
|
||||
""",
|
||||
(fid,),
|
||||
).fetchall()
|
||||
fact_entities[fid] = {r["name"].lower() for r in entity_rows}
|
||||
|
||||
# Compare all pairs: high entity overlap + low content similarity = contradiction
|
||||
facts = [dict(r) for r in rows]
|
||||
contradictions = []
|
||||
|
||||
for i in range(len(facts)):
|
||||
for j in range(i + 1, len(facts)):
|
||||
f1, f2 = facts[i], facts[j]
|
||||
ents1 = fact_entities.get(f1["fact_id"], set())
|
||||
ents2 = fact_entities.get(f2["fact_id"], set())
|
||||
|
||||
if not ents1 or not ents2:
|
||||
continue
|
||||
|
||||
# Entity overlap (Jaccard)
|
||||
entity_overlap = len(ents1 & ents2) / len(ents1 | ents2) if (ents1 | ents2) else 0.0
|
||||
|
||||
if entity_overlap < 0.3:
|
||||
continue # Not enough entity overlap to be contradictory
|
||||
|
||||
# Content similarity via HRR vectors
|
||||
v1 = hrr.bytes_to_phases(f1["hrr_vector"])
|
||||
v2 = hrr.bytes_to_phases(f2["hrr_vector"])
|
||||
content_sim = hrr.similarity(v1, v2)
|
||||
|
||||
# High entity overlap + low content similarity = potential contradiction
|
||||
# contradiction_score: higher = more contradictory
|
||||
contradiction_score = entity_overlap * (1.0 - (content_sim + 1.0) / 2.0)
|
||||
|
||||
if contradiction_score >= threshold:
|
||||
# Strip hrr_vector from output (not JSON serializable)
|
||||
f1_clean = {k: v for k, v in f1.items() if k != "hrr_vector"}
|
||||
f2_clean = {k: v for k, v in f2.items() if k != "hrr_vector"}
|
||||
contradictions.append({
|
||||
"fact_a": f1_clean,
|
||||
"fact_b": f2_clean,
|
||||
"entity_overlap": round(entity_overlap, 3),
|
||||
"content_similarity": round(content_sim, 3),
|
||||
"contradiction_score": round(contradiction_score, 3),
|
||||
"shared_entities": sorted(ents1 & ents2),
|
||||
})
|
||||
|
||||
contradictions.sort(key=lambda x: x["contradiction_score"], reverse=True)
|
||||
return contradictions[:limit]
|
||||
|
||||
def _score_facts_by_vector(
|
||||
self,
|
||||
target_vec: "np.ndarray",
|
||||
category: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Score facts by similarity to a target vector."""
|
||||
conn = self.store._conn
|
||||
|
||||
where = "WHERE hrr_vector IS NOT NULL"
|
||||
params: list = []
|
||||
if category:
|
||||
where += " AND category = ?"
|
||||
params.append(category)
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT fact_id, content, category, tags, trust_score,
|
||||
retrieval_count, helpful_count, created_at, updated_at,
|
||||
hrr_vector
|
||||
FROM facts
|
||||
{where}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
scored = []
|
||||
for row in rows:
|
||||
fact = dict(row)
|
||||
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"))
|
||||
sim = hrr.similarity(target_vec, fact_vec)
|
||||
fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"]
|
||||
scored.append(fact)
|
||||
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def _fts_candidates(
|
||||
self,
|
||||
query: str,
|
||||
category: str | None,
|
||||
min_trust: float,
|
||||
limit: int,
|
||||
) -> list[dict]:
|
||||
"""Get raw FTS5 candidates from the store.
|
||||
|
||||
Uses the store's database connection directly for FTS5 MATCH
|
||||
with rank scoring. Normalizes FTS5 rank to [0, 1] range.
|
||||
"""
|
||||
conn = self.store._conn
|
||||
|
||||
# Build query - FTS5 rank is negative (lower = better match)
|
||||
# We need to join facts_fts with facts to get all columns
|
||||
params: list = []
|
||||
where_clauses = ["facts_fts MATCH ?"]
|
||||
# FTS5 defaults to AND-between-tokens, which kills recall on
|
||||
# natural-language queries ("what happened with the deployment
|
||||
# rollback"). Sanitize: drop stopwords, OR-join content tokens, so
|
||||
# any significant term can match.
|
||||
params.append(self._sanitize_fts_query(query))
|
||||
|
||||
if category:
|
||||
where_clauses.append("f.category = ?")
|
||||
params.append(category)
|
||||
|
||||
where_clauses.append("f.trust_score >= ?")
|
||||
params.append(min_trust)
|
||||
|
||||
where_sql = " AND ".join(where_clauses)
|
||||
|
||||
sql = f"""
|
||||
SELECT f.*, facts_fts.rank as fts_rank_raw
|
||||
FROM facts_fts
|
||||
JOIN facts f ON f.fact_id = facts_fts.rowid
|
||||
WHERE {where_sql}
|
||||
ORDER BY facts_fts.rank
|
||||
LIMIT ?
|
||||
"""
|
||||
params.append(limit)
|
||||
|
||||
try:
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
except Exception:
|
||||
# FTS5 MATCH can fail on malformed queries — fall back to empty
|
||||
return []
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# Normalize FTS5 rank: rank is negative, lower = better
|
||||
# Convert to positive score in [0, 1] range
|
||||
raw_ranks = [abs(row["fts_rank_raw"]) for row in rows]
|
||||
max_rank = max(raw_ranks) if raw_ranks else 1.0
|
||||
max_rank = max(max_rank, 1e-6) # avoid div by zero
|
||||
|
||||
results = []
|
||||
for row, raw_rank in zip(rows, raw_ranks):
|
||||
fact = dict(row)
|
||||
fact.pop("fts_rank_raw", None)
|
||||
fact["fts_rank"] = raw_rank / max_rank # normalize to [0, 1]
|
||||
results.append(fact)
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _tokenize(text: str) -> set[str]:
|
||||
"""Simple whitespace tokenization with lowercasing.
|
||||
|
||||
Strips common punctuation. No stemming/lemmatization (Phase 1).
|
||||
"""
|
||||
if not text:
|
||||
return set()
|
||||
# Split on whitespace, lowercase, strip punctuation
|
||||
tokens = set()
|
||||
for word in text.lower().split():
|
||||
cleaned = word.strip(".,;:!?\"'()[]{}#@<>")
|
||||
if cleaned:
|
||||
tokens.add(cleaned)
|
||||
return tokens
|
||||
|
||||
# Stopwords dropped before FTS5 OR-expansion. Short English function
|
||||
# words that carry no retrieval signal and force false-negative AND
|
||||
# matches when left in the query.
|
||||
_FTS_STOPWORDS = frozenset({
|
||||
"a", "about", "above", "after", "again", "all", "am", "an", "and",
|
||||
"any", "are", "as", "at", "be", "because", "been", "before", "being",
|
||||
"between", "both", "but", "by", "can", "could", "did", "do", "does",
|
||||
"doing", "don", "down", "during", "each", "few", "for", "from",
|
||||
"further", "had", "has", "have", "having", "he", "her", "here",
|
||||
"hers", "herself", "him", "himself", "his", "how", "i", "if", "in",
|
||||
"into", "is", "it", "its", "itself", "just", "me", "more", "most",
|
||||
"my", "myself", "no", "nor", "not", "now", "of", "off", "on", "once",
|
||||
"only", "or", "other", "our", "ours", "ourselves", "out", "over",
|
||||
"own", "same", "she", "should", "so", "some", "such", "than", "that",
|
||||
"the", "their", "theirs", "them", "themselves", "then", "there",
|
||||
"these", "they", "this", "those", "through", "to", "too", "under",
|
||||
"until", "up", "very", "was", "we", "were", "what", "when", "where",
|
||||
"which", "while", "who", "whom", "why", "will", "with", "would",
|
||||
"you", "your", "yours", "yourself", "yourselves",
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def _sanitize_fts_query(cls, query: str) -> str:
|
||||
"""Convert a natural-language query to an FTS5-safe OR expression.
|
||||
|
||||
FTS5 treats a multi-word MATCH argument as AND-joined by default,
|
||||
which tanks recall on prose queries. This helper:
|
||||
- tokenizes the query
|
||||
- drops stopwords and short (<2 char) tokens
|
||||
- strips FTS5 special characters from each token
|
||||
- OR-joins the survivors
|
||||
|
||||
If nothing remains (pathological query), falls back to the raw
|
||||
query so the caller sees zero results instead of a SQL error.
|
||||
"""
|
||||
if not query:
|
||||
return ""
|
||||
# Strip FTS5 operator characters from EACH token to avoid
|
||||
# accidentally creating a malformed query.
|
||||
_FTS_SPECIAL = '"()*^:-+'
|
||||
tokens: list[str] = []
|
||||
for raw in query.lower().split():
|
||||
cleaned = raw.strip(".,;:!?\"'()[]{}#@<>") .translate(
|
||||
str.maketrans("", "", _FTS_SPECIAL)
|
||||
)
|
||||
if len(cleaned) < 2:
|
||||
continue
|
||||
if cleaned in cls._FTS_STOPWORDS:
|
||||
continue
|
||||
# FTS5 phrase-literal each token to ensure no special chars
|
||||
# sneak through as operators.
|
||||
tokens.append(f'"{cleaned}"')
|
||||
if not tokens:
|
||||
# Fallback: raw query (likely returns 0, but never crashes)
|
||||
return query
|
||||
return " OR ".join(tokens)
|
||||
|
||||
@staticmethod
|
||||
def _jaccard_similarity(set_a: set, set_b: set) -> float:
|
||||
"""Jaccard similarity coefficient: |A ∩ B| / |A ∪ B|."""
|
||||
if not set_a or not set_b:
|
||||
return 0.0
|
||||
intersection = len(set_a & set_b)
|
||||
union = len(set_a | set_b)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
def _temporal_decay(self, timestamp_str: str | None) -> float:
|
||||
"""Exponential decay: 0.5^(age_days / half_life_days).
|
||||
|
||||
Returns 1.0 if decay is disabled or timestamp is missing.
|
||||
"""
|
||||
if not self.half_life or not timestamp_str:
|
||||
return 1.0
|
||||
|
||||
try:
|
||||
if isinstance(timestamp_str, str):
|
||||
# Parse ISO format timestamp from SQLite
|
||||
ts = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
|
||||
else:
|
||||
ts = timestamp_str
|
||||
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
|
||||
age_days = (datetime.now(timezone.utc) - ts).total_seconds() / 86400
|
||||
if age_days < 0:
|
||||
return 1.0
|
||||
|
||||
return math.pow(0.5, age_days / self.half_life)
|
||||
except (ValueError, TypeError):
|
||||
return 1.0
|
||||
@@ -0,0 +1,644 @@
|
||||
"""
|
||||
SQLite-backed fact store with entity resolution and trust scoring.
|
||||
Single-user Hermes memory store plugin.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from . import holographic as hrr
|
||||
except ImportError:
|
||||
import holographic as hrr # type: ignore[no-redef]
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS facts (
|
||||
fact_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
content TEXT NOT NULL UNIQUE,
|
||||
category TEXT DEFAULT 'general',
|
||||
tags TEXT DEFAULT '',
|
||||
trust_score REAL DEFAULT 0.5,
|
||||
retrieval_count INTEGER DEFAULT 0,
|
||||
helpful_count INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
hrr_vector BLOB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS entities (
|
||||
entity_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
entity_type TEXT DEFAULT 'unknown',
|
||||
aliases TEXT DEFAULT '',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fact_entities (
|
||||
fact_id INTEGER REFERENCES facts(fact_id),
|
||||
entity_id INTEGER REFERENCES entities(entity_id),
|
||||
PRIMARY KEY (fact_id, entity_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_facts_trust ON facts(trust_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_facts_category ON facts(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS facts_fts
|
||||
USING fts5(content, tags, content=facts, content_rowid=fact_id);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS facts_ai AFTER INSERT ON facts BEGIN
|
||||
INSERT INTO facts_fts(rowid, content, tags)
|
||||
VALUES (new.fact_id, new.content, new.tags);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS facts_ad AFTER DELETE ON facts BEGIN
|
||||
INSERT INTO facts_fts(facts_fts, rowid, content, tags)
|
||||
VALUES ('delete', old.fact_id, old.content, old.tags);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS facts_au AFTER UPDATE ON facts BEGIN
|
||||
INSERT INTO facts_fts(facts_fts, rowid, content, tags)
|
||||
VALUES ('delete', old.fact_id, old.content, old.tags);
|
||||
INSERT INTO facts_fts(rowid, content, tags)
|
||||
VALUES (new.fact_id, new.content, new.tags);
|
||||
END;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_banks (
|
||||
bank_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
bank_name TEXT NOT NULL UNIQUE,
|
||||
vector BLOB NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
fact_count INTEGER DEFAULT 0,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
"""
|
||||
|
||||
# Trust adjustment constants
|
||||
_HELPFUL_DELTA = 0.05
|
||||
_UNHELPFUL_DELTA = -0.10
|
||||
_TRUST_MIN = 0.0
|
||||
_TRUST_MAX = 1.0
|
||||
|
||||
# Entity extraction patterns
|
||||
_RE_CAPITALIZED = re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b')
|
||||
_RE_DOUBLE_QUOTE = re.compile(r'"([^"]+)"')
|
||||
_RE_SINGLE_QUOTE = re.compile(r"'([^']+)'")
|
||||
_RE_AKA = re.compile(
|
||||
r'(\w+(?:\s+\w+)*)\s+(?:aka|also known as)\s+(\w+(?:\s+\w+)*)',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _clamp_trust(value: float) -> float:
|
||||
return max(_TRUST_MIN, min(_TRUST_MAX, value))
|
||||
|
||||
|
||||
class MemoryStore:
|
||||
"""SQLite-backed fact store with entity resolution and trust scoring."""
|
||||
|
||||
# --- Process-wide shared connection registry -------------------------
|
||||
# SQLite permits only one writer at a time. Each MemoryStore instance used
|
||||
# to open its own connection guarded by its own RLock, so the several
|
||||
# providers that coexist in one process (the main agent plus every
|
||||
# delegate_task subagent) raced as independent WAL writers. Combined with
|
||||
# writes that were not rolled back on error, one connection could leave an
|
||||
# open write transaction that pinned the write lock and made every other
|
||||
# connection's write fail with "database is locked" for the full busy
|
||||
# timeout. All instances for the same database now share ONE connection and
|
||||
# ONE re-entrant lock, so access is fully serialized and cross-connection
|
||||
# contention is impossible. The shared connection is refcounted, so closing
|
||||
# one instance never tears the connection out from under a live sibling.
|
||||
_shared: dict = {}
|
||||
_shared_guard = threading.Lock()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_path: "str | Path | None" = None,
|
||||
default_trust: float = 0.5,
|
||||
hrr_dim: int = 1024,
|
||||
) -> None:
|
||||
if db_path is None:
|
||||
from hermes_constants import get_hermes_home
|
||||
db_path = str(get_hermes_home() / "memory_store.db")
|
||||
self.db_path = Path(db_path).expanduser()
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.default_trust = _clamp_trust(default_trust)
|
||||
self.hrr_dim = hrr_dim
|
||||
self._hrr_available = hrr._HAS_NUMPY
|
||||
|
||||
# Acquire (or open) the process-wide shared connection for this DB.
|
||||
# resolve() (not just expanduser) so symlinked/relative paths to the
|
||||
# same file share ONE connection instead of silently reintroducing
|
||||
# the multi-writer contention this registry exists to prevent.
|
||||
try:
|
||||
self._key = str(self.db_path.resolve())
|
||||
except OSError:
|
||||
self._key = str(self.db_path)
|
||||
with MemoryStore._shared_guard:
|
||||
entry = MemoryStore._shared.get(self._key)
|
||||
if entry is None:
|
||||
conn = sqlite3.connect(
|
||||
self._key,
|
||||
check_same_thread=False,
|
||||
timeout=10.0,
|
||||
# Autocommit: every statement is its own transaction, so a
|
||||
# write that raises mid-method can never leave a dangling
|
||||
# transaction (and its write lock) open. The explicit
|
||||
# commit() calls below become harmless no-ops.
|
||||
isolation_level=None,
|
||||
)
|
||||
conn.row_factory = sqlite3.Row
|
||||
entry = {"conn": conn, "lock": threading.RLock(), "refs": 0, "ready": False}
|
||||
MemoryStore._shared[self._key] = entry
|
||||
entry["refs"] += 1
|
||||
self._entry = entry
|
||||
self._conn = entry["conn"]
|
||||
self._lock = entry["lock"]
|
||||
|
||||
# Initialise the schema once per shared connection.
|
||||
with self._lock:
|
||||
if not self._entry["ready"]:
|
||||
self._init_db()
|
||||
self._entry["ready"] = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Initialisation
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _init_db(self) -> None:
|
||||
"""Create tables, indexes, and triggers if they do not exist. Enable WAL mode."""
|
||||
# Use the shared WAL-fallback helper so memory_store.db degrades
|
||||
# gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same issue as
|
||||
# state.db / kanban.db — see hermes_state._WAL_INCOMPAT_MARKERS).
|
||||
from hermes_state import apply_wal_with_fallback
|
||||
apply_wal_with_fallback(self._conn, db_label="memory_store.db (holographic)")
|
||||
self._conn.executescript(_SCHEMA)
|
||||
# Migrate: add hrr_vector column if missing (safe for existing databases)
|
||||
columns = {row[1] for row in self._conn.execute("PRAGMA table_info(facts)").fetchall()}
|
||||
if "hrr_vector" not in columns:
|
||||
self._conn.execute("ALTER TABLE facts ADD COLUMN hrr_vector BLOB")
|
||||
self._conn.commit()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_fact(
|
||||
self,
|
||||
content: str,
|
||||
category: str = "general",
|
||||
tags: str = "",
|
||||
) -> int:
|
||||
"""Insert a fact and return its fact_id.
|
||||
|
||||
Deduplicates by content (UNIQUE constraint). On duplicate, returns
|
||||
the existing fact_id without modifying the row. Extracts entities from
|
||||
the content and links them to the fact.
|
||||
"""
|
||||
with self._lock:
|
||||
content = content.strip()
|
||||
if not content:
|
||||
raise ValueError("content must not be empty")
|
||||
|
||||
try:
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
INSERT INTO facts (content, category, tags, trust_score)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(content, category, tags, self.default_trust),
|
||||
)
|
||||
self._conn.commit()
|
||||
fact_id: int = cur.lastrowid # type: ignore[assignment]
|
||||
except sqlite3.IntegrityError:
|
||||
# Duplicate content — return existing id
|
||||
row = self._conn.execute(
|
||||
"SELECT fact_id FROM facts WHERE content = ?", (content,)
|
||||
).fetchone()
|
||||
return int(row["fact_id"])
|
||||
|
||||
# Entity extraction and linking
|
||||
for name in self._extract_entities(content):
|
||||
entity_id = self._resolve_entity(name)
|
||||
self._link_fact_entity(fact_id, entity_id)
|
||||
|
||||
# Compute HRR vector after entity linking
|
||||
self._compute_hrr_vector(fact_id, content)
|
||||
self._rebuild_bank(category)
|
||||
|
||||
return fact_id
|
||||
|
||||
def search_facts(
|
||||
self,
|
||||
query: str,
|
||||
category: str | None = None,
|
||||
min_trust: float = 0.3,
|
||||
limit: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Full-text search over facts using FTS5.
|
||||
|
||||
Returns a list of fact dicts ordered by FTS5 rank, then trust_score
|
||||
descending. Also increments retrieval_count for matched facts.
|
||||
"""
|
||||
with self._lock:
|
||||
query = query.strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
# FTS5 AND-joins tokens by default, which zeroes out recall on
|
||||
# natural-language queries. Reuse the retriever's sanitizer
|
||||
# (stopword drop + OR-join content tokens). Imported lazily to
|
||||
# avoid a store->retrieval import cycle.
|
||||
from plugins.memory.holographic.retrieval import FactRetriever
|
||||
|
||||
match_query = FactRetriever._sanitize_fts_query(query)
|
||||
params: list = [match_query, min_trust]
|
||||
category_clause = ""
|
||||
if category is not None:
|
||||
category_clause = "AND f.category = ?"
|
||||
params.append(category)
|
||||
params.append(limit)
|
||||
|
||||
sql = f"""
|
||||
SELECT f.fact_id, f.content, f.category, f.tags,
|
||||
f.trust_score, f.retrieval_count, f.helpful_count,
|
||||
f.created_at, f.updated_at
|
||||
FROM facts f
|
||||
JOIN facts_fts fts ON fts.rowid = f.fact_id
|
||||
WHERE facts_fts MATCH ?
|
||||
AND f.trust_score >= ?
|
||||
{category_clause}
|
||||
ORDER BY fts.rank, f.trust_score DESC
|
||||
LIMIT ?
|
||||
"""
|
||||
|
||||
rows = self._conn.execute(sql, params).fetchall()
|
||||
results = [self._row_to_dict(r) for r in rows]
|
||||
|
||||
if results:
|
||||
ids = [r["fact_id"] for r in results]
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
self._conn.execute(
|
||||
f"UPDATE facts SET retrieval_count = retrieval_count + 1 WHERE fact_id IN ({placeholders})",
|
||||
ids,
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
return results
|
||||
|
||||
def update_fact(
|
||||
self,
|
||||
fact_id: int,
|
||||
content: str | None = None,
|
||||
trust_delta: float | None = None,
|
||||
tags: str | None = None,
|
||||
category: str | None = None,
|
||||
) -> bool:
|
||||
"""Partially update a fact. Trust is clamped to [0, 1].
|
||||
|
||||
Returns True if the row existed, False otherwise.
|
||||
"""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT fact_id, trust_score FROM facts WHERE fact_id = ?", (fact_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
|
||||
assignments: list[str] = ["updated_at = CURRENT_TIMESTAMP"]
|
||||
params: list = []
|
||||
|
||||
if content is not None:
|
||||
assignments.append("content = ?")
|
||||
params.append(content.strip())
|
||||
if tags is not None:
|
||||
assignments.append("tags = ?")
|
||||
params.append(tags)
|
||||
if category is not None:
|
||||
assignments.append("category = ?")
|
||||
params.append(category)
|
||||
if trust_delta is not None:
|
||||
new_trust = _clamp_trust(row["trust_score"] + trust_delta)
|
||||
assignments.append("trust_score = ?")
|
||||
params.append(new_trust)
|
||||
|
||||
params.append(fact_id)
|
||||
self._conn.execute(
|
||||
f"UPDATE facts SET {', '.join(assignments)} WHERE fact_id = ?",
|
||||
params,
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
# If content changed, re-extract entities
|
||||
if content is not None:
|
||||
self._conn.execute(
|
||||
"DELETE FROM fact_entities WHERE fact_id = ?", (fact_id,)
|
||||
)
|
||||
for name in self._extract_entities(content):
|
||||
entity_id = self._resolve_entity(name)
|
||||
self._link_fact_entity(fact_id, entity_id)
|
||||
self._conn.commit()
|
||||
|
||||
# Recompute HRR vector if content changed
|
||||
if content is not None:
|
||||
self._compute_hrr_vector(fact_id, content)
|
||||
# Rebuild bank for relevant category
|
||||
cat = category or self._conn.execute(
|
||||
"SELECT category FROM facts WHERE fact_id = ?", (fact_id,)
|
||||
).fetchone()["category"]
|
||||
self._rebuild_bank(cat)
|
||||
|
||||
return True
|
||||
|
||||
def remove_fact(self, fact_id: int) -> bool:
|
||||
"""Delete a fact and its entity links. Returns True if the row existed."""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT fact_id, category FROM facts WHERE fact_id = ?", (fact_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
|
||||
self._conn.execute(
|
||||
"DELETE FROM fact_entities WHERE fact_id = ?", (fact_id,)
|
||||
)
|
||||
self._conn.execute("DELETE FROM facts WHERE fact_id = ?", (fact_id,))
|
||||
self._conn.commit()
|
||||
self._rebuild_bank(row["category"])
|
||||
return True
|
||||
|
||||
def list_facts(
|
||||
self,
|
||||
category: str | None = None,
|
||||
min_trust: float = 0.0,
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
"""Browse facts ordered by trust_score descending.
|
||||
|
||||
Optionally filter by category and minimum trust score.
|
||||
"""
|
||||
with self._lock:
|
||||
params: list = [min_trust]
|
||||
category_clause = ""
|
||||
if category is not None:
|
||||
category_clause = "AND category = ?"
|
||||
params.append(category)
|
||||
params.append(limit)
|
||||
|
||||
sql = f"""
|
||||
SELECT fact_id, content, category, tags, trust_score,
|
||||
retrieval_count, helpful_count, created_at, updated_at
|
||||
FROM facts
|
||||
WHERE trust_score >= ?
|
||||
{category_clause}
|
||||
ORDER BY trust_score DESC
|
||||
LIMIT ?
|
||||
"""
|
||||
rows = self._conn.execute(sql, params).fetchall()
|
||||
return [self._row_to_dict(r) for r in rows]
|
||||
|
||||
def record_feedback(self, fact_id: int, helpful: bool) -> dict:
|
||||
"""Record user feedback and adjust trust asymmetrically.
|
||||
|
||||
helpful=True -> trust += 0.05, helpful_count += 1
|
||||
helpful=False -> trust -= 0.10
|
||||
|
||||
Returns a dict with fact_id, old_trust, new_trust, helpful_count.
|
||||
Raises KeyError if fact_id does not exist.
|
||||
"""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT fact_id, trust_score, helpful_count FROM facts WHERE fact_id = ?",
|
||||
(fact_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise KeyError(f"fact_id {fact_id} not found")
|
||||
|
||||
old_trust: float = row["trust_score"]
|
||||
delta = _HELPFUL_DELTA if helpful else _UNHELPFUL_DELTA
|
||||
new_trust = _clamp_trust(old_trust + delta)
|
||||
|
||||
helpful_increment = 1 if helpful else 0
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE facts
|
||||
SET trust_score = ?,
|
||||
helpful_count = helpful_count + ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE fact_id = ?
|
||||
""",
|
||||
(new_trust, helpful_increment, fact_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
return {
|
||||
"fact_id": fact_id,
|
||||
"old_trust": old_trust,
|
||||
"new_trust": new_trust,
|
||||
"helpful_count": row["helpful_count"] + helpful_increment,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entity helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _extract_entities(self, text: str) -> list[str]:
|
||||
"""Extract entity candidates from text using simple regex rules.
|
||||
|
||||
Rules applied (in order):
|
||||
1. Capitalized multi-word phrases e.g. "John Doe"
|
||||
2. Double-quoted terms e.g. "Python"
|
||||
3. Single-quoted terms e.g. 'pytest'
|
||||
4. AKA patterns e.g. "Guido aka BDFL" -> two entities
|
||||
|
||||
Returns a deduplicated list preserving first-seen order.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
candidates: list[str] = []
|
||||
|
||||
def _add(name: str) -> None:
|
||||
stripped = name.strip()
|
||||
if stripped and stripped.lower() not in seen:
|
||||
seen.add(stripped.lower())
|
||||
candidates.append(stripped)
|
||||
|
||||
for m in _RE_CAPITALIZED.finditer(text):
|
||||
_add(m.group(1))
|
||||
|
||||
for m in _RE_DOUBLE_QUOTE.finditer(text):
|
||||
_add(m.group(1))
|
||||
|
||||
for m in _RE_SINGLE_QUOTE.finditer(text):
|
||||
_add(m.group(1))
|
||||
|
||||
for m in _RE_AKA.finditer(text):
|
||||
_add(m.group(1))
|
||||
_add(m.group(2))
|
||||
|
||||
return candidates
|
||||
|
||||
def _resolve_entity(self, name: str) -> int:
|
||||
"""Find an existing entity by name or alias (case-insensitive) or create one.
|
||||
|
||||
Returns the entity_id.
|
||||
"""
|
||||
# Exact name match
|
||||
row = self._conn.execute(
|
||||
"SELECT entity_id FROM entities WHERE name LIKE ?", (name,)
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
return int(row["entity_id"])
|
||||
|
||||
# Search aliases — aliases stored as comma-separated; use LIKE with % boundaries
|
||||
alias_row = self._conn.execute(
|
||||
"""
|
||||
SELECT entity_id FROM entities
|
||||
WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'
|
||||
""",
|
||||
(name,),
|
||||
).fetchone()
|
||||
if alias_row is not None:
|
||||
return int(alias_row["entity_id"])
|
||||
|
||||
# Create new entity
|
||||
cur = self._conn.execute(
|
||||
"INSERT INTO entities (name) VALUES (?)", (name,)
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cur.lastrowid) # type: ignore[return-value]
|
||||
|
||||
def _link_fact_entity(self, fact_id: int, entity_id: int) -> None:
|
||||
"""Insert into fact_entities, silently ignore if the link already exists."""
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO fact_entities (fact_id, entity_id)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(fact_id, entity_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def _compute_hrr_vector(self, fact_id: int, content: str) -> None:
|
||||
"""Compute and store HRR vector for a fact. No-op if numpy unavailable."""
|
||||
with self._lock:
|
||||
if not self._hrr_available:
|
||||
return
|
||||
|
||||
# Get entities linked to this fact
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT e.name FROM entities e
|
||||
JOIN fact_entities fe ON fe.entity_id = e.entity_id
|
||||
WHERE fe.fact_id = ?
|
||||
""",
|
||||
(fact_id,),
|
||||
).fetchall()
|
||||
entities = [row["name"] for row in rows]
|
||||
|
||||
vector = hrr.encode_fact(content, entities, self.hrr_dim)
|
||||
self._conn.execute(
|
||||
"UPDATE facts SET hrr_vector = ? WHERE fact_id = ?",
|
||||
(hrr.phases_to_bytes(vector), fact_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def _rebuild_bank(self, category: str) -> None:
|
||||
"""Full rebuild of a category's memory bank from all its fact vectors."""
|
||||
with self._lock:
|
||||
if not self._hrr_available:
|
||||
return
|
||||
|
||||
bank_name = f"cat:{category}"
|
||||
rows = self._conn.execute(
|
||||
"SELECT hrr_vector FROM facts WHERE category = ? AND hrr_vector IS NOT NULL",
|
||||
(category,),
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
self._conn.execute("DELETE FROM memory_banks WHERE bank_name = ?", (bank_name,))
|
||||
self._conn.commit()
|
||||
return
|
||||
|
||||
vectors = [hrr.bytes_to_phases(row["hrr_vector"]) for row in rows]
|
||||
bank_vector = hrr.bundle(*vectors)
|
||||
fact_count = len(vectors)
|
||||
|
||||
# Check SNR
|
||||
hrr.snr_estimate(self.hrr_dim, fact_count)
|
||||
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at)
|
||||
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(bank_name) DO UPDATE SET
|
||||
vector = excluded.vector,
|
||||
dim = excluded.dim,
|
||||
fact_count = excluded.fact_count,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, fact_count),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def rebuild_all_vectors(self, dim: int | None = None) -> int:
|
||||
"""Recompute all HRR vectors + banks from text. For recovery/migration.
|
||||
|
||||
Returns the number of facts processed.
|
||||
"""
|
||||
with self._lock:
|
||||
if not self._hrr_available:
|
||||
return 0
|
||||
|
||||
if dim is not None:
|
||||
self.hrr_dim = dim
|
||||
|
||||
rows = self._conn.execute(
|
||||
"SELECT fact_id, content, category FROM facts"
|
||||
).fetchall()
|
||||
|
||||
categories: set[str] = set()
|
||||
for row in rows:
|
||||
self._compute_hrr_vector(row["fact_id"], row["content"])
|
||||
categories.add(row["category"])
|
||||
|
||||
for category in categories:
|
||||
self._rebuild_bank(category)
|
||||
|
||||
return len(rows)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Utilities
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _row_to_dict(self, row: sqlite3.Row) -> dict:
|
||||
"""Convert a sqlite3.Row to a plain dict."""
|
||||
return dict(row)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release this instance's reference to the shared connection.
|
||||
|
||||
The underlying connection is closed only when the last MemoryStore
|
||||
referencing the same database is closed, so closing one instance can
|
||||
never break sibling instances that still hold it. Idempotent.
|
||||
"""
|
||||
if getattr(self, "_entry", None) is None:
|
||||
return
|
||||
with MemoryStore._shared_guard:
|
||||
entry = self._entry
|
||||
if entry is None:
|
||||
return
|
||||
entry["refs"] -= 1
|
||||
if entry["refs"] <= 0:
|
||||
try:
|
||||
entry["conn"].close()
|
||||
finally:
|
||||
MemoryStore._shared.pop(self._key, None)
|
||||
self._entry = None
|
||||
|
||||
def __enter__(self) -> "MemoryStore":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.close()
|
||||
@@ -0,0 +1,388 @@
|
||||
# Honcho Memory Provider
|
||||
|
||||
AI-native cross-session user modeling with multi-pass dialectic reasoning, session summaries, bidirectional peer tools, and persistent conclusions.
|
||||
|
||||
> **Honcho docs:** <https://docs.honcho.dev/v3/guides/integrations/hermes>
|
||||
|
||||
## Requirements
|
||||
|
||||
- `pip install honcho-ai`
|
||||
- A Honcho Cloud account — connect via OAuth sign-in or an API key from
|
||||
[app.honcho.dev](https://app.honcho.dev) — or a self-hosted instance
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup honcho # configure Honcho directly (works on a fresh install)
|
||||
hermes memory setup # generic picker, choose Honcho from the list
|
||||
```
|
||||
|
||||
For cloud, the wizard asks **OAuth or API key**. OAuth opens a browser
|
||||
sign-in and stores the grant itself — nothing to copy; tokens refresh
|
||||
automatically. The desktop app offers the same flow as a **Connect** link
|
||||
next to the memory-provider dropdown.
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider honcho
|
||||
echo "HONCHO_API_KEY=***" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
> `hermes honcho setup` also works, but only **after** Honcho is the active
|
||||
> memory provider — the `honcho` subcommand is registered for the active
|
||||
> provider only. On a fresh install, use `hermes memory setup honcho`.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
### Two-Layer Context Injection
|
||||
|
||||
Context is injected into the **user message** at API-call time (not the system prompt) to preserve prompt caching. Only a static mode header goes in the system prompt. The injected block is wrapped in `<memory-context>` fences with a system note clarifying it's background data, not new user input.
|
||||
|
||||
Two independent layers, each on its own cadence:
|
||||
|
||||
**Layer 1 — Base context** (refreshed every `contextCadence` turns):
|
||||
1. **SESSION SUMMARY** — from `session.context(summary=True)`, placed first
|
||||
2. **User Representation** — Honcho's evolving model of the user
|
||||
3. **User Peer Card** — key facts snapshot
|
||||
4. **AI Self-Representation** — Honcho's model of the AI peer
|
||||
5. **AI Identity Card** — AI peer facts
|
||||
|
||||
**Layer 2 — Dialectic supplement** (fired every `dialecticCadence` turns):
|
||||
Multi-pass `.chat()` reasoning about the user, appended after base context.
|
||||
|
||||
Both layers are joined, then truncated to fit `contextTokens` budget via `_truncate_to_budget` (tokens × 4 chars, word-boundary safe).
|
||||
|
||||
### Cold Start vs Warm Session Prompts
|
||||
|
||||
Dialectic pass 0 automatically selects its prompt based on session state:
|
||||
|
||||
- **Cold** (no base context cached): "Who is this person? What are their preferences, goals, and working style? Focus on facts that would help an AI assistant be immediately useful."
|
||||
- **Warm** (base context exists): "Given what's been discussed in this session so far, what context about this user is most relevant to the current conversation? Prioritize active context over biographical facts."
|
||||
|
||||
Not configurable — determined automatically.
|
||||
|
||||
### Dialectic Depth (Multi-Pass Reasoning)
|
||||
|
||||
`dialecticDepth` (1–3, clamped) controls how many `.chat()` calls fire per dialectic cycle:
|
||||
|
||||
| Depth | Passes | Behavior |
|
||||
|-------|--------|----------|
|
||||
| 1 | single `.chat()` | Base query only (cold or warm prompt) |
|
||||
| 2 | audit + synthesis | Pass 0 result is self-audited; pass 1 does targeted synthesis. Conditional bail-out if pass 0 returns strong signal (>300 chars or structured with bullets/sections >100 chars) |
|
||||
| 3 | audit + synthesis + reconciliation | Pass 2 reconciles contradictions across prior passes into a final synthesis |
|
||||
|
||||
### Proportional Reasoning Levels
|
||||
|
||||
When `dialecticDepthLevels` is not set, each pass uses a proportional level relative to `dialecticReasoningLevel` (the "base"):
|
||||
|
||||
| Depth | Pass levels |
|
||||
|-------|-------------|
|
||||
| 1 | [base] |
|
||||
| 2 | [minimal, base] |
|
||||
| 3 | [minimal, base, low] |
|
||||
|
||||
Override with `dialecticDepthLevels`: an explicit array of reasoning level strings per pass.
|
||||
|
||||
### Query-Adaptive Reasoning Level
|
||||
|
||||
The auto-injected dialectic scales `dialecticReasoningLevel` by query length: +1 level at ≥120 chars, +2 at ≥400, clamped at `reasoningLevelCap` (default `"high"`). Disable with `reasoningHeuristic: false` to pin every auto call to `dialecticReasoningLevel`.
|
||||
|
||||
### Three Orthogonal Dialectic Knobs
|
||||
|
||||
| Knob | Controls | Type |
|
||||
|------|----------|------|
|
||||
| `dialecticCadence` | How often — minimum turns between dialectic firings | int |
|
||||
| `dialecticDepth` | How many — passes per firing (1–3) | int |
|
||||
| `dialecticReasoningLevel` | How hard — reasoning ceiling per `.chat()` call | string |
|
||||
|
||||
### Input Sanitization
|
||||
|
||||
`run_conversation` strips leaked `<memory-context>` blocks from user input before processing. When `saveMessages` persists a turn that included injected context, the block can reappear in subsequent turns via message history. The sanitizer removes `<memory-context>` blocks plus associated system notes.
|
||||
|
||||
## Tools
|
||||
|
||||
Five bidirectional tools. All accept an optional `peer` parameter (`"user"` or `"ai"`, default `"user"`).
|
||||
|
||||
| Tool | LLM call? | Description |
|
||||
|------|-----------|-------------|
|
||||
| `honcho_profile` | No | Peer card — key facts snapshot |
|
||||
| `honcho_search` | No | Semantic search over stored context (800 tok default, 2000 max) |
|
||||
| `honcho_context` | No | Full session context: summary, representation, card, messages |
|
||||
| `honcho_reasoning` | Yes | LLM-synthesized answer via dialectic `.chat()` |
|
||||
| `honcho_conclude` | No | Write a persistent fact/conclusion about the user |
|
||||
|
||||
Tool visibility depends on `recallMode`: hidden in `context` mode, always present in `tools` and `hybrid`.
|
||||
|
||||
## Config Resolution
|
||||
|
||||
Config is read from the first file that exists:
|
||||
|
||||
| Priority | Path | Scope |
|
||||
|----------|------|-------|
|
||||
| 1 | `$HERMES_HOME/honcho.json` | Profile-local (isolated Hermes instances) |
|
||||
| 2 | `~/.hermes/honcho.json` | Default profile (shared host blocks) |
|
||||
| 3 | `~/.honcho/config.json` | Global (cross-app interop) |
|
||||
|
||||
Host key is derived from the active Hermes profile: `hermes` (default) or `hermes_<profile>`.
|
||||
|
||||
For every key, resolution order is: **host block > root > env var > default**.
|
||||
|
||||
## Full Configuration Reference
|
||||
|
||||
### Identity & Connection
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `apiKey` | string | — | API key. Falls back to `HONCHO_API_KEY` env var. When connected via OAuth, holds the auto-refreshing access token instead |
|
||||
| `oauth` | object | — | OAuth grant (refresh token, expiry, client, token endpoint). Written by the Connect/sign-in flows and rotated automatically — not hand-edited. Optional: an API key alone works without it |
|
||||
| `baseUrl` | string | — | Base URL for self-hosted Honcho. Local URLs auto-skip API key auth |
|
||||
| `environment` | string | `"production"` | SDK environment mapping |
|
||||
| `enabled` | bool | auto | Master toggle. Auto-enables when `apiKey` or `baseUrl` present |
|
||||
| `workspace` | string | host key | Honcho workspace ID. Shared environment — all profiles in the same workspace can see the same user identity and related memories |
|
||||
| `peerName` | string | — | User peer identity |
|
||||
| `aiPeer` | string | host key | AI peer identity |
|
||||
|
||||
### Identity Mapping (Gateway Multi-User)
|
||||
|
||||
In gateway deployments (Telegram, Discord, Slack, etc.) each user arrives with a platform-native runtime ID (Telegram UID, Discord snowflake, Slack user). These three keys control how those runtime IDs map to Honcho peers. The resolver is config-driven and deterministic — no automatic merging or runtime inference.
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `pinUserPeer` | bool | `false` | When `true`, every gateway runtime user collapses to `peerName`. Single-operator deployments where you want all your platforms (and any other users) to share one peer |
|
||||
| `userPeerAliases` | object | `{}` | Map of runtime IDs to peer IDs (`{"7654321": "alice"}`). Many-to-one is the intended pattern — alias all your runtime IDs to one peer name. One-to-many is not supported; one runtime ID resolves to exactly one peer |
|
||||
| `runtimePeerPrefix` | string | `""` | Prepended to unknown runtime IDs to namespace them (e.g. `"telegram_"` → `telegram_7654321`). Used only when no alias matches. Prevents collisions between platforms whose runtime IDs share the same shape |
|
||||
|
||||
> **Deprecated:** `pinPeerName` is a legacy alias for `pinUserPeer`, still read for back-compat (`pinUserPeer` wins where both are set). `hermes honcho setup` migrates it onto `pinUserPeer` on touch and never writes it.
|
||||
|
||||
**Resolver ladder** (first match wins):
|
||||
|
||||
```
|
||||
1. pinUserPeer / pinPeerName=true → return peerName (ignore runtime ID)
|
||||
2. userPeerAliases[runtime_id] → return aliased peer
|
||||
3. userPeerAliases[runtime_id_alt] → check alt-ID too (Telegram UID + username, etc.)
|
||||
4. runtimePeerPrefix + runtime_id → namespaced peer, with sha256 collision escalation
|
||||
5. raw sanitized runtime_id → fallback peer
|
||||
6. peerName → no runtime ID at all (CLI/TUI)
|
||||
7. session-key fallback → no config either
|
||||
```
|
||||
|
||||
**Why no `pinAiPeer`?** The AI peer is already pinned by construction — `aiPeer` is the only AI-side identity setting and the resolver never overrides it. Only the user-side peer has the runtime-vs-config tension that `pinUserPeer` resolves.
|
||||
|
||||
**Host vs root semantics.** All three keys are accepted at both root and `hosts.<host>` levels. Host-level wins. For maps and prefixes, host-level *replaces* the root value as a whole (not merge), so a host can intentionally own its identity universe or wipe it with `userPeerAliases: {}` / `runtimePeerPrefix: ""`.
|
||||
|
||||
**Setup — gateway identity tree.** `hermes honcho setup` only asks about identity mapping when it detects a connected gateway platform (it inspects the gateway config; off-gateway the step is skipped because these keys do nothing without a runtime user ID). When it runs, it asks *who talks to this gateway?* and derives the keys:
|
||||
|
||||
- **just me** → `pinUserPeer: true`. Every non-agent gateway user collapses to `peerName`; the pin overrides all aliases, so pick this only when no user-side identity needs its own peer. Personal use where you connect Hermes to your own Telegram/Discord/etc. If separate agents reach the gateway and each needs a distinct peer, do **not** pin — leave `pinUserPeer: false` and map them via `userPeerAliases` (the `[e]` editor).
|
||||
- **me + other people, pooled** → `pinUserPeer: false` + `userPeerAliases` mapping your runtime IDs to `peerName`. You stay on the shared history; everyone else gets their own peer.
|
||||
- **me + other people / only other people** → `pinUserPeer: false`, optional `runtimePeerPrefix`. Each runtime user → own peer. For bots serving many humans.
|
||||
|
||||
Pick **[e]** at the prompt to set the three keys directly instead of going through the tree.
|
||||
|
||||
**Un-pinning (single → per-user).** Flipping `pinUserPeer` from `true` to `false` does not migrate data. Memory accumulated under `peerName` while pinned stays there; runtime users now resolve to fresh, empty peers. To preserve your own continuity, choose the **pooled** path — alias your runtime IDs back to `peerName` so your turns keep landing on the pooled history while other users get their own peers. The wizard offers this steer automatically when it detects you're un-pinning a previously pinned profile.
|
||||
|
||||
### Memory & Recall
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `recallMode` | string | `"hybrid"` | `"hybrid"` (auto-inject + tools), `"context"` (auto-inject only, tools hidden), `"tools"` (tools only, no injection). Legacy `"auto"` → `"hybrid"` |
|
||||
| `observationMode` | string | `"directional"` | Preset: `"directional"` (all on) or `"unified"` (user observes self, AI observes others). Use `observation` object for granular control |
|
||||
| `observation` | object | — | Per-peer observation config (see Observation section) |
|
||||
|
||||
### Write Behavior
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `writeFrequency` | string/int | `"async"` | `"async"` (background), `"turn"` (sync per turn), `"session"` (batch on end), or integer N (every N turns) |
|
||||
| `saveMessages` | bool | `true` | Persist messages to Honcho API |
|
||||
|
||||
### Session Resolution
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `sessionStrategy` | string | `"per-directory"` | `"per-directory"`, `"per-session"`, `"per-repo"` (git root), `"global"` |
|
||||
| `sessionPeerPrefix` | bool | `false` | Prepend peer name to session keys |
|
||||
| `sessions` | object | `{}` | Manual directory-to-session-name mappings |
|
||||
|
||||
#### Session Name Resolution
|
||||
|
||||
The Honcho session name determines which conversation bucket memory lands in. Resolution follows a priority chain — first match wins:
|
||||
|
||||
| Priority | Source | Example session name |
|
||||
|----------|--------|---------------------|
|
||||
| 1 | Manual map (`sessions` config) | `"myproject-main"` |
|
||||
| 2 | `/title` command (mid-session rename) | `"refactor-auth"` |
|
||||
| 3 | Gateway session key (Telegram, Discord, etc.) | `"agent-main-telegram-dm-8439114563"` |
|
||||
| 4 | `per-session` strategy | Hermes session ID (`20260415_a3f2b1`) |
|
||||
| 5 | `per-repo` strategy | Git root directory name (`hermes-agent`) |
|
||||
| 6 | `per-directory` strategy | Current directory basename (`src`) |
|
||||
| 7 | `global` strategy | Workspace name (`hermes`) |
|
||||
|
||||
Gateway platforms always resolve via priority 3 (per-chat isolation) regardless of `sessionStrategy`. The strategy setting only affects CLI sessions.
|
||||
|
||||
If `sessionPeerPrefix` is `true`, the peer name is prepended: `alice-hermes-agent`.
|
||||
|
||||
#### What each strategy produces
|
||||
|
||||
- **`per-directory`** — basename of `$PWD`. Opening hermes in `~/code/myapp` and `~/code/other` gives two separate sessions. Same directory = same session across runs.
|
||||
- **`per-repo`** — git root directory name. All subdirectories within a repo share one session. Falls back to `per-directory` if not inside a git repo.
|
||||
- **`per-session`** — Hermes session ID (timestamp + hex). Every `hermes` invocation starts a fresh Honcho session. Falls back to `per-directory` if no session ID is available.
|
||||
- **`global`** — workspace name. One session for everything. Memory accumulates across all directories and runs.
|
||||
|
||||
### Multi-Profile Pattern
|
||||
|
||||
Multiple Hermes profiles can share one workspace while maintaining separate AI identities. Config resolution is **host block > root > env var > default** — host blocks inherit from root, so shared settings only need to be declared once:
|
||||
|
||||
```json
|
||||
{
|
||||
"apiKey": "***",
|
||||
"workspace": "hermes",
|
||||
"peerName": "yourname",
|
||||
"hosts": {
|
||||
"hermes": {
|
||||
"aiPeer": "hermes",
|
||||
"recallMode": "hybrid",
|
||||
"sessionStrategy": "per-directory"
|
||||
},
|
||||
"hermes_coder": {
|
||||
"aiPeer": "coder",
|
||||
"recallMode": "tools",
|
||||
"sessionStrategy": "per-repo"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Both profiles see the same user (`yourname`) in the same shared environment (`hermes`), but each AI peer builds its own observations, conclusions, and behavior patterns. The coder's memory stays code-oriented; the main agent's stays broad.
|
||||
|
||||
Host key is derived from the active Hermes profile: `hermes` (default) or `hermes_<profile>` (e.g. `hermes -p coder` -> host key `hermes_coder`). Older `hermes.<profile>` host blocks are still read for compatibility and are migrated when the CLI writes profile-scoped Honcho config.
|
||||
|
||||
### Dialectic & Reasoning
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `dialecticDepth` | int | `1` | Passes per dialectic cycle (1–3, clamped). 1=single query, 2=audit+synthesis, 3=audit+synthesis+reconciliation |
|
||||
| `dialecticDepthLevels` | array | — | Optional array of reasoning level strings per pass. Overrides proportional defaults. Example: `["minimal", "low", "medium"]` |
|
||||
| `dialecticReasoningLevel` | string | `"low"` | Base reasoning level for `.chat()`: `"minimal"`, `"low"`, `"medium"`, `"high"`, `"max"` |
|
||||
| `dialecticDynamic` | bool | `true` | When `true`, model can override reasoning level per-call via `honcho_reasoning` tool. When `false`, always uses `dialecticReasoningLevel` |
|
||||
| `dialecticMaxChars` | int | `600` | Max chars of dialectic result injected into system prompt |
|
||||
| `dialecticMaxInputChars` | int | `10000` | Max chars for dialectic query input to `.chat()`. Honcho cloud limit: 10k |
|
||||
| `reasoningHeuristic` | bool | `true` | Query-adaptive: auto-scale the auto-injected dialectic's level up by query length (+1 at ≥120 chars, +2 at ≥400), clamped at `reasoningLevelCap`. `false` pins every auto call to `dialecticReasoningLevel` |
|
||||
| `reasoningLevelCap` | string | `"high"` | Ceiling for `reasoningHeuristic` scaling: `"minimal"`, `"low"`, `"medium"`, `"high"`, `"max"` |
|
||||
|
||||
### Token Budgets
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `contextTokens` | int | SDK default | Token budget for `context()` API calls. Also gates prefetch truncation (tokens × 4 chars) |
|
||||
| `messageMaxChars` | int | `25000` | Max chars per message sent via `add_messages()`. Exceeding this triggers chunking with `[continued]` markers. Honcho cloud limit: 25k |
|
||||
|
||||
### Cadence (Cost Control)
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `contextCadence` | int | `1` | Minimum turns between base context refreshes (session summary + representation + card) |
|
||||
| `dialecticCadence` | int | `1` | Minimum turns between dialectic `.chat()` firings |
|
||||
| `injectionFrequency` | string | `"every-turn"` | `"every-turn"` or `"first-turn"` (inject context on the first user message only, skip from turn 2 onward) |
|
||||
|
||||
### Observation (Granular)
|
||||
|
||||
Maps 1:1 to Honcho's per-peer `SessionPeerConfig`. When present, overrides `observationMode` preset.
|
||||
|
||||
```json
|
||||
"observation": {
|
||||
"user": { "observeMe": true, "observeOthers": true },
|
||||
"ai": { "observeMe": true, "observeOthers": true }
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `user.observeMe` | `true` | User peer self-observation (Honcho builds user representation) |
|
||||
| `user.observeOthers` | `true` | User peer observes AI messages |
|
||||
| `ai.observeMe` | `true` | AI peer self-observation (Honcho builds AI representation) |
|
||||
| `ai.observeOthers` | `true` | AI peer observes user messages (enables cross-peer dialectic) |
|
||||
|
||||
Presets:
|
||||
- `"directional"` (default): all four `true`
|
||||
- `"unified"`: user `observeMe=true`, AI `observeOthers=true`, rest `false`
|
||||
|
||||
### Hardcoded Limits
|
||||
|
||||
| Limit | Value |
|
||||
|-------|-------|
|
||||
| Search tool max tokens | 2000 (hard cap), 800 (default) |
|
||||
| Peer card fetch tokens | 200 |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Fallback for |
|
||||
|----------|-------------|
|
||||
| `HONCHO_API_KEY` | `apiKey` |
|
||||
| `HONCHO_BASE_URL` | `baseUrl` |
|
||||
| `HONCHO_ENVIRONMENT` | `environment` |
|
||||
| `HERMES_HONCHO_HOST` | Host key override |
|
||||
| `HONCHO_OAUTH_DASHBOARD` | OAuth authorize origin (default: cloud dashboard; local-dev `localhost:3000`) |
|
||||
| `HONCHO_OAUTH_AUTHORIZE_URL` | Full authorize URL (overrides the dashboard origin) |
|
||||
| `HONCHO_OAUTH_TOKEN_URL` | Token endpoint (default: cloud API; local-dev `localhost:8000`) |
|
||||
| `HONCHO_OAUTH_CLIENT_ID` | OAuth client (default `hermes-agent`) |
|
||||
| `HONCHO_OAUTH_SCOPE` | Requested scope (default `write`) |
|
||||
|
||||
## CLI Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `hermes memory setup honcho` | Configure Honcho directly — works on a fresh install |
|
||||
| `hermes honcho setup` | Interactive setup wizard (only registered once Honcho is the active provider; redirects to `hermes memory setup`) |
|
||||
| `hermes honcho status` | Show resolved config for active profile |
|
||||
| `hermes honcho enable` / `disable` | Toggle Honcho for active profile |
|
||||
| `hermes honcho mode <mode>` | Change recall or observation mode |
|
||||
| `hermes honcho peer --user <name>` | Update user peer name |
|
||||
| `hermes honcho peer --ai <name>` | Update AI peer name |
|
||||
| `hermes honcho tokens --context <N>` | Set context token budget |
|
||||
| `hermes honcho tokens --dialectic <N>` | Set dialectic max chars |
|
||||
| `hermes honcho map <name>` | Map current directory to a session name |
|
||||
| `hermes honcho sync` | Create host blocks for all Hermes profiles |
|
||||
|
||||
## Example Config
|
||||
|
||||
```json
|
||||
{
|
||||
"apiKey": "***",
|
||||
"workspace": "hermes",
|
||||
"peerName": "username",
|
||||
"contextCadence": 2,
|
||||
"dialecticCadence": 3,
|
||||
"dialecticDepth": 2,
|
||||
"hosts": {
|
||||
"hermes": {
|
||||
"enabled": true,
|
||||
"aiPeer": "hermes",
|
||||
"recallMode": "hybrid",
|
||||
"observation": {
|
||||
"user": { "observeMe": true, "observeOthers": true },
|
||||
"ai": { "observeMe": true, "observeOthers": true }
|
||||
},
|
||||
"writeFrequency": "async",
|
||||
"sessionStrategy": "per-directory",
|
||||
"dialecticReasoningLevel": "low",
|
||||
"dialecticDepth": 2,
|
||||
"dialecticMaxChars": 600,
|
||||
"saveMessages": true
|
||||
},
|
||||
"hermes_coder": {
|
||||
"enabled": true,
|
||||
"aiPeer": "coder",
|
||||
"sessionStrategy": "per-repo",
|
||||
"dialecticDepth": 1,
|
||||
"dialecticDepthLevels": ["low"],
|
||||
"observation": {
|
||||
"user": { "observeMe": true, "observeOthers": false },
|
||||
"ai": { "observeMe": true, "observeOthers": true }
|
||||
}
|
||||
}
|
||||
},
|
||||
"sessions": {
|
||||
"/home/user/myproject": "myproject-main"
|
||||
}
|
||||
}
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,920 @@
|
||||
"""Honcho client initialization and configuration.
|
||||
|
||||
Resolution order for config file:
|
||||
1. $HERMES_HOME/honcho.json (instance-local, enables isolated Hermes instances)
|
||||
2. ~/.honcho/config.json (global, shared across all Honcho-enabled apps)
|
||||
3. Environment variables (HONCHO_API_KEY, HONCHO_ENVIRONMENT)
|
||||
|
||||
Resolution order for host-specific settings:
|
||||
1. Explicit host block fields (always win)
|
||||
2. Flat/global fields from config root
|
||||
3. Defaults (host name as workspace/peer)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from hermes_cli.profiles import _get_default_hermes_home
|
||||
from plugins.plugin_utils import SingletonSlot
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from honcho import Honcho
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HOST = "hermes"
|
||||
|
||||
|
||||
def profile_host_key(profile: str | None) -> str:
|
||||
"""Return the safe Honcho host key for a Hermes profile."""
|
||||
if not profile or profile in {"default", "custom"}:
|
||||
return HOST
|
||||
sanitized = "".join(c if c.isalnum() or c in "_-" else "_" for c in profile).strip("_")
|
||||
return f"{HOST}_{sanitized or 'profile'}"
|
||||
|
||||
|
||||
def _host_block(raw: dict, host: str) -> dict:
|
||||
"""Return host config, accepting legacy dot-form profile host keys."""
|
||||
hosts = raw.get("hosts") or {}
|
||||
block = hosts.get(host, {})
|
||||
if block or not host.startswith(f"{HOST}_"):
|
||||
return block
|
||||
legacy = f"{HOST}.{host[len(HOST) + 1:]}"
|
||||
return hosts.get(legacy, {})
|
||||
|
||||
|
||||
def resolve_active_host() -> str:
|
||||
"""Derive the Honcho host key from the active Hermes profile.
|
||||
|
||||
Resolution order:
|
||||
1. HERMES_HONCHO_HOST env var (explicit override)
|
||||
2. Active profile name via profiles system -> ``hermes.<profile>``
|
||||
3. Fallback: ``"hermes"`` (default profile)
|
||||
"""
|
||||
explicit = os.environ.get("HERMES_HONCHO_HOST", "").strip()
|
||||
if explicit:
|
||||
return explicit
|
||||
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
profile = get_active_profile_name()
|
||||
return profile_host_key(profile)
|
||||
except Exception:
|
||||
pass
|
||||
return HOST
|
||||
|
||||
|
||||
def resolve_global_config_path() -> Path:
|
||||
"""Return the shared Honcho config path for the current HOME."""
|
||||
return Path.home() / ".honcho" / "config.json"
|
||||
|
||||
|
||||
def resolve_config_path() -> Path:
|
||||
"""Return the active Honcho config path.
|
||||
|
||||
Resolution order:
|
||||
1. $HERMES_HOME/honcho.json (profile-local, if it exists)
|
||||
2. ~/.hermes/honcho.json (default profile — shared host blocks live here)
|
||||
3. ~/.honcho/config.json (global, cross-app interop)
|
||||
|
||||
Returns the global path if none exist (for first-time setup writes).
|
||||
"""
|
||||
local_path = get_hermes_home() / "honcho.json"
|
||||
if local_path.exists():
|
||||
return local_path
|
||||
|
||||
# Default profile's config — host blocks accumulate here via setup/clone
|
||||
default_path = _get_default_hermes_home() / "honcho.json"
|
||||
if default_path != local_path and default_path.exists():
|
||||
return default_path
|
||||
|
||||
return resolve_global_config_path()
|
||||
|
||||
|
||||
_RECALL_MODE_ALIASES = {"auto": "hybrid"}
|
||||
_VALID_RECALL_MODES = {"hybrid", "context", "tools"}
|
||||
|
||||
|
||||
def _normalize_recall_mode(val: str) -> str:
|
||||
"""Normalize legacy recall mode values (e.g. 'auto' → 'hybrid')."""
|
||||
val = _RECALL_MODE_ALIASES.get(val, val)
|
||||
return val if val in _VALID_RECALL_MODES else "hybrid"
|
||||
|
||||
|
||||
def _resolve_bool(*vals, default: bool) -> bool:
|
||||
"""Resolve a bool config field: first non-None wins, else default.
|
||||
|
||||
Variadic to support aliased keys (e.g. ``pinUserPeer`` shadowing
|
||||
``pinPeerName`` for backwards compatibility). Pass values in
|
||||
precedence order: caller's preferred alias first, then fallback
|
||||
aliases, in (host, root) interleaving as needed.
|
||||
"""
|
||||
for val in vals:
|
||||
if val is not None:
|
||||
return bool(val)
|
||||
return default
|
||||
|
||||
|
||||
def _parse_context_tokens(host_val, root_val) -> int | None:
|
||||
"""Parse contextTokens: host wins, then root, then None (uncapped)."""
|
||||
for val in (host_val, root_val):
|
||||
if val is not None:
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_int_config(host_val, root_val, default: int) -> int:
|
||||
"""Parse an integer config: host wins, then root, then default."""
|
||||
for val in (host_val, root_val):
|
||||
if val is not None:
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def _parse_string_map(host_obj: dict, root_obj: dict, key: str) -> dict[str, str]:
|
||||
"""Parse a string-to-string map with host-level whole-map override."""
|
||||
source = host_obj[key] if key in host_obj else root_obj.get(key)
|
||||
if not isinstance(source, dict):
|
||||
return {}
|
||||
|
||||
result: dict[str, str] = {}
|
||||
for raw_key, raw_value in source.items():
|
||||
alias_key = str(raw_key).strip()
|
||||
alias_value = str(raw_value).strip() if raw_value is not None else ""
|
||||
if alias_key and alias_value:
|
||||
result[alias_key] = alias_value
|
||||
return result
|
||||
|
||||
|
||||
def _parse_optional_string(
|
||||
host_obj: dict, root_obj: dict, key: str, default: str = ""
|
||||
) -> str:
|
||||
"""Parse a string field where host-level empty string can override root."""
|
||||
if key in host_obj:
|
||||
value = host_obj.get(key)
|
||||
else:
|
||||
value = root_obj.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def _parse_dialectic_depth(host_val, root_val) -> int:
|
||||
"""Parse dialecticDepth: host wins, then root, then 1. Clamped to 1-3."""
|
||||
for val in (host_val, root_val):
|
||||
if val is not None:
|
||||
try:
|
||||
return max(1, min(int(val), 3))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return 1
|
||||
|
||||
|
||||
_VALID_REASONING_LEVELS = ("minimal", "low", "medium", "high", "max")
|
||||
|
||||
|
||||
def _parse_dialectic_depth_levels(host_val, root_val, depth: int) -> list[str] | None:
|
||||
"""Parse dialecticDepthLevels: optional array of reasoning levels per pass.
|
||||
|
||||
Returns None when not configured (use proportional defaults).
|
||||
When configured, validates each level and truncates/pads to match depth.
|
||||
"""
|
||||
for val in (host_val, root_val):
|
||||
if val is not None and isinstance(val, list):
|
||||
levels = [
|
||||
lvl if lvl in _VALID_REASONING_LEVELS else "low"
|
||||
for lvl in val[:depth]
|
||||
]
|
||||
# Pad with "low" if array is shorter than depth
|
||||
while len(levels) < depth:
|
||||
levels.append("low")
|
||||
return levels
|
||||
return None
|
||||
|
||||
|
||||
# Default HTTP timeout (seconds) applied when no explicit timeout is
|
||||
# configured via HonchoClientConfig.timeout, honcho.timeout / requestTimeout,
|
||||
# or HONCHO_TIMEOUT. Honcho calls happen on the post-response path of
|
||||
# run_conversation; without a cap the agent can block indefinitely when
|
||||
# the Honcho backend is unreachable, preventing the gateway from
|
||||
# delivering the already-generated response.
|
||||
_DEFAULT_HTTP_TIMEOUT = 30.0
|
||||
|
||||
|
||||
def _resolve_optional_float(*values: Any) -> float | None:
|
||||
"""Return the first non-empty value coerced to a positive float."""
|
||||
for value in values:
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
continue
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if parsed > 0:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
|
||||
_VALID_OBSERVATION_MODES = {"unified", "directional"}
|
||||
_OBSERVATION_MODE_ALIASES = {"shared": "unified", "separate": "directional", "cross": "directional"}
|
||||
|
||||
|
||||
def _normalize_observation_mode(val: str) -> str:
|
||||
"""Normalize observation mode values."""
|
||||
val = _OBSERVATION_MODE_ALIASES.get(val, val)
|
||||
return val if val in _VALID_OBSERVATION_MODES else "directional"
|
||||
|
||||
|
||||
# Observation presets — granular booleans derived from legacy string mode.
|
||||
# Explicit per-peer config always wins over presets.
|
||||
_OBSERVATION_PRESETS = {
|
||||
"directional": {
|
||||
"user_observe_me": True, "user_observe_others": True,
|
||||
"ai_observe_me": True, "ai_observe_others": True,
|
||||
},
|
||||
"unified": {
|
||||
"user_observe_me": True, "user_observe_others": False,
|
||||
"ai_observe_me": False, "ai_observe_others": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _resolve_observation(
|
||||
mode: str,
|
||||
observation_obj: dict | None,
|
||||
) -> dict:
|
||||
"""Resolve per-peer observation booleans.
|
||||
|
||||
Config forms:
|
||||
String shorthand: ``"observationMode": "directional"``
|
||||
Granular object: ``"observation": {"user": {"observeMe": true, "observeOthers": true},
|
||||
"ai": {"observeMe": true, "observeOthers": false}}``
|
||||
|
||||
Granular fields override preset defaults.
|
||||
"""
|
||||
preset = _OBSERVATION_PRESETS.get(mode, _OBSERVATION_PRESETS["directional"])
|
||||
if not observation_obj or not isinstance(observation_obj, dict):
|
||||
return dict(preset)
|
||||
|
||||
user_block = observation_obj.get("user") or {}
|
||||
ai_block = observation_obj.get("ai") or {}
|
||||
|
||||
return {
|
||||
"user_observe_me": user_block.get("observeMe", preset["user_observe_me"]),
|
||||
"user_observe_others": user_block.get("observeOthers", preset["user_observe_others"]),
|
||||
"ai_observe_me": ai_block.get("observeMe", preset["ai_observe_me"]),
|
||||
"ai_observe_others": ai_block.get("observeOthers", preset["ai_observe_others"]),
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@dataclass
|
||||
class HonchoClientConfig:
|
||||
"""Configuration for Honcho client, resolved for a specific host."""
|
||||
|
||||
host: str = HOST
|
||||
workspace_id: str = "hermes"
|
||||
api_key: str | None = None
|
||||
environment: str = "production"
|
||||
# Optional base URL for self-hosted Honcho (overrides environment mapping)
|
||||
base_url: str | None = None
|
||||
# Optional request timeout in seconds for Honcho SDK HTTP calls
|
||||
timeout: float | None = None
|
||||
# Identity
|
||||
peer_name: str | None = None
|
||||
ai_peer: str = "hermes"
|
||||
# When True, ``peer_name`` wins over any gateway-supplied runtime
|
||||
# identity (Telegram UID, Discord ID, …) when resolving the user peer.
|
||||
# This keeps memory unified across platforms for single-user deployments
|
||||
# where Honcho's one peer-name is an unambiguous identity — otherwise
|
||||
# each platform would fork memory into its own peer (#14984). Default
|
||||
# ``False`` preserves existing multi-user behaviour.
|
||||
pin_peer_name: bool = False
|
||||
# Map gateway runtime user IDs to stable Honcho user peers. Host-level
|
||||
# config replaces the root map as a whole so profiles can intentionally
|
||||
# own their identity mappings.
|
||||
user_peer_aliases: dict[str, str] = field(default_factory=dict)
|
||||
# Optional prefix for unknown gateway runtime user IDs, e.g. "telegram_".
|
||||
runtime_peer_prefix: str = ""
|
||||
# Toggles
|
||||
enabled: bool = False
|
||||
save_messages: bool = True
|
||||
# Write frequency: "async" (background thread), "turn" (sync per turn),
|
||||
# "session" (flush on session end), or int (every N turns)
|
||||
write_frequency: str | int = "async"
|
||||
# Prefetch budget (None = no cap; set to an integer to bound auto-injected context)
|
||||
context_tokens: int | None = None
|
||||
# Dialectic (peer.chat) settings
|
||||
# reasoning_level: "minimal" | "low" | "medium" | "high" | "max"
|
||||
dialectic_reasoning_level: str = "low"
|
||||
# When true, the model can override reasoning_level per-call via the
|
||||
# honcho_reasoning tool param (agentic). When false, always uses
|
||||
# dialecticReasoningLevel and ignores model-provided overrides.
|
||||
dialectic_dynamic: bool = True
|
||||
# Max chars of dialectic result to inject into Hermes system prompt
|
||||
dialectic_max_chars: int = 600
|
||||
# Dialectic depth: how many .chat() calls per dialectic cycle (1-3).
|
||||
# Depth 1: single call. Depth 2: self-audit + targeted synthesis.
|
||||
# Depth 3: self-audit + synthesis + reconciliation.
|
||||
dialectic_depth: int = 1
|
||||
# Optional per-pass reasoning level override. Array of reasoning levels
|
||||
# matching dialectic_depth length. When None, uses proportional defaults
|
||||
# derived from dialectic_reasoning_level.
|
||||
dialectic_depth_levels: list[str] | None = None
|
||||
# When true, the auto-injected dialectic scales reasoning level up on
|
||||
# longer queries. See HonchoMemoryProvider for thresholds.
|
||||
reasoning_heuristic: bool = True
|
||||
# Ceiling for the heuristic-selected reasoning level.
|
||||
reasoning_level_cap: str = "high"
|
||||
# Honcho API limits — configurable for self-hosted instances
|
||||
# Max chars per message sent via add_messages() (Honcho cloud: 25000)
|
||||
message_max_chars: int = 25000
|
||||
# Max chars for dialectic query input to peer.chat() (Honcho cloud: 10000)
|
||||
dialectic_max_input_chars: int = 10000
|
||||
# Recall mode: how memory retrieval works when Honcho is active.
|
||||
# "hybrid" — auto-injected context + Honcho tools available (model decides)
|
||||
# "context" — auto-injected context only, Honcho tools removed
|
||||
# "tools" — Honcho tools only, no auto-injected context
|
||||
recall_mode: str = "hybrid"
|
||||
# Eager init in tools mode — when true, initializes session during
|
||||
# initialize() instead of deferring to first tool call
|
||||
init_on_session_start: bool = False
|
||||
# Observation mode: legacy string shorthand ("directional" or "unified").
|
||||
# Kept for backward compat; granular per-peer booleans below are preferred.
|
||||
observation_mode: str = "directional"
|
||||
# Per-peer observation booleans — maps 1:1 to Honcho's SessionPeerConfig.
|
||||
# Resolved from "observation" object in config, falling back to observation_mode preset.
|
||||
user_observe_me: bool = True
|
||||
user_observe_others: bool = True
|
||||
ai_observe_me: bool = True
|
||||
ai_observe_others: bool = True
|
||||
# Session resolution
|
||||
session_strategy: str = "per-directory"
|
||||
session_peer_prefix: bool = False
|
||||
sessions: dict[str, str] = field(default_factory=dict)
|
||||
# Raw global config for anything else consumers need
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
# True when Honcho was explicitly configured for this host (hosts.hermes
|
||||
# block exists or enabled was set explicitly), vs auto-enabled from a
|
||||
# stray HONCHO_API_KEY env var.
|
||||
explicitly_configured: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
workspace_id: str = "hermes",
|
||||
host: str | None = None,
|
||||
) -> HonchoClientConfig:
|
||||
"""Create config from environment variables (fallback)."""
|
||||
resolved_host = host or resolve_active_host()
|
||||
api_key = os.environ.get("HONCHO_API_KEY")
|
||||
base_url = os.environ.get("HONCHO_BASE_URL", "").strip() or None
|
||||
timeout = _resolve_optional_float(os.environ.get("HONCHO_TIMEOUT"))
|
||||
return cls(
|
||||
host=resolved_host,
|
||||
workspace_id=workspace_id,
|
||||
api_key=api_key,
|
||||
environment=os.environ.get("HONCHO_ENVIRONMENT", "production"),
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
ai_peer=resolved_host,
|
||||
enabled=bool(api_key or base_url),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_global_config(
|
||||
cls,
|
||||
host: str | None = None,
|
||||
config_path: Path | None = None,
|
||||
) -> HonchoClientConfig:
|
||||
"""Create config from the resolved Honcho config path.
|
||||
|
||||
Resolution: $HERMES_HOME/honcho.json -> ~/.honcho/config.json -> env vars.
|
||||
When host is None, derives it from the active Hermes profile.
|
||||
"""
|
||||
resolved_host = host or resolve_active_host()
|
||||
path = config_path or resolve_config_path()
|
||||
if not path.exists():
|
||||
logger.debug("No global Honcho config at %s, falling back to env", path)
|
||||
return cls.from_env(host=resolved_host)
|
||||
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning("Failed to read %s: %s, falling back to env", path, e)
|
||||
return cls.from_env(host=resolved_host)
|
||||
|
||||
host_block = _host_block(raw, resolved_host)
|
||||
# A hosts.hermes block or explicit enabled flag means the user
|
||||
# intentionally configured Honcho for this host.
|
||||
_explicitly_configured = bool(host_block) or raw.get("enabled") is True
|
||||
|
||||
# Explicit host block fields win, then flat/global, then defaults
|
||||
workspace = (
|
||||
host_block.get("workspace")
|
||||
or raw.get("workspace")
|
||||
or resolved_host
|
||||
)
|
||||
ai_peer = (
|
||||
host_block.get("aiPeer")
|
||||
or raw.get("aiPeer")
|
||||
or resolved_host
|
||||
)
|
||||
api_key = (
|
||||
host_block.get("apiKey")
|
||||
or raw.get("apiKey")
|
||||
or os.environ.get("HONCHO_API_KEY")
|
||||
)
|
||||
|
||||
environment = (
|
||||
host_block.get("environment")
|
||||
or raw.get("environment", "production")
|
||||
)
|
||||
|
||||
base_url = (
|
||||
raw.get("baseUrl")
|
||||
or raw.get("base_url")
|
||||
or os.environ.get("HONCHO_BASE_URL", "").strip()
|
||||
or None
|
||||
)
|
||||
timeout = _resolve_optional_float(
|
||||
raw.get("timeout"),
|
||||
raw.get("requestTimeout"),
|
||||
os.environ.get("HONCHO_TIMEOUT"),
|
||||
)
|
||||
|
||||
# Auto-enable when API key or base_url is present (unless explicitly disabled)
|
||||
# Host-level enabled wins, then root-level, then auto-enable if key/url exists.
|
||||
host_enabled = host_block.get("enabled")
|
||||
root_enabled = raw.get("enabled")
|
||||
if host_enabled is not None:
|
||||
enabled = host_enabled
|
||||
elif root_enabled is not None:
|
||||
enabled = root_enabled
|
||||
else:
|
||||
# Not explicitly set anywhere -> auto-enable if API key or base_url exists
|
||||
enabled = bool(api_key or base_url)
|
||||
|
||||
# write_frequency: accept int or string
|
||||
raw_wf = (
|
||||
host_block.get("writeFrequency")
|
||||
or raw.get("writeFrequency")
|
||||
or "async"
|
||||
)
|
||||
try:
|
||||
write_frequency: str | int = int(raw_wf)
|
||||
except (TypeError, ValueError):
|
||||
write_frequency = str(raw_wf)
|
||||
|
||||
# saveMessages: host wins (None-aware since False is valid)
|
||||
host_save = host_block.get("saveMessages")
|
||||
save_messages = host_save if host_save is not None else raw.get("saveMessages", True)
|
||||
|
||||
# sessionStrategy / sessionPeerPrefix: host first, root fallback
|
||||
session_strategy = (
|
||||
host_block.get("sessionStrategy")
|
||||
or raw.get("sessionStrategy", "per-directory")
|
||||
)
|
||||
host_prefix = host_block.get("sessionPeerPrefix")
|
||||
session_peer_prefix = (
|
||||
host_prefix if host_prefix is not None
|
||||
else raw.get("sessionPeerPrefix", False)
|
||||
)
|
||||
|
||||
return cls(
|
||||
host=resolved_host,
|
||||
workspace_id=workspace,
|
||||
api_key=api_key,
|
||||
environment=environment,
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
peer_name=host_block.get("peerName") or raw.get("peerName"),
|
||||
ai_peer=ai_peer,
|
||||
pin_peer_name=_resolve_bool(
|
||||
# ``pinUserPeer`` is the clearer name (the resolver pins
|
||||
# the user-side peer to ``peerName``, ignoring runtime
|
||||
# identity). ``pinPeerName`` is the original key from
|
||||
# #14984 and stays accepted for backward compatibility.
|
||||
# Host-level keys win over root-level; among same-level
|
||||
# keys, ``pinUserPeer`` wins over ``pinPeerName``.
|
||||
host_block.get("pinUserPeer"),
|
||||
host_block.get("pinPeerName"),
|
||||
raw.get("pinUserPeer"),
|
||||
raw.get("pinPeerName"),
|
||||
default=False,
|
||||
),
|
||||
user_peer_aliases=_parse_string_map(
|
||||
host_block,
|
||||
raw,
|
||||
"userPeerAliases",
|
||||
),
|
||||
runtime_peer_prefix=_parse_optional_string(
|
||||
host_block,
|
||||
raw,
|
||||
"runtimePeerPrefix",
|
||||
),
|
||||
enabled=enabled,
|
||||
save_messages=save_messages,
|
||||
write_frequency=write_frequency,
|
||||
context_tokens=_parse_context_tokens(
|
||||
host_block.get("contextTokens"),
|
||||
raw.get("contextTokens"),
|
||||
),
|
||||
dialectic_reasoning_level=(
|
||||
host_block.get("dialecticReasoningLevel")
|
||||
or raw.get("dialecticReasoningLevel")
|
||||
or "low"
|
||||
),
|
||||
dialectic_dynamic=_resolve_bool(
|
||||
host_block.get("dialecticDynamic"),
|
||||
raw.get("dialecticDynamic"),
|
||||
default=True,
|
||||
),
|
||||
dialectic_max_chars=_parse_int_config(
|
||||
host_block.get("dialecticMaxChars"),
|
||||
raw.get("dialecticMaxChars"),
|
||||
default=600,
|
||||
),
|
||||
dialectic_depth=_parse_dialectic_depth(
|
||||
host_block.get("dialecticDepth"),
|
||||
raw.get("dialecticDepth"),
|
||||
),
|
||||
dialectic_depth_levels=_parse_dialectic_depth_levels(
|
||||
host_block.get("dialecticDepthLevels"),
|
||||
raw.get("dialecticDepthLevels"),
|
||||
depth=_parse_dialectic_depth(host_block.get("dialecticDepth"), raw.get("dialecticDepth")),
|
||||
),
|
||||
reasoning_heuristic=_resolve_bool(
|
||||
host_block.get("reasoningHeuristic"),
|
||||
raw.get("reasoningHeuristic"),
|
||||
default=True,
|
||||
),
|
||||
reasoning_level_cap=(
|
||||
host_block.get("reasoningLevelCap")
|
||||
or raw.get("reasoningLevelCap")
|
||||
or "high"
|
||||
),
|
||||
message_max_chars=_parse_int_config(
|
||||
host_block.get("messageMaxChars"),
|
||||
raw.get("messageMaxChars"),
|
||||
default=25000,
|
||||
),
|
||||
dialectic_max_input_chars=_parse_int_config(
|
||||
host_block.get("dialecticMaxInputChars"),
|
||||
raw.get("dialecticMaxInputChars"),
|
||||
default=10000,
|
||||
),
|
||||
recall_mode=_normalize_recall_mode(
|
||||
host_block.get("recallMode")
|
||||
or raw.get("recallMode")
|
||||
or "hybrid"
|
||||
),
|
||||
init_on_session_start=_resolve_bool(
|
||||
host_block.get("initOnSessionStart"),
|
||||
raw.get("initOnSessionStart"),
|
||||
default=False,
|
||||
),
|
||||
# Migration guard: existing configs without an explicit
|
||||
# observationMode keep the old "unified" default so users
|
||||
# aren't silently switched to full bidirectional observation.
|
||||
# New installations (no host block, no credentials) get
|
||||
# "directional" (all observations on) as the new default.
|
||||
observation_mode=_normalize_observation_mode(
|
||||
host_block.get("observationMode")
|
||||
or raw.get("observationMode")
|
||||
or ("unified" if _explicitly_configured else "directional")
|
||||
),
|
||||
**_resolve_observation(
|
||||
_normalize_observation_mode(
|
||||
host_block.get("observationMode")
|
||||
or raw.get("observationMode")
|
||||
or ("unified" if _explicitly_configured else "directional")
|
||||
),
|
||||
host_block.get("observation") or raw.get("observation"),
|
||||
),
|
||||
session_strategy=session_strategy,
|
||||
session_peer_prefix=session_peer_prefix,
|
||||
sessions=raw.get("sessions", {}),
|
||||
raw=raw,
|
||||
explicitly_configured=_explicitly_configured,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _git_repo_name(cwd: str) -> str | None:
|
||||
"""Return the git repo root directory name, or None if not in a repo."""
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
root = subprocess.run(
|
||||
["git", "rev-parse", "--show-toplevel"],
|
||||
capture_output=True, text=True, cwd=cwd, timeout=5,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
if root.returncode == 0:
|
||||
return Path(root.stdout.strip()).name
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
return None
|
||||
|
||||
# Honcho enforces a 100-char limit on session IDs. Long gateway session keys
|
||||
# (Matrix "!room:server" + thread event IDs, Telegram supergroup reply
|
||||
# chains, Slack thread IDs with long workspace prefixes) can overflow this
|
||||
# limit after sanitization; the Honcho API then rejects every call for that
|
||||
# session with "session_id too long". See issue #13868.
|
||||
_HONCHO_SESSION_ID_MAX_LEN = 100
|
||||
_HONCHO_SESSION_ID_HASH_LEN = 8
|
||||
|
||||
@classmethod
|
||||
def _enforce_session_id_limit(cls, sanitized: str, original: str) -> str:
|
||||
"""Truncate a sanitized session ID to Honcho's 100-char limit.
|
||||
|
||||
The common case (short keys) short-circuits with no modification.
|
||||
For over-limit keys, keep a prefix of the sanitized ID and append a
|
||||
deterministic ``-<sha256 prefix>`` suffix so two distinct long keys
|
||||
that share a leading segment don't collide onto the same truncated ID.
|
||||
The hash is taken over the *original* pre-sanitization key, so two
|
||||
inputs that sanitize to the same string still collide intentionally
|
||||
(same logical session), but two inputs that only share a prefix do not.
|
||||
"""
|
||||
max_len = cls._HONCHO_SESSION_ID_MAX_LEN
|
||||
if len(sanitized) <= max_len:
|
||||
return sanitized
|
||||
|
||||
hash_len = cls._HONCHO_SESSION_ID_HASH_LEN
|
||||
digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:hash_len]
|
||||
# max_len - hash_len - 1 (for the '-' separator) chars of the sanitized
|
||||
# prefix, then '-<hash>'. Strip any trailing hyphen from the prefix so
|
||||
# the result doesn't double up on separators.
|
||||
prefix_len = max_len - hash_len - 1
|
||||
prefix = sanitized[:prefix_len].rstrip("-")
|
||||
return f"{prefix}-{digest}"
|
||||
|
||||
def resolve_session_name(
|
||||
self,
|
||||
cwd: str | None = None,
|
||||
session_title: str | None = None,
|
||||
session_id: str | None = None,
|
||||
gateway_session_key: str | None = None,
|
||||
) -> str | None:
|
||||
"""Resolve Honcho session name.
|
||||
|
||||
Resolution order:
|
||||
1. Gateway session key (stable per-chat identifier from gateway platforms)
|
||||
2. per-session strategy — Hermes session_id ({timestamp}_{hex}); authoritative,
|
||||
so a generated title never remaps a live conversation
|
||||
3. Manual directory override from sessions map
|
||||
4. Hermes session title (from /title command; non-per-session)
|
||||
5. per-repo strategy — git repo root directory name
|
||||
6. per-directory strategy — directory basename
|
||||
7. global strategy — workspace name
|
||||
"""
|
||||
import re
|
||||
|
||||
if not cwd:
|
||||
cwd = os.getcwd()
|
||||
|
||||
# Gateway per-chat key wins everywhere — gateways (telegram/discord/…)
|
||||
# need per-chat isolation no cwd/strategy name can provide.
|
||||
if gateway_session_key:
|
||||
sanitized = re.sub(r'[^a-zA-Z0-9_-]+', '-', gateway_session_key).strip('-')
|
||||
if sanitized:
|
||||
return self._enforce_session_id_limit(sanitized, gateway_session_key)
|
||||
|
||||
# per-session: the run's session_id IS the identity — resolve before the
|
||||
# cwd map / title so an auto-generated title can't remap a live
|
||||
# conversation onto a second Honcho session mid-stream.
|
||||
if self.session_strategy == "per-session" and session_id:
|
||||
if self.session_peer_prefix and self.peer_name:
|
||||
return f"{self.peer_name}-{session_id}"
|
||||
return session_id
|
||||
|
||||
# Manual override (cwd → name), for non-per-session strategies.
|
||||
manual = self.sessions.get(cwd)
|
||||
if manual:
|
||||
return manual
|
||||
|
||||
# /title mid-session remap (non-per-session).
|
||||
if session_title:
|
||||
sanitized = re.sub(r'[^a-zA-Z0-9_-]+', '-', session_title).strip('-')
|
||||
if sanitized:
|
||||
if self.session_peer_prefix and self.peer_name:
|
||||
return f"{self.peer_name}-{sanitized}"
|
||||
return sanitized
|
||||
|
||||
# per-repo: one Honcho session per git repository
|
||||
if self.session_strategy == "per-repo":
|
||||
base = self._git_repo_name(cwd) or Path(cwd).name
|
||||
if self.session_peer_prefix and self.peer_name:
|
||||
return f"{self.peer_name}-{base}"
|
||||
return base
|
||||
|
||||
# per-directory: one Honcho session per working directory (default)
|
||||
if self.session_strategy in {"per-directory", "per-session"}:
|
||||
base = Path(cwd).name
|
||||
if self.session_peer_prefix and self.peer_name:
|
||||
return f"{self.peer_name}-{base}"
|
||||
return base
|
||||
|
||||
# global: single session across all directories
|
||||
return self.workspace_id
|
||||
|
||||
|
||||
_honcho_client_slot: SingletonSlot = SingletonSlot()
|
||||
|
||||
|
||||
def _apply_fresh_oauth_token(config: HonchoClientConfig) -> None:
|
||||
"""Refresh a near-expiry OAuth grant and point ``config.api_key`` at it.
|
||||
|
||||
No-op for static API keys or when refresh fails (fail-open: the stale token
|
||||
is left in place and the existing 401 handling degrades gracefully).
|
||||
"""
|
||||
try:
|
||||
from plugins.memory.honcho import oauth
|
||||
|
||||
token, _ = oauth.ensure_fresh_token(resolve_config_path(), config.host)
|
||||
if token:
|
||||
config.api_key = token
|
||||
except Exception:
|
||||
logger.warning("Honcho OAuth pre-build refresh failed", exc_info=True)
|
||||
|
||||
|
||||
def _refresh_cached_oauth(client: "Honcho", config: HonchoClientConfig | None) -> None:
|
||||
"""Rotate the cached client's Bearer in place when its OAuth token is stale.
|
||||
|
||||
If the SDK shape changed and the in-place rotation can't apply, the slot is
|
||||
reset so the next acquisition rebuilds with the fresh token.
|
||||
"""
|
||||
try:
|
||||
from plugins.memory.honcho import oauth
|
||||
|
||||
host = config.host if config is not None else resolve_active_host()
|
||||
token, refreshed = oauth.ensure_fresh_token(resolve_config_path(), host)
|
||||
if refreshed and token and not oauth.apply_token_to_client(client, token):
|
||||
_honcho_client_slot.reset()
|
||||
except Exception:
|
||||
logger.warning("Honcho OAuth cached refresh failed", exc_info=True)
|
||||
|
||||
|
||||
def get_honcho_client(config: HonchoClientConfig | None = None) -> Honcho:
|
||||
"""Get or create the Honcho client singleton.
|
||||
|
||||
When no config is provided, attempts to load ~/.honcho/config.json
|
||||
first, falling back to environment variables.
|
||||
|
||||
Thread-safe: the client is built exactly once even under concurrent
|
||||
first calls (double-checked locking via ``SingletonSlot``), so racing
|
||||
threads can't each construct a client and leak the loser's connection.
|
||||
"""
|
||||
cached = _honcho_client_slot.peek()
|
||||
if cached is not None:
|
||||
_refresh_cached_oauth(cached, config)
|
||||
return cached
|
||||
|
||||
if config is None:
|
||||
config = HonchoClientConfig.from_global_config()
|
||||
|
||||
# Refresh a near-expiry OAuth grant before the first build so the client
|
||||
# starts with a live access token rather than 401ing an hour in.
|
||||
_apply_fresh_oauth_token(config)
|
||||
|
||||
if not config.api_key and not config.base_url:
|
||||
raise ValueError(
|
||||
"Honcho API key not found. "
|
||||
"Get your API key at https://app.honcho.dev, "
|
||||
"then run 'hermes honcho setup' or set HONCHO_API_KEY. "
|
||||
"For local instances, set HONCHO_BASE_URL instead."
|
||||
)
|
||||
|
||||
# Everything below is the expensive part the issue flags: lazy SDK
|
||||
# install, config resolution, and client construction. Run it inside the
|
||||
# slot's factory so it executes exactly once even when several threads
|
||||
# race the first call — the slot's double-checked lock serializes them and
|
||||
# the losers get the winner's client instead of building their own.
|
||||
def _build() -> "Honcho":
|
||||
# Lazy-install the honcho SDK on demand. ensure() honors
|
||||
# security.allow_lazy_installs (default true). On failure we surface
|
||||
# the original ImportError-shape message so existing callers still get
|
||||
# the "go run hermes honcho setup" hint they used to.
|
||||
try:
|
||||
from tools.lazy_deps import FeatureUnavailable, ensure as _lazy_ensure
|
||||
_lazy_ensure("memory.honcho", prompt=False)
|
||||
except ImportError:
|
||||
# lazy_deps module missing — fall through to the raw import below.
|
||||
pass
|
||||
except Exception:
|
||||
# FeatureUnavailable or unexpected error. Don't crash here; let the
|
||||
# actual import attempt produce the canonical error message.
|
||||
pass
|
||||
|
||||
try:
|
||||
from honcho import Honcho
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"honcho-ai is required for Honcho integration. "
|
||||
"Install it with: pip install honcho-ai "
|
||||
"(or run `hermes honcho setup` to configure)."
|
||||
)
|
||||
|
||||
# Allow config.yaml honcho.base_url to override the SDK's environment
|
||||
# mapping, enabling remote self-hosted Honcho deployments without
|
||||
# requiring the server to live on localhost.
|
||||
resolved_base_url = config.base_url
|
||||
resolved_timeout = config.timeout
|
||||
if not resolved_base_url or resolved_timeout is None:
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
hermes_cfg = load_config()
|
||||
honcho_cfg = hermes_cfg.get("honcho", {})
|
||||
if isinstance(honcho_cfg, dict):
|
||||
if not resolved_base_url:
|
||||
resolved_base_url = honcho_cfg.get("base_url", "").strip() or None
|
||||
if resolved_timeout is None:
|
||||
resolved_timeout = _resolve_optional_float(
|
||||
honcho_cfg.get("timeout"),
|
||||
honcho_cfg.get("request_timeout"),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fall back to the default so an unconfigured install cannot hang
|
||||
# indefinitely on a stalled Honcho request.
|
||||
if resolved_timeout is None:
|
||||
resolved_timeout = _DEFAULT_HTTP_TIMEOUT
|
||||
|
||||
if resolved_base_url:
|
||||
logger.info("Initializing Honcho client (base_url: %s, workspace: %s)", resolved_base_url, config.workspace_id)
|
||||
else:
|
||||
logger.info("Initializing Honcho client (host: %s, workspace: %s)", config.host, config.workspace_id)
|
||||
|
||||
# Local Honcho instances don't require an API key, but the SDK
|
||||
# expects a non-empty string. Use a placeholder for local URLs.
|
||||
# For local: only use config.api_key if the host block explicitly
|
||||
# sets apiKey (meaning the user wants local auth). Otherwise skip
|
||||
# the stored key -- it's likely a cloud key that would break local.
|
||||
_is_local = resolved_base_url and (
|
||||
"localhost" in resolved_base_url
|
||||
or "127.0.0.1" in resolved_base_url
|
||||
or "::1" in resolved_base_url
|
||||
)
|
||||
if _is_local:
|
||||
# Check if the host block has its own apiKey (explicit local auth).
|
||||
# Auth-skipping is loopback-only: a stored key is likely a cloud key
|
||||
# that would break a no-auth local server, so we substitute the SDK's
|
||||
# required-non-empty placeholder unless the host block opts in.
|
||||
_raw = config.raw or {}
|
||||
_host_block = (_raw.get("hosts") or {}).get(config.host, {})
|
||||
_host_has_key = bool(_host_block.get("apiKey"))
|
||||
effective_api_key = config.api_key if _host_has_key else "local"
|
||||
else:
|
||||
effective_api_key = config.api_key
|
||||
|
||||
# The Honcho SDK's route builders (e.g. routes.workspaces()) already
|
||||
# include the version prefix (e.g. "/v3/workspaces"). When a user-supplied
|
||||
# base_url already ends in a version segment (e.g.
|
||||
# "http://localhost:38000/v3", "https://honcho.my.ts.net/v3"), concatenating
|
||||
# the two produces "/v3/v3/workspaces" → 404 on every call. This is a pure
|
||||
# routing concern independent of host, so strip a trailing version segment
|
||||
# from ANY base_url — loopback, LAN, custom domain, or cloud alike. The
|
||||
# SDK then appends its own versioned paths correctly.
|
||||
if resolved_base_url:
|
||||
import re as _re
|
||||
resolved_base_url = _re.sub(r"/v\d+/*$", "", resolved_base_url).rstrip("/")
|
||||
|
||||
kwargs: dict = {
|
||||
"workspace_id": config.workspace_id,
|
||||
"api_key": effective_api_key,
|
||||
"environment": config.environment,
|
||||
}
|
||||
if resolved_base_url:
|
||||
kwargs["base_url"] = resolved_base_url
|
||||
if resolved_timeout is not None:
|
||||
kwargs["timeout"] = resolved_timeout
|
||||
|
||||
return Honcho(**kwargs)
|
||||
|
||||
return _honcho_client_slot.get(_build)
|
||||
|
||||
|
||||
def reset_honcho_client() -> None:
|
||||
"""Reset the Honcho client singleton (useful for testing)."""
|
||||
_honcho_client_slot.reset()
|
||||
@@ -0,0 +1,371 @@
|
||||
"""OAuth credential storage and refresh for the Honcho memory provider.
|
||||
|
||||
An access token authenticates exactly like a scoped API key, so it is stored
|
||||
as the host's ``apiKey``; this module exchanges the refresh token before
|
||||
expiry to keep it live.
|
||||
|
||||
Refresh tokens rotate with single-use reuse detection: a replayed stale token
|
||||
revokes the whole grant. So every refresh must persist the rotated token
|
||||
atomically and be serialized — and a failed refresh never raises into the
|
||||
agent (stale token stays; the fail-open path absorbs the eventual 401).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ACCESS_TOKEN_PREFIX = "hch-at-"
|
||||
REFRESH_TOKEN_PREFIX = "hch-rt-"
|
||||
|
||||
# Refresh this many seconds before the access token actually expires, so an
|
||||
# in-flight request never races the expiry boundary.
|
||||
_REFRESH_SKEW_SECONDS = 120
|
||||
|
||||
# Default HTTP timeout for the token exchange. Kept short — the refresh happens
|
||||
# on the path to a memory call, and a stalled auth server must not hang it.
|
||||
_REFRESH_TIMEOUT_SECONDS = 15.0
|
||||
|
||||
# Serializes refresh across threads sharing one process's config. Re-checked
|
||||
# under the lock (double-checked) so racing callers don't replay a rotated
|
||||
# refresh token and trip reuse detection.
|
||||
_refresh_lock = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _config_refresh_lock(path: Path):
|
||||
"""Machine-wide advisory lock around read-refresh-persist.
|
||||
|
||||
The in-process ``_refresh_lock`` can't stop a second process (a sibling
|
||||
Hermes profile or the desktop app sharing this honcho.json) from replaying
|
||||
the single-use refresh token and tripping reuse-detection — which revokes
|
||||
the whole grant. An OS file lock on ``<config>.lock`` serializes rotation
|
||||
across processes; best-effort, so a platform without flock degrades to
|
||||
in-process serialization only.
|
||||
"""
|
||||
lock_path = Path(f"{path}.lock")
|
||||
fh = None
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fh = open(lock_path, "a+b")
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
fh.seek(0)
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
logger.debug("Honcho OAuth cross-process lock unavailable; in-process only", exc_info=True)
|
||||
if fh is not None:
|
||||
fh.close()
|
||||
fh = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if fh is not None:
|
||||
try:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
fh.seek(0)
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
|
||||
except Exception:
|
||||
pass
|
||||
fh.close()
|
||||
|
||||
# In-memory expiry cache keyed by (config path, host) → (expires_at, access).
|
||||
# Lets the hot path (every memory access calls this) skip the honcho.json read
|
||||
# while the token is comfortably live; disk is only touched near expiry, on a
|
||||
# cache miss, or when an explicit ``raw`` is supplied. Single-key dict ops are
|
||||
# atomic under the GIL, so no separate lock is needed. An access token stays
|
||||
# valid until its own expiry regardless of out-of-band rotation, so a stale
|
||||
# cache entry can't break auth — it just defers picking up external changes
|
||||
# until the token nears expiry and disk is read again.
|
||||
_expiry_cache: dict[tuple[str, str], tuple[float, str]] = {}
|
||||
|
||||
|
||||
def is_oauth_access_token(value: str | None) -> bool:
|
||||
"""True when ``value`` is an OAuth access token (vs a static API key)."""
|
||||
return bool(value) and value.startswith(ACCESS_TOKEN_PREFIX)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OAuthCredential:
|
||||
"""An OAuth grant as stored in a honcho.json host block.
|
||||
|
||||
``access_token`` mirrors the host's ``apiKey``; the remaining fields live in
|
||||
the host's ``oauth`` sub-block. ``expires_at`` is absolute epoch seconds.
|
||||
"""
|
||||
|
||||
access_token: str
|
||||
refresh_token: str
|
||||
expires_at: float
|
||||
client_id: str
|
||||
token_endpoint: str
|
||||
scope: str = "write"
|
||||
token_type: str = "Bearer"
|
||||
# Transient consent peer name — set only on a fresh grant, never persisted.
|
||||
consent_peer_name: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_host_block(cls, block: dict[str, Any]) -> "OAuthCredential | None":
|
||||
"""Build a credential from a honcho.json host block, or None if incomplete."""
|
||||
oauth = block.get("oauth")
|
||||
access = block.get("apiKey")
|
||||
if not isinstance(oauth, dict) or not is_oauth_access_token(access):
|
||||
return None
|
||||
refresh = oauth.get("refreshToken")
|
||||
endpoint = oauth.get("tokenEndpoint")
|
||||
client_id = oauth.get("clientId")
|
||||
if not (refresh and endpoint and client_id):
|
||||
return None
|
||||
try:
|
||||
expires_at = float(oauth.get("expiresAt", 0))
|
||||
except (TypeError, ValueError):
|
||||
expires_at = 0.0
|
||||
return cls(
|
||||
access_token=access,
|
||||
refresh_token=str(refresh),
|
||||
expires_at=expires_at,
|
||||
client_id=str(client_id),
|
||||
token_endpoint=str(endpoint),
|
||||
scope=str(oauth.get("scope", "write")),
|
||||
token_type=str(oauth.get("tokenType", "Bearer")),
|
||||
)
|
||||
|
||||
def oauth_block(self) -> dict[str, Any]:
|
||||
"""The ``oauth`` sub-block to persist (the access token lives in apiKey)."""
|
||||
return {
|
||||
"refreshToken": self.refresh_token,
|
||||
"expiresAt": int(self.expires_at),
|
||||
"clientId": self.client_id,
|
||||
"tokenEndpoint": self.token_endpoint,
|
||||
"scope": self.scope,
|
||||
"tokenType": self.token_type,
|
||||
}
|
||||
|
||||
def is_expired(self, *, now: float, skew: float = _REFRESH_SKEW_SECONDS) -> bool:
|
||||
"""True when the access token is within ``skew`` seconds of expiry."""
|
||||
return now >= (self.expires_at - skew)
|
||||
|
||||
|
||||
# Indirection so tests can drive the exchange without a live server.
|
||||
def _http_post_form(url: str, data: dict[str, str], timeout: float) -> dict[str, Any]:
|
||||
"""POST form-encoded ``data`` to ``url`` and return the parsed JSON body."""
|
||||
import httpx
|
||||
|
||||
resp = httpx.post(url, data=data, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCredential:
|
||||
"""Run the refresh_token grant and return the rotated credential.
|
||||
|
||||
Raises on any transport/protocol failure; callers fail open.
|
||||
"""
|
||||
body = _http_post_form(
|
||||
cred.token_endpoint,
|
||||
{
|
||||
"grant_type": "refresh_token",
|
||||
"client_id": cred.client_id,
|
||||
"refresh_token": cred.refresh_token,
|
||||
},
|
||||
_REFRESH_TIMEOUT_SECONDS,
|
||||
)
|
||||
access = body.get("access_token")
|
||||
refresh = body.get("refresh_token")
|
||||
if not is_oauth_access_token(access) or not refresh:
|
||||
raise ValueError("refresh response missing access_token/refresh_token")
|
||||
try:
|
||||
expires_in = int(body.get("expires_in", 0))
|
||||
except (TypeError, ValueError):
|
||||
expires_in = 0
|
||||
return OAuthCredential(
|
||||
access_token=access,
|
||||
refresh_token=str(refresh),
|
||||
expires_at=now + expires_in,
|
||||
client_id=cred.client_id,
|
||||
token_endpoint=cred.token_endpoint,
|
||||
scope=str(body.get("scope", cred.scope)),
|
||||
token_type=str(body.get("token_type", cred.token_type)),
|
||||
)
|
||||
|
||||
|
||||
def _read_config(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _atomic_write_config(path: Path, raw: dict[str, Any]) -> None:
|
||||
"""Write ``raw`` to ``path`` atomically, preserving 0600 on the new file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".{path.name}.tmp")
|
||||
text = json.dumps(raw, indent=2) + "\n"
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
except Exception:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Recursively merge ``overlay`` into ``base`` (overlay wins on scalars/lists)."""
|
||||
for key, value in overlay.items():
|
||||
if isinstance(value, dict) and isinstance(base.get(key), dict):
|
||||
_deep_merge(base[key], value)
|
||||
else:
|
||||
base[key] = value
|
||||
return base
|
||||
|
||||
|
||||
def _persist_credential(path: Path, host: str, cred: OAuthCredential) -> None:
|
||||
"""Persist ``cred`` into ``host``'s block (apiKey + oauth), leaving all else intact."""
|
||||
raw = _read_config(path)
|
||||
hosts = raw.setdefault("hosts", {})
|
||||
block = hosts.setdefault(host, {})
|
||||
block["apiKey"] = cred.access_token
|
||||
block["oauth"] = cred.oauth_block()
|
||||
_atomic_write_config(path, raw)
|
||||
_expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token)
|
||||
|
||||
|
||||
def ensure_fresh_token(
|
||||
path: Path,
|
||||
host: str,
|
||||
raw: dict[str, Any] | None = None,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> tuple[str | None, bool]:
|
||||
"""Return ``(access_token, refreshed)`` for ``host``, refreshing if near expiry.
|
||||
|
||||
Returns ``(None, False)`` when the host has no OAuth credential (e.g. a plain
|
||||
API key) so callers leave the existing token untouched. Refresh failures are
|
||||
swallowed: the current (possibly stale) token is returned with
|
||||
``refreshed=False`` and the fail-open path handles any resulting 401.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
key = (str(path), host)
|
||||
|
||||
# Hot path: trust the cached expiry while the token is well clear of the
|
||||
# skew window — no disk read. Bypassed when an explicit ``raw`` is supplied.
|
||||
if raw is None:
|
||||
cached = _expiry_cache.get(key)
|
||||
if cached is not None and now < cached[0] - _REFRESH_SKEW_SECONDS:
|
||||
return cached[1], False
|
||||
|
||||
source = raw if raw is not None else _read_config(path)
|
||||
block = (source.get("hosts") or {}).get(host) or {}
|
||||
cred = OAuthCredential.from_host_block(block)
|
||||
if cred is None:
|
||||
_expiry_cache.pop(key, None)
|
||||
return None, False
|
||||
|
||||
_expiry_cache[key] = (cred.expires_at, cred.access_token)
|
||||
if not cred.is_expired(now=now):
|
||||
return cred.access_token, False
|
||||
|
||||
with _refresh_lock, _config_refresh_lock(path):
|
||||
# Re-read under both locks: another thread or process may have just
|
||||
# rotated the token — adopt theirs instead of replaying the old one.
|
||||
fresh_block = (_read_config(path).get("hosts") or {}).get(host) or {}
|
||||
current = OAuthCredential.from_host_block(fresh_block) or cred
|
||||
if not current.is_expired(now=now):
|
||||
return current.access_token, current.access_token != cred.access_token
|
||||
try:
|
||||
rotated = _exchange_refresh_token(current, now=now)
|
||||
except Exception as exc:
|
||||
logger.warning("Honcho OAuth refresh failed for host %s: %s", host, exc)
|
||||
return current.access_token, False
|
||||
_persist_credential(path, host, rotated)
|
||||
logger.info("Honcho OAuth token refreshed for host %s", host)
|
||||
return rotated.access_token, True
|
||||
|
||||
|
||||
def install_grant(
|
||||
path: Path,
|
||||
host: str,
|
||||
grant: dict[str, Any],
|
||||
*,
|
||||
client_id: str,
|
||||
token_endpoint: str,
|
||||
apply_config: bool = True,
|
||||
now: float | None = None,
|
||||
) -> OAuthCredential:
|
||||
"""Apply a fresh OAuth grant to ``path`` for ``host``.
|
||||
|
||||
Deep-merges the grant's ``config`` (the manifest default_config) into the
|
||||
file root — preserving other hosts and root keys — then writes the host's
|
||||
``apiKey`` and ``oauth`` block. ``grant`` is an OAuthTokenResponse dict
|
||||
(access_token, refresh_token, expires_in, scope, config).
|
||||
``apply_config=False`` skips the config merge and stores tokens only.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
access = grant.get("access_token")
|
||||
refresh = grant.get("refresh_token")
|
||||
if not is_oauth_access_token(access) or not refresh:
|
||||
raise ValueError("grant missing access_token/refresh_token")
|
||||
try:
|
||||
expires_in = int(grant.get("expires_in", 0))
|
||||
except (TypeError, ValueError):
|
||||
expires_in = 0
|
||||
|
||||
cred = OAuthCredential(
|
||||
access_token=access,
|
||||
refresh_token=str(refresh),
|
||||
expires_at=now + expires_in,
|
||||
client_id=client_id,
|
||||
token_endpoint=token_endpoint,
|
||||
scope=str(grant.get("scope", "write")),
|
||||
token_type=str(grant.get("token_type", "Bearer")),
|
||||
)
|
||||
|
||||
raw = _read_config(path)
|
||||
granted_config = grant.get("config")
|
||||
if isinstance(granted_config, dict):
|
||||
cred.consent_peer_name = granted_config.get("peerName")
|
||||
if apply_config:
|
||||
_deep_merge(raw, granted_config)
|
||||
_expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token)
|
||||
hosts = raw.setdefault("hosts", {})
|
||||
block = hosts.setdefault(host, {})
|
||||
block["apiKey"] = cred.access_token
|
||||
block["oauth"] = cred.oauth_block()
|
||||
_atomic_write_config(path, raw)
|
||||
return cred
|
||||
|
||||
|
||||
def apply_token_to_client(client: Any, token: str) -> bool:
|
||||
"""Rotate the live Honcho client's Bearer in place. Returns success.
|
||||
|
||||
The SDK builds its auth header per request from the HTTP client's
|
||||
``api_key``, so mutating it rotates every holder of the singleton without a
|
||||
rebuild. Guarded: an SDK shape change degrades to False and the caller can
|
||||
fall back to resetting the client.
|
||||
"""
|
||||
http = getattr(client, "_http", None)
|
||||
if http is None or not hasattr(http, "api_key"):
|
||||
return False
|
||||
http.api_key = token
|
||||
return True
|
||||
@@ -0,0 +1,431 @@
|
||||
"""Browser sign-in flow for the Honcho memory provider — no CLI step.
|
||||
|
||||
``begin_authorization`` / ``complete_authorization`` are the transport-agnostic
|
||||
core: the code can arrive via the loopback listener here or a future
|
||||
``hermes://`` handler. Endpoints are env-overridable with local-dev defaults
|
||||
because ``/authorize`` (dashboard) and ``/oauth/token`` (API) live on
|
||||
different origins.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
||||
from plugins.memory.honcho import oauth
|
||||
from plugins.memory.honcho.client import resolve_active_host, resolve_config_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The loopback redirect registered for the Hermes OAuth client. IP-literal so
|
||||
# the browser can't resolve the advertised host to ::1 and miss the IPv4 bind.
|
||||
LOOPBACK_HOST = "127.0.0.1"
|
||||
LOOPBACK_PORT = 8765
|
||||
LOOPBACK_REDIRECT_URI = f"http://{LOOPBACK_HOST}:{LOOPBACK_PORT}/callback"
|
||||
|
||||
# Pending authorizations live only until their callback returns; keyed by the
|
||||
# CSRF ``state`` so a stray/forged callback can't complete a grant.
|
||||
_PENDING_TTL_SECONDS = 600
|
||||
|
||||
|
||||
def _display_config_path(path: object) -> str:
|
||||
"""Home-relative display string for the consent screen.
|
||||
|
||||
The absolute path (username + home layout) never leaves the machine — it's
|
||||
only shown to the user. Collapse ``$HOME`` to ``~``; for a path outside
|
||||
home, send the bare filename rather than leak an arbitrary absolute path.
|
||||
"""
|
||||
from pathlib import Path as _Path
|
||||
|
||||
p = _Path(str(path))
|
||||
try:
|
||||
return "~/" + str(p.relative_to(_Path.home()))
|
||||
except ValueError:
|
||||
return p.name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OAuthEndpoints:
|
||||
"""Resolved authorization-server URLs and client identity."""
|
||||
|
||||
authorize_url: str # dashboard /authorize
|
||||
token_url: str # API /oauth/token
|
||||
client_id: str
|
||||
scope: str
|
||||
|
||||
|
||||
# Cloud (production) hosts; dashboard serves /authorize, API serves /oauth/token.
|
||||
_CLOUD_DASHBOARD = "https://app.honcho.dev"
|
||||
_CLOUD_TOKEN_URL = "https://api.honcho.dev/oauth/token"
|
||||
_LOCAL_DASHBOARD = "http://localhost:3000"
|
||||
_LOCAL_TOKEN_URL = "http://localhost:8000/oauth/token"
|
||||
|
||||
# One OAuth client for every surface. Consent branding/UI adapt via the
|
||||
# ``source`` query param (not a separate client_id), so there's a single grant
|
||||
# identity to refresh — no clientId-vs-refresh-token desync to revoke the grant.
|
||||
_DEFAULT_CLIENT_ID = "hermes-agent"
|
||||
|
||||
|
||||
def _is_loopback_url(url: str | None) -> bool:
|
||||
return bool(url) and any(h in url for h in ("localhost", "127.0.0.1", "::1"))
|
||||
|
||||
|
||||
def resolve_endpoints(
|
||||
environment: str | None = None, base_url: str | None = None
|
||||
) -> OAuthEndpoints:
|
||||
"""Resolve OAuth endpoints, zero-config by default.
|
||||
|
||||
Keys off the host's honcho ``environment`` (production → cloud, local →
|
||||
localhost); a self-hosted ``base_url`` derives the token endpoint from the
|
||||
API host. Env vars override every field for unusual deployments.
|
||||
"""
|
||||
if environment is None or base_url is None:
|
||||
try:
|
||||
from plugins.memory.honcho.client import HonchoClientConfig
|
||||
|
||||
cfg = HonchoClientConfig.from_global_config()
|
||||
environment = environment or cfg.environment
|
||||
base_url = base_url if base_url is not None else cfg.base_url
|
||||
except Exception:
|
||||
environment = environment or "production"
|
||||
|
||||
is_local = (environment or "").lower() == "local" or _is_loopback_url(base_url)
|
||||
default_dashboard = _LOCAL_DASHBOARD if is_local else _CLOUD_DASHBOARD
|
||||
default_token = _LOCAL_TOKEN_URL if is_local else _CLOUD_TOKEN_URL
|
||||
# Self-hosted API (non-loopback base_url): token rides the same host.
|
||||
if base_url and not is_local:
|
||||
default_token = f"{base_url.rstrip('/')}/oauth/token"
|
||||
|
||||
dashboard = os.environ.get("HONCHO_OAUTH_DASHBOARD", default_dashboard).rstrip("/")
|
||||
return OAuthEndpoints(
|
||||
authorize_url=os.environ.get("HONCHO_OAUTH_AUTHORIZE_URL", f"{dashboard}/authorize"),
|
||||
token_url=os.environ.get("HONCHO_OAUTH_TOKEN_URL", default_token),
|
||||
client_id=os.environ.get("HONCHO_OAUTH_CLIENT_ID", _DEFAULT_CLIENT_ID),
|
||||
scope=os.environ.get("HONCHO_OAUTH_SCOPE", "write"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pending:
|
||||
verifier: str
|
||||
redirect_uri: str
|
||||
created_at: float
|
||||
|
||||
|
||||
_pending: dict[str, _Pending] = {}
|
||||
_pending_lock = threading.Lock()
|
||||
|
||||
|
||||
def _pkce() -> tuple[str, str]:
|
||||
"""Return (verifier, S256 challenge) for an authorization-code request."""
|
||||
verifier = secrets.token_urlsafe(64)
|
||||
challenge = (
|
||||
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
|
||||
.rstrip(b"=")
|
||||
.decode()
|
||||
)
|
||||
return verifier, challenge
|
||||
|
||||
|
||||
def _prune_pending(now: float) -> None:
|
||||
expired = [s for s, p in _pending.items() if now - p.created_at > _PENDING_TTL_SECONDS]
|
||||
for state in expired:
|
||||
_pending.pop(state, None)
|
||||
|
||||
|
||||
def begin_authorization(
|
||||
endpoints: OAuthEndpoints,
|
||||
redirect_uri: str = LOOPBACK_REDIRECT_URI,
|
||||
*,
|
||||
source: str | None = None,
|
||||
config_path: str | None = None,
|
||||
now: float | None = None,
|
||||
) -> tuple[str, str]:
|
||||
"""Start an authorization: return ``(authorize_url, state)`` and stash PKCE.
|
||||
|
||||
``source`` tags the authorize link with the initiating surface
|
||||
(``hermes-desktop`` / ``hermes-cli``) so the consent side can attribute
|
||||
connects and vary behavior per surface. ``config_path`` is a home-relative
|
||||
*display* string for the consent screen (never the absolute path); callers
|
||||
pass the actual write path separately to ``complete_authorization``.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
verifier, challenge = _pkce()
|
||||
state = secrets.token_urlsafe(32)
|
||||
with _pending_lock:
|
||||
_prune_pending(now)
|
||||
_pending[state] = _Pending(verifier=verifier, redirect_uri=redirect_uri, created_at=now)
|
||||
params = {
|
||||
"client_id": endpoints.client_id,
|
||||
"redirect_uri": redirect_uri,
|
||||
"scope": endpoints.scope,
|
||||
"code_challenge": challenge,
|
||||
"code_challenge_method": "S256",
|
||||
"response_type": "code",
|
||||
"state": state,
|
||||
}
|
||||
if source:
|
||||
params["source"] = source
|
||||
if config_path:
|
||||
params["config_path"] = config_path
|
||||
return f"{endpoints.authorize_url}?{urlencode(params)}", state
|
||||
|
||||
|
||||
def complete_authorization(
|
||||
endpoints: OAuthEndpoints,
|
||||
code: str,
|
||||
state: str,
|
||||
*,
|
||||
config_path: Path | None = None,
|
||||
host: str | None = None,
|
||||
apply_config: bool = True,
|
||||
now: float | None = None,
|
||||
) -> oauth.OAuthCredential:
|
||||
"""Exchange ``code`` for a grant and persist it. Raises on bad state/exchange.
|
||||
|
||||
``apply_config=False`` stores the tokens only, skipping the grant's config
|
||||
block — the CLI path, where settings stay wizard-owned.
|
||||
"""
|
||||
with _pending_lock:
|
||||
pending = _pending.pop(state, None)
|
||||
if pending is None:
|
||||
raise ValueError("unknown or expired authorization state")
|
||||
|
||||
grant = oauth._http_post_form(
|
||||
endpoints.token_url,
|
||||
{
|
||||
"grant_type": "authorization_code",
|
||||
"client_id": endpoints.client_id,
|
||||
"code": code,
|
||||
"redirect_uri": pending.redirect_uri,
|
||||
"code_verifier": pending.verifier,
|
||||
},
|
||||
oauth._REFRESH_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
path = config_path or resolve_config_path()
|
||||
target_host = host or resolve_active_host()
|
||||
cred = oauth.install_grant(
|
||||
path,
|
||||
target_host,
|
||||
grant,
|
||||
client_id=endpoints.client_id,
|
||||
token_endpoint=endpoints.token_url,
|
||||
apply_config=apply_config,
|
||||
now=now,
|
||||
)
|
||||
# Drop the singleton so the next acquisition builds with the new token.
|
||||
from plugins.memory.honcho.client import reset_honcho_client
|
||||
|
||||
reset_honcho_client()
|
||||
logger.info("Honcho OAuth grant installed for host %s", target_host)
|
||||
return cred
|
||||
|
||||
|
||||
_CALLBACK_HTML = (
|
||||
b"<!doctype html><meta charset=utf-8>"
|
||||
b"<title>Honcho connected</title>"
|
||||
b"<body style='font:14px ui-monospace,monospace;background:#0b0e14;color:#c9d1d9;"
|
||||
b"display:flex;align-items:center;justify-content:center;height:100vh;margin:0'>"
|
||||
b"<div>Connected to Honcho. You can close this tab and return to Hermes.</div>"
|
||||
)
|
||||
|
||||
|
||||
def _bind_loopback_server() -> tuple[HTTPServer, dict[str, str]]:
|
||||
"""Bind the one-shot callback server, returning it and its capture dict.
|
||||
|
||||
Prefers :8765; if that's taken, falls back to an OS-assigned port. groudon's
|
||||
redirect matcher relaxes the port for loopback hosts, so the fallback still
|
||||
matches the seeded ``127.0.0.1`` redirect URI — the caller advertises the
|
||||
actual bound port.
|
||||
"""
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self): # noqa: N802 - stdlib API name
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path != "/callback":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
params = parse_qs(parsed.query)
|
||||
captured["code"] = (params.get("code") or [""])[0]
|
||||
captured["state"] = (params.get("state") or [""])[0]
|
||||
captured["error"] = (params.get("error") or [""])[0]
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(_CALLBACK_HTML)
|
||||
|
||||
def log_message(self, *args): # silence stdlib request logging
|
||||
return
|
||||
|
||||
try:
|
||||
server = HTTPServer((LOOPBACK_HOST, LOOPBACK_PORT), _Handler)
|
||||
except OSError:
|
||||
server = HTTPServer((LOOPBACK_HOST, 0), _Handler) # OS-assigned fallback
|
||||
return server, captured
|
||||
|
||||
|
||||
def capture_loopback_code(
|
||||
server: HTTPServer, captured: dict[str, str], *, timeout: float = 300.0
|
||||
) -> tuple[str, str]:
|
||||
"""Serve a single ``/callback`` GET on ``server`` and return ``(code, state)``.
|
||||
|
||||
Replies with a close-this-tab page, then stops. Raises ``TimeoutError`` if no
|
||||
callback arrives within ``timeout``.
|
||||
"""
|
||||
server.timeout = timeout
|
||||
try:
|
||||
# handle_request honors server.timeout; loop until our callback lands so a
|
||||
# stray probe to another path doesn't end the wait empty-handed.
|
||||
deadline = time.monotonic() + timeout
|
||||
while "code" not in captured and time.monotonic() < deadline:
|
||||
server.handle_request()
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
if captured.get("error"):
|
||||
raise ValueError(f"authorization denied: {captured['error']}")
|
||||
if "code" not in captured:
|
||||
raise TimeoutError("no OAuth callback received before timeout")
|
||||
return captured["code"], captured.get("state", "")
|
||||
|
||||
|
||||
def authorize_via_loopback(
|
||||
*,
|
||||
config_path: Path | None = None,
|
||||
host: str | None = None,
|
||||
source: str | None = None,
|
||||
apply_config: bool = True,
|
||||
open_url: Callable[[str], None] | None = None,
|
||||
timeout: float = 300.0,
|
||||
) -> oauth.OAuthCredential:
|
||||
"""Drive the full loopback flow: open browser → capture code → exchange → persist.
|
||||
|
||||
``open_url`` defaults to the system browser; tests inject a driver that
|
||||
follows the authorize redirect into the loopback callback. It always
|
||||
receives the authorize URL, so a CLI caller can also print it for
|
||||
browserless environments.
|
||||
"""
|
||||
# Bind first so the advertised redirect_uri carries the actual bound port
|
||||
# (which may differ from :8765 if it was taken).
|
||||
server, captured = _bind_loopback_server()
|
||||
redirect_uri = f"http://{LOOPBACK_HOST}:{server.server_address[1]}/callback"
|
||||
|
||||
endpoints = resolve_endpoints()
|
||||
path = config_path or resolve_config_path()
|
||||
authorize_url, state = begin_authorization(
|
||||
endpoints, redirect_uri, source=source, config_path=_display_config_path(path)
|
||||
)
|
||||
|
||||
if open_url is None:
|
||||
import webbrowser
|
||||
|
||||
open_url = webbrowser.open
|
||||
|
||||
# Browser opens from a short-lived thread; the socket is already bound, so a
|
||||
# fast redirect can't beat it.
|
||||
opener = threading.Thread(target=lambda: open_url(authorize_url), daemon=True)
|
||||
opener.start()
|
||||
|
||||
code, returned_state = capture_loopback_code(server, captured, timeout=timeout)
|
||||
if returned_state != state:
|
||||
raise ValueError("OAuth state mismatch — possible CSRF, aborting")
|
||||
return complete_authorization(
|
||||
endpoints,
|
||||
code,
|
||||
returned_state,
|
||||
config_path=path,
|
||||
host=host,
|
||||
apply_config=apply_config,
|
||||
)
|
||||
|
||||
|
||||
# — Background launcher + status, for the desktop "Connect" button —
|
||||
# The flow blocks on a browser round-trip, so the web_server endpoint kicks it
|
||||
# off in a thread and the UI polls status rather than holding the request open.
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlowStatus:
|
||||
state: str = "idle" # idle | pending | connected | error
|
||||
detail: str = ""
|
||||
|
||||
|
||||
_status = FlowStatus()
|
||||
_status_lock = threading.Lock()
|
||||
_flow_thread: threading.Thread | None = None
|
||||
|
||||
|
||||
def _detect_connection() -> tuple[bool, str | None]:
|
||||
"""Report whether a credential is already stored: 'oauth', 'apikey', or none."""
|
||||
try:
|
||||
from plugins.memory.honcho.client import HonchoClientConfig
|
||||
|
||||
cfg = HonchoClientConfig.from_global_config()
|
||||
block = (cfg.raw.get("hosts") or {}).get(cfg.host) or {}
|
||||
if oauth.OAuthCredential.from_host_block(block) is not None:
|
||||
return True, "oauth"
|
||||
if cfg.api_key:
|
||||
return True, "apikey"
|
||||
except Exception:
|
||||
pass
|
||||
return False, None
|
||||
|
||||
|
||||
def get_flow_status() -> dict[str, object]:
|
||||
with _status_lock:
|
||||
state, detail = _status.state, _status.detail
|
||||
connected, auth = _detect_connection()
|
||||
return {"state": state, "detail": detail, "connected": connected, "auth": auth}
|
||||
|
||||
|
||||
def _set_status(state: str, detail: str = "") -> None:
|
||||
with _status_lock:
|
||||
_status.state, _status.detail = state, detail
|
||||
|
||||
|
||||
def start_loopback_flow_background(
|
||||
*,
|
||||
config_path: Path | None = None,
|
||||
host: str | None = None,
|
||||
source: str = "hermes-desktop",
|
||||
timeout: float = 300.0,
|
||||
) -> dict[str, str]:
|
||||
"""Launch the loopback flow in a daemon thread; returns the initial status.
|
||||
|
||||
Idempotent while a flow is pending — a second call is a no-op so a
|
||||
double-clicked button can't open two browser tabs / bind :8765 twice.
|
||||
"""
|
||||
global _flow_thread
|
||||
# Resolve under the caller's profile scope NOW — the worker thread outlives
|
||||
# the request, where a context-local HERMES_HOME override can't reach.
|
||||
config_path = config_path or resolve_config_path()
|
||||
host = host or resolve_active_host()
|
||||
with _status_lock:
|
||||
if _status.state == "pending" and _flow_thread and _flow_thread.is_alive():
|
||||
return {"state": _status.state, "detail": _status.detail}
|
||||
_status.state, _status.detail = "pending", "waiting for browser consent"
|
||||
|
||||
def _run() -> None:
|
||||
try:
|
||||
authorize_via_loopback(config_path=config_path, host=host, source=source, timeout=timeout)
|
||||
_set_status("connected", "Honcho connected")
|
||||
except Exception as exc:
|
||||
logger.warning("Honcho OAuth loopback flow failed: %s", exc)
|
||||
_set_status("error", str(exc))
|
||||
|
||||
_flow_thread = threading.Thread(target=_run, name="honcho-oauth-loopback", daemon=True)
|
||||
_flow_thread.start()
|
||||
return get_flow_status()
|
||||
@@ -0,0 +1,7 @@
|
||||
name: honcho
|
||||
version: 1.0.0
|
||||
description: "Honcho AI-native memory — cross-session user modeling with dialectic Q&A, semantic search, and persistent conclusions."
|
||||
pip_dependencies:
|
||||
- honcho-ai
|
||||
hooks:
|
||||
- on_session_end
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,187 @@
|
||||
# Mem0 Memory Provider
|
||||
|
||||
Server-side LLM fact extraction with semantic search and hybrid multi-signal retrieval via the Mem0 Platform v3 API.
|
||||
|
||||
## Requirements
|
||||
|
||||
- `pip install mem0ai`
|
||||
- Mem0 API key from [app.mem0.ai](https://app.mem0.ai)
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "mem0"
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider mem0
|
||||
echo "MEM0_API_KEY=your-key" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
Behavioral settings live in `$HERMES_HOME/mem0.json` (set them via `hermes memory setup`). Only the secret `MEM0_API_KEY` belongs in `~/.hermes/.env`.
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `mode` | `platform` | `platform` (Mem0 Cloud) or `oss` (self-managed, in-process) |
|
||||
| `host` | — | Self-hosted Mem0 server URL (the Docker dashboard). When set, connects over HTTP with `X-API-Key`. Don't combine with `mode: oss` |
|
||||
| `user_id` | `hermes-user` | User identifier on Mem0 |
|
||||
| `agent_id` | `hermes` | Agent identifier |
|
||||
| `rerank` | `false` | Rerank search results for relevance (platform mode only) |
|
||||
|
||||
The plugin has three connection modes:
|
||||
|
||||
- **Platform** — Mem0's hosted cloud (`api.mem0.ai`). Set `MEM0_API_KEY`. (default)
|
||||
- **Self-hosted dashboard** — a Mem0 server you run yourself via Docker. Set `host`. See below.
|
||||
- **OSS** — run Mem0 in-process with your own LLM + vector store. Set `mode: oss`. See below.
|
||||
|
||||
## Self-Hosted Dashboard (Server) Mode
|
||||
|
||||
Connect the plugin to a standalone Mem0 server you run yourself — the Docker-shipped Mem0 dashboard/server with its own REST API. Unlike OSS mode (which runs `mem0ai` in-process with your own vector store), here the plugin just talks HTTP to your server.
|
||||
|
||||
1. Run the Mem0 server (FastAPI + pgvector) from its Docker image and note its URL and `ADMIN_API_KEY`.
|
||||
2. Point the plugin at it — via the setup wizard:
|
||||
```bash
|
||||
hermes memory setup # select "mem0" → "Self-hosted server"
|
||||
# Or non-interactive:
|
||||
hermes memory setup mem0 --mode selfhosted --host http://localhost:8888 --api-key your-admin-api-key
|
||||
```
|
||||
or via env vars:
|
||||
```bash
|
||||
echo "MEM0_HOST=http://localhost:8888" >> ~/.hermes/.env
|
||||
echo "MEM0_API_KEY=your-admin-api-key" >> ~/.hermes/.env
|
||||
```
|
||||
or in `$HERMES_HOME/mem0.json`:
|
||||
```json
|
||||
{
|
||||
"host": "http://localhost:8888",
|
||||
"api_key": "your-admin-api-key"
|
||||
}
|
||||
```
|
||||
3. Start a fresh Hermes session and call `mem0_search` — it connects to your server.
|
||||
|
||||
The plugin authenticates with `X-API-Key` and uses the server's `/search` and `/memories` routes. `api_key` is optional — omit it only for servers running with `AUTH_DISABLED`.
|
||||
|
||||
> Setting `host` routes to the self-hosted server automatically. Don't set `mode: oss` — OSS takes precedence and ignores `host`.
|
||||
|
||||
## OSS (Self-Hosted) Mode
|
||||
|
||||
Run Mem0 locally with your own LLM, embedder, and vector store. This is the in-process SDK mode. To instead connect to a Mem0 server you run via Docker, see [Self-Hosted Dashboard (Server) Mode](#self-hosted-dashboard-server-mode) above.
|
||||
|
||||
### Interactive Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup
|
||||
# Select "mem0" → "Open Source (self-hosted)"
|
||||
# Follow prompts for LLM, embedder, and vector store
|
||||
```
|
||||
|
||||
### Agent-Driven Setup (Flags)
|
||||
|
||||
```bash
|
||||
hermes memory setup mem0 --mode oss \
|
||||
--oss-llm openai --oss-llm-key sk-... \
|
||||
--oss-vector qdrant
|
||||
```
|
||||
|
||||
### Supported Providers
|
||||
|
||||
| Component | Providers |
|
||||
|-----------|-----------|
|
||||
| LLM | openai, ollama |
|
||||
| Embedder | openai, ollama |
|
||||
| Vector Store | qdrant (local/server), pgvector |
|
||||
|
||||
### Flags Reference
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--mode` | `platform` or `oss` |
|
||||
| `--oss-llm` | LLM provider (default: openai) |
|
||||
| `--oss-llm-key` | LLM API key |
|
||||
| `--oss-embedder` | Embedder provider (default: openai) |
|
||||
| `--oss-vector` | Vector store (default: qdrant) |
|
||||
| `--oss-vector-path` | Qdrant local path |
|
||||
| `--user-id` | User identifier |
|
||||
|
||||
## Switching Modes
|
||||
|
||||
### Platform to OSS
|
||||
|
||||
```bash
|
||||
hermes memory setup mem0 --mode oss --oss-llm-key sk-...
|
||||
```
|
||||
|
||||
Or edit `$HERMES_HOME/mem0.json` directly:
|
||||
```json
|
||||
{
|
||||
"mode": "oss",
|
||||
"oss": {
|
||||
"llm": {"provider": "openai", "config": {"model": "gpt-5-mini"}},
|
||||
"embedder": {"provider": "openai", "config": {"model": "text-embedding-3-small"}},
|
||||
"vector_store": {"provider": "qdrant", "config": {"path": "~/.hermes/mem0_qdrant"}}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### OSS to Platform
|
||||
|
||||
```bash
|
||||
hermes memory setup mem0 --mode platform --api-key sk-...
|
||||
```
|
||||
|
||||
### Dry Run (preview without writing)
|
||||
|
||||
```bash
|
||||
hermes memory setup mem0 --mode oss --oss-llm-key sk-... --dry-run
|
||||
```
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `mem0_search` | Semantic search by meaning |
|
||||
| `mem0_add` | Store a fact verbatim (no LLM extraction) |
|
||||
| `mem0_update` | Update a memory's text by ID |
|
||||
| `mem0_delete` | Delete a memory by ID |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Mem0 temporarily unavailable"
|
||||
|
||||
Circuit breaker tripped after 5 consecutive failures. Resets after 2 minutes.
|
||||
|
||||
- **Platform mode**: Check API key and internet connectivity.
|
||||
- **OSS mode**: Check that your vector store (qdrant/pgvector) is running.
|
||||
|
||||
### OSS: Qdrant connection refused
|
||||
|
||||
```bash
|
||||
# If using local Qdrant, check the storage path is writable:
|
||||
ls -la ~/.hermes/mem0_qdrant
|
||||
|
||||
# If using Qdrant server, check it's reachable:
|
||||
curl http://localhost:6333/healthz
|
||||
```
|
||||
|
||||
### OSS: PGVector connection refused
|
||||
|
||||
```bash
|
||||
# Verify PostgreSQL is running and accepting connections:
|
||||
pg_isready -h localhost -p 5432
|
||||
```
|
||||
|
||||
### OSS: Ollama not reachable
|
||||
|
||||
```bash
|
||||
# Check Ollama is running:
|
||||
curl http://localhost:11434/api/tags
|
||||
```
|
||||
|
||||
### Memories not appearing
|
||||
|
||||
- `mem0_add` stores verbatim (no extraction). Use `sync_turn` for LLM extraction.
|
||||
- Search uses semantic matching — try broader queries.
|
||||
- Check `user_id` matches between sessions (`$HERMES_HOME/mem0.json`).
|
||||
@@ -0,0 +1,627 @@
|
||||
"""Mem0 memory plugin — MemoryProvider interface.
|
||||
|
||||
Server-side LLM fact extraction, semantic search, and automatic deduplication
|
||||
via the Mem0 Platform API (cloud) or OSS (self-hosted) via Memory.
|
||||
|
||||
Original PR #2933 by kartik-mem0, adapted to MemoryProvider ABC.
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
Secret (lives in $HERMES_HOME/.env or the environment):
|
||||
MEM0_API_KEY — Mem0 Platform API key (required for platform mode)
|
||||
MEM0_HOST — Base URL of a self-hosted Mem0 server. When set, the
|
||||
plugin talks to that server directly over HTTP
|
||||
(X-API-Key auth) instead of the cloud API.
|
||||
|
||||
Behavioral settings (live in $HERMES_HOME/mem0.json, set via `hermes memory
|
||||
setup`):
|
||||
mode — Backend mode: "platform" (default) or "oss"
|
||||
host — Self-hosted Mem0 server URL (alt: MEM0_HOST env var).
|
||||
When set, routes to the self-hosted HTTP backend.
|
||||
user_id — Canonical user identifier. When set, it is applied
|
||||
uniformly across every gateway (CLI, Telegram, Slack,
|
||||
Discord, …) so the same human gets one merged memory
|
||||
store. When unset, the gateway-native id (e.g. Telegram
|
||||
numeric id, Discord snowflake) is used instead.
|
||||
agent_id — Agent identifier (default: hermes)
|
||||
|
||||
The matching MEM0_MODE / MEM0_USER_ID / MEM0_AGENT_ID environment variables are
|
||||
still read as a backward-compatible fallback, but mem0.json is the canonical
|
||||
home for these non-secret settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from agent.memory_provider import MemoryProvider
|
||||
from tools.registry import tool_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Circuit breaker: after this many consecutive failures, pause API calls
|
||||
# for _BREAKER_COOLDOWN_SECS to avoid hammering a down server.
|
||||
_BREAKER_THRESHOLD = 5
|
||||
_BREAKER_COOLDOWN_SECS = 120
|
||||
_PREFETCH_WAIT_SECS = 3
|
||||
|
||||
_CLIENT_ERROR_TYPES = ("MemoryNotFoundError", "ValidationError")
|
||||
|
||||
# Sentinel returned when neither MEM0_USER_ID nor a gateway-native id is
|
||||
# available. Treated as "no operator-configured user_id" by initialize() so
|
||||
# that legacy mem0.json files written by the setup wizard (which historically
|
||||
# wrote this exact placeholder) still allow gateway-native ids to flow
|
||||
# through instead of silently overriding them with the placeholder.
|
||||
_DEFAULT_USER_ID = "hermes-user"
|
||||
|
||||
|
||||
def _is_client_error(exc: Exception) -> bool:
|
||||
"""True for user-caused errors (bad ID, not found) that should NOT trip circuit breaker."""
|
||||
etype = type(exc).__name__
|
||||
if etype in _CLIENT_ERROR_TYPES:
|
||||
return True
|
||||
err_str = str(exc).lower()
|
||||
return "404" in err_str or "not found" in err_str or "valid uuid" in err_str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_config() -> dict:
|
||||
"""Load config from env vars, with $HERMES_HOME/mem0.json overrides.
|
||||
|
||||
Environment variables provide defaults; mem0.json (if present) overrides
|
||||
individual keys. This avoids a silent failure when the JSON file exists
|
||||
but is missing fields like ``api_key`` that the user set in ``.env``.
|
||||
"""
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
config = {
|
||||
"mode": os.environ.get("MEM0_MODE", "platform"),
|
||||
"api_key": os.environ.get("MEM0_API_KEY", ""),
|
||||
"host": os.environ.get("MEM0_HOST", ""),
|
||||
"agent_id": os.environ.get("MEM0_AGENT_ID", "hermes"),
|
||||
"oss": {},
|
||||
}
|
||||
# Only carry user_id when the operator explicitly configured one (env or
|
||||
# mem0.json). An absent key tells initialize() to fall back to the
|
||||
# gateway-native id from kwargs instead of overriding it with a placeholder.
|
||||
env_user_id = os.environ.get("MEM0_USER_ID")
|
||||
if env_user_id:
|
||||
config["user_id"] = env_user_id
|
||||
|
||||
config_path = get_hermes_home() / "mem0.json"
|
||||
if config_path.exists():
|
||||
try:
|
||||
file_cfg = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
config.update({k: v for k, v in file_cfg.items()
|
||||
if v is not None and v != ""})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return config
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEARCH_SCHEMA = {
|
||||
"name": "mem0_search",
|
||||
"description": (
|
||||
"Search the user's memories by meaning; returns facts ranked by "
|
||||
"relevance. Use this before answering any question that may depend on "
|
||||
"what you know about the user (preferences, facts, history, people, "
|
||||
"projects, past decisions). For multi-part or multi-hop questions, "
|
||||
"call it several times — vary the wording and run follow-up searches "
|
||||
"on what earlier results reveal; one search is rarely enough."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "What to search for."},
|
||||
"top_k": {"type": "integer", "description": "Max results (default: 10, max: 50)."},
|
||||
"rerank": {"type": "boolean", "description": "Rerank results for relevance (default: false, platform mode only)."},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
}
|
||||
|
||||
ADD_SCHEMA = {
|
||||
"name": "mem0_add",
|
||||
"description": (
|
||||
"Store a durable fact about the user, verbatim (no LLM extraction). "
|
||||
"Call this the moment the user states a lasting preference, correction, "
|
||||
"decision, or personal detail worth recalling on future turns — don't "
|
||||
"wait to be asked to remember. Skip transient chit-chat and facts you've "
|
||||
"already stored."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string", "description": "The fact to store."},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
}
|
||||
|
||||
UPDATE_SCHEMA = {
|
||||
"name": "mem0_update",
|
||||
"description": (
|
||||
"Replace the text of an existing memory by its ID (take the ID from a "
|
||||
"mem0_search result). Use when a stored fact has changed "
|
||||
"or was wrong — correct it in place instead of adding a duplicate."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"memory_id": {"type": "string", "description": "Memory UUID to update."},
|
||||
"text": {"type": "string", "description": "New text content."},
|
||||
},
|
||||
"required": ["memory_id", "text"],
|
||||
},
|
||||
}
|
||||
|
||||
DELETE_SCHEMA = {
|
||||
"name": "mem0_delete",
|
||||
"description": (
|
||||
"Delete a memory by its ID (take the ID from a mem0_search "
|
||||
"result). Use when a stored fact is obsolete or the user asks you to "
|
||||
"forget it; prefer mem0_update if the fact merely changed."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"memory_id": {"type": "string", "description": "Memory UUID to delete."},
|
||||
},
|
||||
"required": ["memory_id"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryProvider implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Mem0MemoryProvider(MemoryProvider):
|
||||
"""Mem0 memory with server-side extraction and semantic search.
|
||||
|
||||
Supports Platform API (cloud) and OSS (self-hosted) modes via MEM0_MODE.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._config = None
|
||||
self._backend = None
|
||||
self._mode = "platform"
|
||||
self._api_key = ""
|
||||
self._host = ""
|
||||
self._user_id = _DEFAULT_USER_ID
|
||||
self._agent_id = "hermes"
|
||||
self._rerank_default = False
|
||||
self._channel = "cli" # gateway channel name (cli/telegram/discord/...)
|
||||
self._sync_thread = None
|
||||
self._prefetch_thread = None
|
||||
self._prefetch_query = ""
|
||||
self._prefetch_result = ""
|
||||
self._prefetch_done = False
|
||||
# Circuit breaker state
|
||||
self._consecutive_failures = 0
|
||||
self._breaker_open_until = 0.0
|
||||
self._breaker_lock = threading.Lock()
|
||||
self._sync_lock = threading.Lock()
|
||||
self._prefetch_lock = threading.Lock()
|
||||
self._atexit_registered = False
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "mem0"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
cfg = _load_config()
|
||||
mode = cfg.get("mode", "platform")
|
||||
if mode == "oss":
|
||||
return bool(cfg.get("oss", {}).get("vector_store"))
|
||||
# Platform needs an api_key; self-hosted needs a host (api_key optional
|
||||
# when the server runs with AUTH_DISABLED).
|
||||
return bool(cfg.get("api_key") or cfg.get("host"))
|
||||
|
||||
def save_config(self, values, hermes_home):
|
||||
"""Write config to $HERMES_HOME/mem0.json."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
config_path = Path(hermes_home) / "mem0.json"
|
||||
existing = {}
|
||||
if config_path.exists():
|
||||
try:
|
||||
existing = json.loads(config_path.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
existing.update(values)
|
||||
from utils import atomic_json_write
|
||||
atomic_json_write(config_path, existing, mode=0o600)
|
||||
|
||||
def get_config_schema(self):
|
||||
cfg = _load_config()
|
||||
mode = cfg.get("mode", "platform")
|
||||
api_key_required = mode != "oss"
|
||||
return [
|
||||
{"key": "api_key", "description": "Mem0 Platform API key", "secret": True, "required": api_key_required, "env_var": "MEM0_API_KEY", "url": "https://app.mem0.ai"},
|
||||
{"key": "host", "description": "Self-hosted Mem0 server URL (leave blank for cloud)", "required": False, "env_var": "MEM0_HOST"},
|
||||
{"key": "user_id", "description": "User identifier", "default": "hermes-user"},
|
||||
{"key": "agent_id", "description": "Agent identifier", "default": "hermes"},
|
||||
{"key": "rerank", "description": "Enable reranking for recall", "default": "false", "choices": ["true", "false"]},
|
||||
]
|
||||
|
||||
def post_setup(self, hermes_home: str, config: dict) -> None:
|
||||
from ._setup import post_setup
|
||||
post_setup(hermes_home, config)
|
||||
|
||||
def _create_backend(self):
|
||||
# Lazy-install the mem0 SDK on demand before either backend imports
|
||||
# it. ensure() honors security.allow_lazy_installs (default true) and,
|
||||
# on a sealed Docker venv, redirects the install to the durable
|
||||
# target. On failure we fall through so the import inside the backend
|
||||
# produces the canonical error, captured below.
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
_lazy_ensure("memory.mem0", prompt=False)
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if self._mode == "oss":
|
||||
from ._backend import OSSBackend
|
||||
return OSSBackend(self._config.get("oss", {}))
|
||||
if self._host:
|
||||
from ._backend import SelfHostedBackend
|
||||
return SelfHostedBackend(self._api_key, self._host)
|
||||
from ._backend import PlatformBackend
|
||||
return PlatformBackend(self._api_key)
|
||||
except Exception as e:
|
||||
logger.error("Mem0 backend failed to initialize (%s mode): %s", self._mode, e)
|
||||
self._init_error = str(e)
|
||||
return None
|
||||
|
||||
def _is_breaker_open(self) -> bool:
|
||||
"""Return True if the circuit breaker is tripped (too many failures)."""
|
||||
with self._breaker_lock:
|
||||
if self._consecutive_failures < _BREAKER_THRESHOLD:
|
||||
return False
|
||||
if time.monotonic() >= self._breaker_open_until:
|
||||
self._consecutive_failures = 0
|
||||
return False
|
||||
return True
|
||||
|
||||
def _format_error(self, prefix: str, exc: Exception) -> str:
|
||||
msg = f"{prefix}: {exc}"
|
||||
if self._mode == "oss":
|
||||
err_str = str(exc).lower()
|
||||
if "connection" in err_str or "refused" in err_str or "timeout" in err_str:
|
||||
vs = self._config.get("oss", {}).get("vector_store", {})
|
||||
msg += f" (check that {vs.get('provider', 'vector store')} is running)"
|
||||
return msg
|
||||
|
||||
def _record_success(self):
|
||||
with self._breaker_lock:
|
||||
self._consecutive_failures = 0
|
||||
|
||||
def _record_failure(self):
|
||||
with self._breaker_lock:
|
||||
self._consecutive_failures += 1
|
||||
count = self._consecutive_failures
|
||||
if count >= _BREAKER_THRESHOLD:
|
||||
self._breaker_open_until = time.monotonic() + _BREAKER_COOLDOWN_SECS
|
||||
else:
|
||||
count = 0
|
||||
if count >= _BREAKER_THRESHOLD:
|
||||
hint = ""
|
||||
if self._mode == "oss":
|
||||
vs = self._config.get("oss", {}).get("vector_store", {})
|
||||
provider = vs.get("provider", "unknown")
|
||||
hint = f" Check that your {provider} vector store is running and reachable."
|
||||
logger.warning(
|
||||
"Mem0 circuit breaker tripped after %d consecutive failures. "
|
||||
"Pausing API calls for %ds.%s",
|
||||
count, _BREAKER_COOLDOWN_SECS, hint,
|
||||
)
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
self._config = _load_config()
|
||||
self._mode = self._config.get("mode", "platform")
|
||||
self._api_key = self._config.get("api_key", "")
|
||||
self._host = self._config.get("host", "")
|
||||
# Resolution order for user_id:
|
||||
# 1. Operator-configured MEM0_USER_ID (env or $HERMES_HOME/mem0.json) —
|
||||
# the canonical principal, applied across every gateway so the same
|
||||
# human gets one merged memory store.
|
||||
# 2. Gateway-native id from kwargs (Telegram numeric id, Discord
|
||||
# snowflake, etc.) — preserves per-platform isolation when no
|
||||
# override is configured.
|
||||
# 3. Hardcoded fallback _DEFAULT_USER_ID (CLI with no auth).
|
||||
# The literal _DEFAULT_USER_ID string is treated as unset so users who
|
||||
# ran the setup wizard with the suggested default still get gateway-
|
||||
# native ids instead of being silently bucketed together.
|
||||
configured = self._config.get("user_id")
|
||||
if configured == _DEFAULT_USER_ID:
|
||||
configured = None
|
||||
self._user_id = configured or kwargs.get("user_id") or _DEFAULT_USER_ID
|
||||
self._agent_id = self._config.get("agent_id", "hermes")
|
||||
# Persisted rerank preference (setup wizard / mem0.json). Used as the
|
||||
# DEFAULT for mem0_search when the model doesn't pass ``rerank``
|
||||
# explicitly; per-call args still win. Platform-only feature — other
|
||||
# backends accept-and-ignore the flag.
|
||||
_rr = self._config.get("rerank", False)
|
||||
self._rerank_default = (
|
||||
_rr.lower() in ("true", "1", "yes") if isinstance(_rr, str) else bool(_rr)
|
||||
)
|
||||
self._channel = kwargs.get("platform") or "cli"
|
||||
self._backend = self._create_backend()
|
||||
if self._backend and not self._atexit_registered:
|
||||
atexit.register(self._shutdown_backend)
|
||||
self._atexit_registered = True
|
||||
|
||||
def _read_filters(self) -> Dict[str, Any]:
|
||||
# Scoped to user_id only — by design — so recall surfaces memories
|
||||
# written from any gateway/agent under this principal. Writes attach
|
||||
# agent_id (and metadata.channel) so per-agent / per-channel views are
|
||||
# still possible at query time when needed; reads default to the wider
|
||||
# cross-agent recall.
|
||||
return {"user_id": self._user_id}
|
||||
|
||||
def _write_metadata(self) -> Dict[str, Any]:
|
||||
# Tag every write with the gateway channel so the dashboard can offer
|
||||
# per-channel filtered views without coupling identity to the channel.
|
||||
return {"channel": self._channel} if self._channel else {}
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
# Mirror the precedence in _create_backend (oss > host > platform) so
|
||||
# the label always names the backend that actually runs. Checking
|
||||
# ``host`` first here would mislabel an ``oss``+``host`` config as
|
||||
# self-hosted HTTP even though OSS wins the routing.
|
||||
if self._mode == "oss":
|
||||
mode_label = "OSS (self-hosted)"
|
||||
elif self._host:
|
||||
mode_label = "self-hosted (HTTP API)"
|
||||
else:
|
||||
mode_label = "platform (cloud API)"
|
||||
# Rerank is a Mem0 Platform feature only.
|
||||
rerank_note = " Rerank is available on search." if (self._mode == "platform" and not self._host) else ""
|
||||
return (
|
||||
"# Mem0 Memory\n"
|
||||
f"Active. Mode: {mode_label}. User: {self._user_id}.\n"
|
||||
"You have persistent memory of this user from past conversations. "
|
||||
"You should call mem0_search before answering anything that could depend "
|
||||
"on prior context (the user's preferences, facts, history, people, "
|
||||
"projects, or earlier decisions) — do not rely on the chat window "
|
||||
"alone, and do not assume you have no memory.\n"
|
||||
"For multi-part or multi-hop questions, run several searches with "
|
||||
"different wording/angles and follow-up searches on what the first "
|
||||
"results surface; one search is rarely enough. Keep searching until "
|
||||
"you have every fact the question needs before you answer.\n"
|
||||
"Tools: mem0_search to find memories, mem0_add to store facts, "
|
||||
f"mem0_update and mem0_delete to manage by ID.{rerank_note}"
|
||||
)
|
||||
|
||||
def on_turn_start(self, turn_number: int, message: str, **kwargs) -> None:
|
||||
self._start_prefetch(message)
|
||||
|
||||
def _consume_prefetch_result(self, query: str) -> str | None:
|
||||
with self._prefetch_lock:
|
||||
if self._prefetch_query != query or not self._prefetch_done:
|
||||
return None
|
||||
result = self._prefetch_result
|
||||
self._prefetch_result = ""
|
||||
self._prefetch_done = False
|
||||
return result
|
||||
|
||||
def _start_prefetch(self, query: str) -> None:
|
||||
if not query or self._backend is None or self._is_breaker_open():
|
||||
return
|
||||
backend = self._backend
|
||||
with self._prefetch_lock:
|
||||
if self._prefetch_query == query:
|
||||
if self._prefetch_done:
|
||||
return
|
||||
if self._prefetch_thread and self._prefetch_thread.is_alive():
|
||||
return
|
||||
self._prefetch_query = query
|
||||
self._prefetch_result = ""
|
||||
self._prefetch_done = False
|
||||
|
||||
def _run():
|
||||
body = ""
|
||||
try:
|
||||
results = backend.search(
|
||||
query, filters=self._read_filters(), top_k=10, rerank=False,
|
||||
)
|
||||
lines = [r.get("memory", "") for r in (results or []) if r.get("memory")]
|
||||
if lines:
|
||||
body = "## Mem0 Memory\n" + "\n".join(f"- {l}" for l in lines)
|
||||
self._record_success()
|
||||
except Exception as e:
|
||||
self._record_failure()
|
||||
logger.debug("Mem0 prefetch failed: %s", e)
|
||||
with self._prefetch_lock:
|
||||
if self._prefetch_query == query:
|
||||
self._prefetch_result = body
|
||||
self._prefetch_done = True
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True, name="mem0-prefetch")
|
||||
with self._prefetch_lock:
|
||||
self._prefetch_thread = t
|
||||
t.start()
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
"""Recall memories for the CURRENT question with a short hot-path wait."""
|
||||
cached = self._consume_prefetch_result(query)
|
||||
if cached is not None:
|
||||
return cached
|
||||
self._start_prefetch(query)
|
||||
with self._prefetch_lock:
|
||||
thread = self._prefetch_thread if self._prefetch_query == query else None
|
||||
if thread:
|
||||
thread.join(timeout=_PREFETCH_WAIT_SECS)
|
||||
cached = self._consume_prefetch_result(query)
|
||||
if cached is not None:
|
||||
return cached
|
||||
# Slow backend: skip injection; mem0_search tool remains the backstop.
|
||||
return ""
|
||||
|
||||
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
||||
"""Send the turn to Mem0 for server-side fact extraction (non-blocking)."""
|
||||
if self._backend is None or self._is_breaker_open():
|
||||
return
|
||||
|
||||
def _sync():
|
||||
backend = self._backend
|
||||
if backend is None:
|
||||
return
|
||||
try:
|
||||
messages = [
|
||||
{"role": "user", "content": user_content},
|
||||
{"role": "assistant", "content": assistant_content},
|
||||
]
|
||||
backend.add(
|
||||
messages,
|
||||
user_id=self._user_id,
|
||||
agent_id=self._agent_id,
|
||||
infer=True,
|
||||
metadata=self._write_metadata(),
|
||||
)
|
||||
self._record_success()
|
||||
except Exception as e:
|
||||
self._record_failure()
|
||||
logger.warning("Mem0 sync failed: %s", e)
|
||||
|
||||
with self._sync_lock:
|
||||
if self._sync_thread and self._sync_thread.is_alive():
|
||||
self._sync_thread.join(timeout=5.0)
|
||||
# If still alive after timeout, skip to avoid duplicate ingestion.
|
||||
if self._sync_thread and self._sync_thread.is_alive():
|
||||
return
|
||||
self._sync_thread = threading.Thread(target=_sync, daemon=True, name="mem0-sync")
|
||||
self._sync_thread.start()
|
||||
|
||||
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
||||
return [SEARCH_SCHEMA, ADD_SCHEMA, UPDATE_SCHEMA, DELETE_SCHEMA]
|
||||
|
||||
def handle_tool_call(self, tool_name: str, args: dict, **kwargs) -> str:
|
||||
if self._backend is None:
|
||||
err = getattr(self, "_init_error", "unknown error")
|
||||
hint = ""
|
||||
if self._mode == "oss":
|
||||
vs = self._config.get("oss", {}).get("vector_store", {})
|
||||
provider = vs.get("provider", "vector store")
|
||||
hint = f" Check that {provider} is running and reachable."
|
||||
return json.dumps({"error": f"Mem0 backend not initialized: {err}.{hint}"})
|
||||
|
||||
if self._is_breaker_open():
|
||||
msg = "Mem0 temporarily unavailable (multiple consecutive failures). Will retry automatically."
|
||||
if self._mode == "oss":
|
||||
vs = self._config.get("oss", {}).get("vector_store", {})
|
||||
msg += f" Check that your {vs.get('provider', 'vector store')} is running."
|
||||
return json.dumps({"error": msg})
|
||||
|
||||
if tool_name == "mem0_search":
|
||||
query = args.get("query", "")
|
||||
if not query:
|
||||
return tool_error("Missing required parameter: query")
|
||||
try:
|
||||
top_k = max(1, min(int(args.get("top_k", 10)), 50))
|
||||
rerank_raw = args.get("rerank", getattr(self, "_rerank_default", False))
|
||||
if isinstance(rerank_raw, str):
|
||||
rerank = rerank_raw.lower() not in ("false", "0", "no")
|
||||
else:
|
||||
rerank = bool(rerank_raw)
|
||||
results = self._backend.search(query, filters=self._read_filters(), top_k=top_k, rerank=rerank)
|
||||
self._record_success()
|
||||
if not results:
|
||||
return json.dumps({"result": "No relevant memories found."})
|
||||
items = [{"id": r.get("id"), "memory": r.get("memory", ""),
|
||||
"score": r.get("score", 0)} for r in results]
|
||||
return json.dumps({"results": items, "count": len(items)})
|
||||
except Exception as e:
|
||||
if not _is_client_error(e):
|
||||
self._record_failure()
|
||||
return tool_error(self._format_error("Search failed", e))
|
||||
|
||||
elif tool_name == "mem0_add":
|
||||
content = args.get("content", "")
|
||||
if not content:
|
||||
return tool_error("Missing required parameter: content")
|
||||
try:
|
||||
result = self._backend.add(
|
||||
[{"role": "user", "content": content}],
|
||||
user_id=self._user_id,
|
||||
agent_id=self._agent_id,
|
||||
infer=False,
|
||||
metadata=self._write_metadata(),
|
||||
)
|
||||
self._record_success()
|
||||
event_id = result.get("event_id") if isinstance(result, dict) else None
|
||||
# Cloud add is async (server-side extraction); OSS and self-hosted store synchronously.
|
||||
msg = "Fact stored." if (self._mode == "oss" or self._host) else "Fact queued for storage."
|
||||
return json.dumps({"result": msg, "event_id": event_id})
|
||||
except Exception as e:
|
||||
self._record_failure()
|
||||
return tool_error(self._format_error("Failed to store", e))
|
||||
|
||||
elif tool_name == "mem0_update":
|
||||
memory_id = args.get("memory_id", "")
|
||||
text = args.get("text", "")
|
||||
if not memory_id:
|
||||
return tool_error("Missing required parameter: memory_id")
|
||||
if not text:
|
||||
return tool_error("Missing required parameter: text")
|
||||
try:
|
||||
result = self._backend.update(memory_id, text)
|
||||
self._record_success()
|
||||
return json.dumps(result)
|
||||
except Exception as e:
|
||||
if _is_client_error(e):
|
||||
return tool_error(f"Memory not found: {memory_id}")
|
||||
self._record_failure()
|
||||
return tool_error(self._format_error("Update failed", e))
|
||||
|
||||
elif tool_name == "mem0_delete":
|
||||
memory_id = args.get("memory_id", "")
|
||||
if not memory_id:
|
||||
return tool_error("Missing required parameter: memory_id")
|
||||
try:
|
||||
result = self._backend.delete(memory_id)
|
||||
self._record_success()
|
||||
return json.dumps(result)
|
||||
except Exception as e:
|
||||
if _is_client_error(e):
|
||||
return tool_error(f"Memory not found: {memory_id}")
|
||||
self._record_failure()
|
||||
return tool_error(self._format_error("Delete failed", e))
|
||||
|
||||
return tool_error(f"Unknown tool: {tool_name}")
|
||||
|
||||
def _shutdown_backend(self):
|
||||
try:
|
||||
if self._backend:
|
||||
self._backend.close()
|
||||
self._backend = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for t in (self._prefetch_thread, self._sync_thread):
|
||||
if t and t.is_alive():
|
||||
t.join(timeout=5.0)
|
||||
self._shutdown_backend()
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register Mem0 as a memory provider plugin."""
|
||||
ctx.register_memory_provider(Mem0MemoryProvider())
|
||||
@@ -0,0 +1,298 @@
|
||||
"""Backend abstraction for Mem0 Platform and OSS modes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Mem0Backend(ABC):
|
||||
"""Unified interface over Platform (MemoryClient) and OSS (Memory) backends."""
|
||||
|
||||
@abstractmethod
|
||||
def search(self, query: str, *, filters: dict, top_k: int = 10, rerank: bool = False) -> list[dict]:
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def add(
|
||||
self,
|
||||
messages: list,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
infer: bool = False,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def update(self, memory_id: str, text: str) -> dict:
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, memory_id: str) -> dict:
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _unwrap_results(response: Any) -> list:
|
||||
"""Normalize API response — extract results list from dict or pass through."""
|
||||
if isinstance(response, dict):
|
||||
return response.get("results", [])
|
||||
if isinstance(response, list):
|
||||
return response
|
||||
return []
|
||||
|
||||
|
||||
class PlatformBackend(Mem0Backend):
|
||||
"""Wraps mem0.MemoryClient for Mem0 Platform (cloud API)."""
|
||||
|
||||
def __init__(self, api_key: str):
|
||||
from mem0 import MemoryClient
|
||||
self._client = MemoryClient(api_key=api_key)
|
||||
|
||||
def search(self, query: str, *, filters: dict, top_k: int = 10, rerank: bool = False) -> list[dict]:
|
||||
response = self._client.search(query, filters=filters, top_k=top_k, rerank=rerank)
|
||||
return _unwrap_results(response)
|
||||
|
||||
def add(
|
||||
self,
|
||||
messages: list,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
infer: bool = False,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
kwargs: dict[str, Any] = {"user_id": user_id, "agent_id": agent_id, "infer": infer}
|
||||
if metadata:
|
||||
kwargs["metadata"] = metadata
|
||||
return self._client.add(messages, **kwargs)
|
||||
|
||||
def update(self, memory_id: str, text: str) -> dict:
|
||||
self._client.update(memory_id=memory_id, text=text)
|
||||
return {"result": "Memory updated.", "memory_id": memory_id}
|
||||
|
||||
def delete(self, memory_id: str) -> dict:
|
||||
self._client.delete(memory_id=memory_id)
|
||||
return {"result": "Memory deleted.", "memory_id": memory_id}
|
||||
|
||||
|
||||
class SelfHostedBackend(Mem0Backend):
|
||||
"""Direct HTTP backend for a self-hosted Mem0 server (the FastAPI ``server/``).
|
||||
|
||||
mem0.MemoryClient can't be reused for self-hosted: it is hardwired to the
|
||||
cloud API — ``Authorization: Token`` auth and a ``GET /v1/ping/`` validation
|
||||
call in ``__init__`` that the self-hosted server does not expose (it would
|
||||
404 before any real request). This client talks to that server directly,
|
||||
using its actual contract: ``X-API-Key`` auth and the ``/memories`` /
|
||||
``/search`` routes.
|
||||
"""
|
||||
|
||||
def __init__(self, api_key: str, host: str, transport=None):
|
||||
import httpx
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if api_key:
|
||||
headers["X-API-Key"] = api_key # omitted only for AUTH_DISABLED servers
|
||||
# Connect-level retries smooth over transient blips so a single
|
||||
# dropped SYN doesn't count toward the provider failure breaker.
|
||||
# ``transport`` is injectable for tests (httpx.MockTransport).
|
||||
if transport is None:
|
||||
transport = httpx.HTTPTransport(retries=2)
|
||||
self._client = httpx.Client(
|
||||
base_url=host.rstrip("/"), headers=headers, timeout=30.0,
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
def _json(self, method: str, path: str, **kwargs) -> Any:
|
||||
resp = self._client.request(method, path, **kwargs)
|
||||
resp.raise_for_status()
|
||||
return resp.json() if resp.content else {}
|
||||
|
||||
def search(self, query: str, *, filters: dict, top_k: int = 10, rerank: bool = False) -> list[dict]:
|
||||
# rerank is a platform-only feature; the self-hosted /search ignores it.
|
||||
body: dict[str, Any] = {"query": query, "top_k": top_k}
|
||||
if filters:
|
||||
body["filters"] = filters # user_id belongs in filters (top-level is deprecated)
|
||||
return _unwrap_results(self._json("POST", "/search", json=body))
|
||||
|
||||
def add(
|
||||
self,
|
||||
messages: list,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
infer: bool = False,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
body: dict[str, Any] = {
|
||||
"messages": messages,
|
||||
"user_id": user_id,
|
||||
"agent_id": agent_id,
|
||||
"infer": infer,
|
||||
}
|
||||
if metadata:
|
||||
body["metadata"] = metadata
|
||||
return self._json("POST", "/memories", json=body)
|
||||
|
||||
def update(self, memory_id: str, text: str) -> dict:
|
||||
self._json("PUT", f"/memories/{memory_id}", json={"text": text})
|
||||
return {"result": "Memory updated.", "memory_id": memory_id}
|
||||
|
||||
def delete(self, memory_id: str) -> dict:
|
||||
self._json("DELETE", f"/memories/{memory_id}")
|
||||
return {"result": "Memory deleted.", "memory_id": memory_id}
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class OSSBackend(Mem0Backend):
|
||||
"""Wraps mem0.Memory for self-hosted (OSS) mode."""
|
||||
|
||||
def __init__(self, oss_config: dict):
|
||||
import os
|
||||
from mem0 import Memory
|
||||
|
||||
vector_store = dict(oss_config["vector_store"])
|
||||
vs_config = dict(vector_store.get("config", {}))
|
||||
|
||||
if "path" in vs_config:
|
||||
vs_config["path"] = os.path.expanduser(vs_config["path"])
|
||||
|
||||
embedder_config = oss_config.get("embedder", {}).get("config", {})
|
||||
dims = embedder_config.get("embedding_dims")
|
||||
if not dims:
|
||||
from ._oss_providers import KNOWN_DIMS
|
||||
model = embedder_config.get("model", "")
|
||||
dims = KNOWN_DIMS.get(model)
|
||||
if dims:
|
||||
vs_config["embedding_model_dims"] = dims
|
||||
self._recreate_collection_if_dims_changed(
|
||||
vector_store.get("provider", "qdrant"), vs_config, dims,
|
||||
)
|
||||
|
||||
vector_store["config"] = vs_config
|
||||
|
||||
config = {
|
||||
"vector_store": vector_store,
|
||||
"llm": oss_config["llm"],
|
||||
"embedder": oss_config["embedder"],
|
||||
"version": "v1.1",
|
||||
}
|
||||
self._memory = Memory.from_config(config)
|
||||
|
||||
@staticmethod
|
||||
def _recreate_collection_if_dims_changed(provider: str, vs_config: dict, expected_dims: int) -> None:
|
||||
"""Delete stale vector collection when embedding dimensions change."""
|
||||
collection_name = vs_config.get("collection_name", "mem0")
|
||||
if provider == "qdrant":
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
path = vs_config.get("path")
|
||||
url = vs_config.get("url")
|
||||
if path:
|
||||
client = QdrantClient(path=path)
|
||||
elif url:
|
||||
client = QdrantClient(url=url, api_key=vs_config.get("api_key"))
|
||||
else:
|
||||
return
|
||||
try:
|
||||
if not client.collection_exists(collection_name):
|
||||
return
|
||||
info = client.get_collection(collection_name)
|
||||
vectors = info.config.params.vectors
|
||||
# Named-vector collections expose a dict; unnamed expose an object with .size.
|
||||
if isinstance(vectors, dict):
|
||||
first = next(iter(vectors.values()), None)
|
||||
current_dims = first.size if first else None
|
||||
else:
|
||||
current_dims = getattr(vectors, "size", None)
|
||||
if current_dims is not None and current_dims != expected_dims:
|
||||
client.delete_collection(collection_name)
|
||||
finally:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
elif provider == "pgvector":
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2 import sql as pgsql
|
||||
conn_params = {}
|
||||
for k in ("host", "port", "user", "password", "dbname"):
|
||||
if vs_config.get(k):
|
||||
conn_params[k] = vs_config[k]
|
||||
if vs_config.get("sslmode"):
|
||||
conn_params["sslmode"] = vs_config["sslmode"]
|
||||
conn = psycopg2.connect(**conn_params)
|
||||
conn.autocommit = True
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(
|
||||
"SELECT atttypmod FROM pg_attribute "
|
||||
"WHERE attrelid = %s::regclass AND attname = 'vector'",
|
||||
(collection_name,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row and row[0] > 0 and row[0] != expected_dims:
|
||||
cur.execute(pgsql.SQL("DROP TABLE IF EXISTS {}").format(
|
||||
pgsql.Identifier(collection_name)
|
||||
))
|
||||
finally:
|
||||
cur.close()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def search(self, query: str, *, filters: dict, top_k: int = 10, rerank: bool = False) -> list[dict]:
|
||||
response = self._memory.search(query, filters=filters, top_k=top_k)
|
||||
return _unwrap_results(response)
|
||||
|
||||
def add(
|
||||
self,
|
||||
messages: list,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
infer: bool = False,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
kwargs: dict[str, Any] = {"user_id": user_id, "agent_id": agent_id, "infer": infer}
|
||||
if metadata:
|
||||
kwargs["metadata"] = metadata
|
||||
return self._memory.add(messages, **kwargs)
|
||||
|
||||
def update(self, memory_id: str, text: str) -> dict:
|
||||
self._memory.update(memory_id, data=text)
|
||||
return {"result": "Memory updated.", "memory_id": memory_id}
|
||||
|
||||
def delete(self, memory_id: str) -> dict:
|
||||
self._memory.delete(memory_id)
|
||||
return {"result": "Memory deleted.", "memory_id": memory_id}
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
telemetry = getattr(self._memory, "telemetry", None)
|
||||
if telemetry and hasattr(telemetry, "posthog"):
|
||||
try:
|
||||
telemetry.posthog.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
if hasattr(self._memory, "close"):
|
||||
self._memory.close()
|
||||
vs = getattr(self._memory, "vector_store", None)
|
||||
if vs and hasattr(vs, "close"):
|
||||
vs.close()
|
||||
client = getattr(vs, "client", None)
|
||||
if client and hasattr(client, "close"):
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,84 @@
|
||||
"""OSS provider definitions for LLM, embedder, and vector store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
LLM_PROVIDERS: dict[str, dict[str, Any]] = {
|
||||
"openai": {
|
||||
"label": "OpenAI",
|
||||
"needs_key": True,
|
||||
"env_var": "OPENAI_API_KEY",
|
||||
"default_model": "gpt-5-mini",
|
||||
},
|
||||
"ollama": {
|
||||
"label": "Ollama (local)",
|
||||
"needs_key": False,
|
||||
"default_model": "llama3.1:8b",
|
||||
"default_url": "http://localhost:11434",
|
||||
"pip_dep": "ollama",
|
||||
},
|
||||
}
|
||||
|
||||
EMBEDDER_PROVIDERS: dict[str, dict[str, Any]] = {
|
||||
"openai": {
|
||||
"label": "OpenAI",
|
||||
"needs_key": True,
|
||||
"env_var": "OPENAI_API_KEY",
|
||||
"default_model": "text-embedding-3-small",
|
||||
"dims": 1536,
|
||||
},
|
||||
"ollama": {
|
||||
"label": "Ollama (local)",
|
||||
"needs_key": False,
|
||||
"default_model": "nomic-embed-text",
|
||||
"default_url": "http://localhost:11434",
|
||||
"dims": 768,
|
||||
"pip_dep": "ollama",
|
||||
},
|
||||
}
|
||||
|
||||
VECTOR_PROVIDERS: dict[str, dict[str, Any]] = {
|
||||
"qdrant": {
|
||||
"label": "Qdrant",
|
||||
"default_config": {"path": os.path.expanduser("~/.hermes/mem0_qdrant")},
|
||||
"pip_dep": "qdrant-client",
|
||||
},
|
||||
"pgvector": {
|
||||
"label": "PGVector",
|
||||
"default_config": {"host": "localhost", "port": 5432, "user": os.getenv("USER", "postgres"), "dbname": "postgres"},
|
||||
"pip_dep": "psycopg2-binary",
|
||||
},
|
||||
}
|
||||
|
||||
KNOWN_DIMS: dict[str, int] = {
|
||||
"text-embedding-3-small": 1536,
|
||||
"text-embedding-3-large": 3072,
|
||||
"text-embedding-ada-002": 1536,
|
||||
"nomic-embed-text": 768,
|
||||
}
|
||||
|
||||
|
||||
def validate_oss_config(oss_config: dict) -> list[str]:
|
||||
"""Validate an OSS config dict. Returns list of error strings (empty = valid)."""
|
||||
errors: list[str] = []
|
||||
|
||||
for section, registry in [("llm", LLM_PROVIDERS), ("embedder", EMBEDDER_PROVIDERS),
|
||||
("vector_store", VECTOR_PROVIDERS)]:
|
||||
block = oss_config.get(section)
|
||||
if not block or not isinstance(block, dict):
|
||||
errors.append(f"Missing required section: {section}")
|
||||
continue
|
||||
provider_id = block.get("provider", "")
|
||||
if provider_id not in registry:
|
||||
valid = ", ".join(registry.keys())
|
||||
errors.append(f"Unknown {section} provider '{provider_id}'. Valid: {valid}")
|
||||
|
||||
vs = oss_config.get("vector_store", {})
|
||||
if vs.get("provider") == "pgvector":
|
||||
cfg = vs.get("config", {})
|
||||
if not cfg.get("user"):
|
||||
errors.append("PGVector requires 'user' in vector_store.config")
|
||||
|
||||
return errors
|
||||
@@ -0,0 +1,981 @@
|
||||
"""Setup wizard for Mem0 plugin — interactive and flag-based modes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
from ._oss_providers import (
|
||||
LLM_PROVIDERS,
|
||||
EMBEDDER_PROVIDERS,
|
||||
VECTOR_PROVIDERS,
|
||||
KNOWN_DIMS,
|
||||
validate_oss_config,
|
||||
)
|
||||
|
||||
|
||||
def _curses_select(title: str, items: list[tuple[str, str]], default: int = 0) -> int:
|
||||
"""Interactive single-select with arrow keys."""
|
||||
from hermes_cli.curses_ui import curses_radiolist
|
||||
display_items = [
|
||||
f"{label} {desc}" if desc else label
|
||||
for label, desc in items
|
||||
]
|
||||
return curses_radiolist(title, display_items, selected=default, cancel_returns=default)
|
||||
|
||||
|
||||
def _prompt(label: str, default: str | None = None, secret: bool = False) -> str:
|
||||
"""Prompt for a value with optional default and secret masking."""
|
||||
suffix = f" [{default}]" if default else ""
|
||||
if secret:
|
||||
sys.stdout.write(f" {label}{suffix}: ")
|
||||
sys.stdout.flush()
|
||||
if sys.stdin.isatty():
|
||||
val = getpass.getpass(prompt="")
|
||||
else:
|
||||
val = sys.stdin.readline().strip()
|
||||
else:
|
||||
sys.stdout.write(f" {label}{suffix}: ")
|
||||
sys.stdout.flush()
|
||||
val = sys.stdin.readline().strip()
|
||||
return val or (default or "")
|
||||
|
||||
|
||||
def has_oss_flags() -> bool:
|
||||
"""Check if OSS-related flags are present in sys.argv."""
|
||||
flags = parse_flags(sys.argv[1:])
|
||||
if flags["mode"] == "oss":
|
||||
return True
|
||||
if any(flags.get(k) for k in ("oss_llm_key", "oss_vector_path", "oss_vector_url")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def parse_flags(argv: list[str] | None = None) -> dict[str, str]:
|
||||
"""Parse CLI flags from argv. Returns dict of flag values."""
|
||||
args = argv if argv is not None else sys.argv[1:]
|
||||
flags: dict[str, str] = {
|
||||
"mode": "",
|
||||
"api_key": "",
|
||||
"host": "",
|
||||
"oss_llm": "openai",
|
||||
"oss_llm_key": "",
|
||||
"oss_llm_model": "",
|
||||
"oss_llm_url": "",
|
||||
"oss_embedder": "openai",
|
||||
"oss_embedder_key": "",
|
||||
"oss_embedder_model": "",
|
||||
"oss_embedder_url": "",
|
||||
"oss_vector": "qdrant",
|
||||
"oss_vector_path": "",
|
||||
"oss_vector_url": "",
|
||||
"oss_vector_host": "",
|
||||
"oss_vector_port": "",
|
||||
"oss_vector_user": "",
|
||||
"oss_vector_password": "",
|
||||
"oss_vector_dbname": "",
|
||||
"user_id": "",
|
||||
"dry_run": False,
|
||||
}
|
||||
|
||||
flag_map = {
|
||||
"--mode": "mode",
|
||||
"--api-key": "api_key",
|
||||
"--host": "host",
|
||||
"--oss-llm": "oss_llm",
|
||||
"--oss-llm-key": "oss_llm_key",
|
||||
"--oss-llm-model": "oss_llm_model",
|
||||
"--oss-llm-url": "oss_llm_url",
|
||||
"--oss-embedder": "oss_embedder",
|
||||
"--oss-embedder-key": "oss_embedder_key",
|
||||
"--oss-embedder-model": "oss_embedder_model",
|
||||
"--oss-embedder-url": "oss_embedder_url",
|
||||
"--oss-vector": "oss_vector",
|
||||
"--oss-vector-path": "oss_vector_path",
|
||||
"--oss-vector-url": "oss_vector_url",
|
||||
"--oss-vector-host": "oss_vector_host",
|
||||
"--oss-vector-port": "oss_vector_port",
|
||||
"--oss-vector-user": "oss_vector_user",
|
||||
"--oss-vector-password": "oss_vector_password",
|
||||
"--oss-vector-dbname": "oss_vector_dbname",
|
||||
"--user-id": "user_id",
|
||||
}
|
||||
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--dry-run":
|
||||
flags["dry_run"] = True
|
||||
i += 1
|
||||
elif args[i] in flag_map and i + 1 < len(args):
|
||||
flags[flag_map[args[i]]] = args[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
return flags
|
||||
|
||||
|
||||
def build_oss_config(flags: dict[str, str]) -> tuple[dict, dict[str, str]]:
|
||||
"""Build OSS config dict + env_writes from parsed flags.
|
||||
|
||||
Returns (oss_config, env_writes) where oss_config goes into mem0.json
|
||||
and env_writes maps env var names to secret values for .env.
|
||||
"""
|
||||
llm_id = flags.get("oss_llm", "openai")
|
||||
llm_def = LLM_PROVIDERS[llm_id]
|
||||
llm_model = flags.get("oss_llm_model") or llm_def["default_model"]
|
||||
llm_config: dict[str, Any] = {"model": llm_model}
|
||||
if "default_url" in llm_def:
|
||||
llm_config["ollama_base_url"] = flags.get("oss_llm_url") or llm_def["default_url"]
|
||||
|
||||
embedder_id = flags.get("oss_embedder", "openai")
|
||||
embedder_def = EMBEDDER_PROVIDERS[embedder_id]
|
||||
embedder_model = flags.get("oss_embedder_model") or embedder_def["default_model"]
|
||||
embedder_config: dict[str, Any] = {"model": embedder_model}
|
||||
if "default_url" in embedder_def:
|
||||
embedder_config["ollama_base_url"] = flags.get("oss_embedder_url") or embedder_def["default_url"]
|
||||
dims = KNOWN_DIMS.get(embedder_model)
|
||||
if dims:
|
||||
embedder_config["embedding_dims"] = dims
|
||||
|
||||
vector_id = flags.get("oss_vector", "qdrant")
|
||||
vector_def = VECTOR_PROVIDERS[vector_id]
|
||||
vector_config = dict(vector_def["default_config"])
|
||||
if vector_id == "qdrant":
|
||||
if flags.get("oss_vector_path"):
|
||||
vector_config["path"] = flags["oss_vector_path"]
|
||||
if flags.get("oss_vector_url"):
|
||||
vector_config.pop("path", None)
|
||||
vector_config["url"] = flags["oss_vector_url"]
|
||||
elif vector_id == "pgvector":
|
||||
if flags.get("oss_vector_host"):
|
||||
vector_config["host"] = flags["oss_vector_host"]
|
||||
if flags.get("oss_vector_port"):
|
||||
vector_config["port"] = int(flags["oss_vector_port"])
|
||||
if flags.get("oss_vector_user"):
|
||||
vector_config["user"] = flags["oss_vector_user"]
|
||||
if flags.get("oss_vector_password"):
|
||||
vector_config["password"] = flags["oss_vector_password"]
|
||||
if flags.get("oss_vector_dbname"):
|
||||
vector_config["dbname"] = flags["oss_vector_dbname"]
|
||||
|
||||
oss_config = {
|
||||
"llm": {"provider": llm_id, "config": llm_config},
|
||||
"embedder": {"provider": embedder_id, "config": embedder_config},
|
||||
"vector_store": {"provider": vector_id, "config": vector_config},
|
||||
}
|
||||
|
||||
env_writes: dict[str, str] = {}
|
||||
if llm_def.get("needs_key") and flags.get("oss_llm_key"):
|
||||
env_writes[llm_def["env_var"]] = flags["oss_llm_key"]
|
||||
if embedder_def.get("needs_key") and flags.get("oss_embedder_key"):
|
||||
env_writes[embedder_def["env_var"]] = flags["oss_embedder_key"]
|
||||
elif embedder_def.get("needs_key") and embedder_id == llm_id and flags.get("oss_llm_key"):
|
||||
env_writes[embedder_def["env_var"]] = flags["oss_llm_key"]
|
||||
|
||||
return oss_config, env_writes
|
||||
|
||||
|
||||
def _write_env(env_path: Path, env_writes: dict[str, str]) -> None:
|
||||
"""Append or update env vars in .env file."""
|
||||
env_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing_lines: list[str] = []
|
||||
if env_path.exists():
|
||||
existing_lines = env_path.read_text().splitlines()
|
||||
|
||||
updated_keys: set[str] = set()
|
||||
new_lines: list[str] = []
|
||||
for line in existing_lines:
|
||||
key_match = line.split("=", 1)[0].strip() if "=" in line and not line.startswith("#") else None
|
||||
if key_match and key_match in env_writes:
|
||||
new_lines.append(f"{key_match}={env_writes[key_match]}")
|
||||
updated_keys.add(key_match)
|
||||
else:
|
||||
new_lines.append(line)
|
||||
for k, v in env_writes.items():
|
||||
if k not in updated_keys:
|
||||
new_lines.append(f"{k}={v}")
|
||||
|
||||
env_path.write_text("\n".join(new_lines) + "\n")
|
||||
|
||||
|
||||
def _save_mem0_json(hermes_home: str, data: dict) -> None:
|
||||
"""Merge-write to mem0.json."""
|
||||
config_path = Path(hermes_home) / "mem0.json"
|
||||
existing = {}
|
||||
if config_path.exists():
|
||||
try:
|
||||
existing = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
existing.update(data)
|
||||
config_path.write_text(json.dumps(existing, indent=2) + "\n")
|
||||
|
||||
|
||||
def _setup_platform(hermes_home: str, config: dict, flags: dict[str, str]) -> None:
|
||||
"""Platform mode setup — uses the framework's schema-based flow.
|
||||
|
||||
Delegates to the same code path the framework uses when post_setup
|
||||
doesn't exist, preserving the original platform onboarding experience.
|
||||
"""
|
||||
schema = [
|
||||
{"key": "api_key", "description": "Mem0 Platform API key", "secret": True, "required": True, "env_var": "MEM0_API_KEY", "url": "https://app.mem0.ai"},
|
||||
{"key": "user_id", "description": "User identifier", "default": "hermes-user"},
|
||||
{"key": "agent_id", "description": "Agent identifier", "default": "hermes"},
|
||||
{"key": "rerank", "description": "Enable reranking for recall", "default": "false", "choices": ["true", "false"]},
|
||||
]
|
||||
|
||||
existing_config = {}
|
||||
config_path = Path(hermes_home) / "mem0.json"
|
||||
if config_path.exists():
|
||||
try:
|
||||
existing_config = json.loads(config_path.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
provider_config = dict(existing_config)
|
||||
env_writes: dict[str, str] = {}
|
||||
|
||||
print("\n Configuring mem0:\n")
|
||||
|
||||
for field in schema:
|
||||
key = field["key"]
|
||||
desc = field.get("description", key)
|
||||
default = field.get("default")
|
||||
is_secret = field.get("secret", False)
|
||||
choices = field.get("choices")
|
||||
env_var = field.get("env_var")
|
||||
url = field.get("url")
|
||||
|
||||
if flags.get("api_key") and key == "api_key":
|
||||
env_writes["MEM0_API_KEY"] = flags["api_key"]
|
||||
continue
|
||||
|
||||
if choices and not is_secret:
|
||||
choice_items = [(c, "") for c in choices]
|
||||
current = provider_config.get(key, default)
|
||||
current_idx = 0
|
||||
if current and str(current).lower() in choices:
|
||||
current_idx = choices.index(str(current).lower())
|
||||
sel = _curses_select(f" {desc}", choice_items, default=current_idx)
|
||||
provider_config[key] = choices[sel]
|
||||
elif is_secret:
|
||||
existing = os.environ.get(env_var, "") if env_var else ""
|
||||
if existing:
|
||||
masked = f"...{existing[-4:]}" if len(existing) > 4 else "set"
|
||||
val = _prompt(f"{desc} (current: {masked}, blank to keep)", secret=True)
|
||||
else:
|
||||
if url:
|
||||
print(f" Get yours at {url}")
|
||||
val = _prompt(desc, secret=True)
|
||||
if val and env_var:
|
||||
env_writes[env_var] = val
|
||||
else:
|
||||
current = provider_config.get(key)
|
||||
effective_default = current or default
|
||||
val = _prompt(desc, default=str(effective_default) if effective_default else None)
|
||||
if val:
|
||||
provider_config[key] = val
|
||||
|
||||
if flags.get("dry_run"):
|
||||
print(f"\n [dry-run] Would save config: {provider_config}")
|
||||
if env_writes:
|
||||
print(" [dry-run] Would write API key to .env")
|
||||
print(" [dry-run] No files written.\n")
|
||||
return
|
||||
|
||||
provider_config["mode"] = "platform"
|
||||
# Clear any stale self-hosted host: routing checks ``host`` before platform
|
||||
# (see _create_backend), so leaving it would silently keep routing to the
|
||||
# self-hosted server even though the user just chose platform mode. Set it
|
||||
# to "" rather than pop() — save_config merges into the existing mem0.json
|
||||
# (existing.update), so a popped key would survive; an empty value overwrites
|
||||
# it and reads as falsy at routing time.
|
||||
provider_config["host"] = ""
|
||||
# The json-file clear above can't help when the host comes from the
|
||||
# environment: _load_config() seeds ``host`` from MEM0_HOST, and the
|
||||
# docs tell self-hosted users to put MEM0_HOST in ~/.hermes/.env. Warn
|
||||
# so the user knows platform mode won't take effect until it's removed.
|
||||
if os.environ.get("MEM0_HOST", "").strip():
|
||||
print(
|
||||
"\n ⚠ MEM0_HOST is set in your environment "
|
||||
f"({os.environ['MEM0_HOST']}). It overrides platform mode — "
|
||||
"remove it from ~/.hermes/.env (or unset it) or Hermes will keep "
|
||||
"routing to the self-hosted server."
|
||||
)
|
||||
|
||||
from hermes_cli.config import save_config
|
||||
config["memory"]["provider"] = "mem0"
|
||||
save_config(config)
|
||||
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.save_config(provider_config, hermes_home)
|
||||
|
||||
if env_writes:
|
||||
_write_env(Path(hermes_home) / ".env", env_writes)
|
||||
|
||||
print("\n Memory provider: mem0")
|
||||
print(" Activation saved to config.yaml")
|
||||
print(" Provider config saved")
|
||||
if env_writes:
|
||||
print(" API keys saved to .env")
|
||||
print("\n Start a new session to activate.\n")
|
||||
|
||||
|
||||
def _check_selfhosted_server(host: str) -> None:
|
||||
"""Best-effort reachability check for a self-hosted Mem0 server (non-fatal)."""
|
||||
import urllib.error
|
||||
import urllib.request as _urlreq
|
||||
|
||||
try:
|
||||
req = _urlreq.Request(f"{host.rstrip('/')}/docs", method="GET")
|
||||
_urlreq.urlopen(req, timeout=5)
|
||||
print(f" ✓ Mem0 server reachable at {host}")
|
||||
except urllib.error.HTTPError:
|
||||
# Any HTTP response (401/403/404) still means something is listening.
|
||||
print(f" ✓ Mem0 server responding at {host}")
|
||||
except Exception:
|
||||
print(f" ⚠ Could not reach {host} — check the URL and that the server is running.")
|
||||
|
||||
|
||||
def _setup_selfhosted(hermes_home: str, config: dict, flags: dict[str, str]) -> None:
|
||||
"""Self-hosted mode setup — point at an existing Mem0 dashboard server.
|
||||
|
||||
For users already running the Dockerized Mem0 FastAPI server: stores the
|
||||
server URL (behavioral -> mem0.json) and an optional API key
|
||||
(secret -> .env as MEM0_API_KEY).
|
||||
"""
|
||||
existing_config = {}
|
||||
config_path = Path(hermes_home) / "mem0.json"
|
||||
if config_path.exists():
|
||||
try:
|
||||
existing_config = json.loads(config_path.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
provider_config = dict(existing_config)
|
||||
|
||||
print("\n Configuring mem0 (self-hosted server):\n")
|
||||
|
||||
host = flags.get("host") or _prompt(
|
||||
"Mem0 server URL (e.g. http://localhost:8888)",
|
||||
default=provider_config.get("host") or None,
|
||||
)
|
||||
if not host:
|
||||
print(" Error: a server URL is required for self-hosted mode.", file=sys.stderr)
|
||||
return
|
||||
host = host.rstrip("/")
|
||||
|
||||
env_writes: dict[str, str] = {}
|
||||
if flags.get("api_key"):
|
||||
env_writes["MEM0_API_KEY"] = flags["api_key"]
|
||||
else:
|
||||
existing_key = os.environ.get("MEM0_API_KEY", "")
|
||||
if existing_key:
|
||||
masked = f"...{existing_key[-4:]}" if len(existing_key) > 4 else "set"
|
||||
val = _prompt(f"Server API key (current: {masked}, blank to keep)", secret=True)
|
||||
else:
|
||||
val = _prompt("Server API key (blank if AUTH_DISABLED)", secret=True)
|
||||
if val:
|
||||
env_writes["MEM0_API_KEY"] = val
|
||||
|
||||
user_id = flags.get("user_id") or _prompt(
|
||||
"User identifier", default=provider_config.get("user_id") or "hermes-user"
|
||||
)
|
||||
agent_id = _prompt("Agent identifier", default=provider_config.get("agent_id") or "hermes")
|
||||
|
||||
if flags.get("dry_run"):
|
||||
print(f"\n [dry-run] Would save config: host={host}, user_id={user_id}, agent_id={agent_id}")
|
||||
if env_writes:
|
||||
print(" [dry-run] Would write API key to .env")
|
||||
_check_selfhosted_server(host)
|
||||
print(" [dry-run] No files written.\n")
|
||||
return
|
||||
|
||||
provider_config["mode"] = "platform" # routing: oss > host > platform; host wins
|
||||
provider_config["host"] = host
|
||||
provider_config["user_id"] = user_id
|
||||
provider_config["agent_id"] = agent_id
|
||||
|
||||
from hermes_cli.config import save_config
|
||||
config["memory"]["provider"] = "mem0"
|
||||
save_config(config)
|
||||
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.save_config(provider_config, hermes_home)
|
||||
|
||||
if env_writes:
|
||||
_write_env(Path(hermes_home) / ".env", env_writes)
|
||||
|
||||
_check_selfhosted_server(host)
|
||||
print("\n Memory provider: mem0 (self-hosted)")
|
||||
print(f" Server: {host}")
|
||||
print(" Activation saved to config.yaml")
|
||||
print(" Provider config saved")
|
||||
if env_writes:
|
||||
print(" API key saved to .env")
|
||||
print("\n Start a new session to activate.\n")
|
||||
|
||||
|
||||
def _setup_oss(hermes_home: str, config: dict, flags: dict[str, str]) -> None:
|
||||
"""OSS mode setup — build config from flags or interactive prompts.
|
||||
|
||||
Non-interactive when --mode was set explicitly via flags (post_setup already
|
||||
resolved mode). Interactive only when mode was chosen via curses picker.
|
||||
"""
|
||||
if not flags.get("_mode_from_flag"):
|
||||
_setup_oss_interactive(hermes_home, config)
|
||||
return
|
||||
|
||||
oss_config, env_writes = build_oss_config(flags)
|
||||
errors = validate_oss_config(oss_config)
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f" Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
user_id = flags.get("user_id") or os.getenv("USER", "hermes-user")
|
||||
|
||||
llm_id = oss_config["llm"]["provider"]
|
||||
embedder_id = oss_config["embedder"]["provider"]
|
||||
vector_id = oss_config["vector_store"]["provider"]
|
||||
|
||||
if flags.get("dry_run"):
|
||||
print("\n [dry-run] OSS config would be:")
|
||||
print(f" LLM: {oss_config['llm']['provider']} ({oss_config['llm']['config'].get('model', '')})")
|
||||
print(f" Embedder: {oss_config['embedder']['provider']} ({oss_config['embedder']['config'].get('model', '')})")
|
||||
print(f" Vector: {vector_id}")
|
||||
if env_writes:
|
||||
print(f" Env vars: {', '.join(env_writes.keys())}")
|
||||
_run_connectivity_checks(oss_config)
|
||||
print(" [dry-run] No files written.\n")
|
||||
return
|
||||
|
||||
if env_writes:
|
||||
_write_env(Path(hermes_home) / ".env", env_writes)
|
||||
_save_mem0_json(hermes_home, {"mode": "oss", "user_id": user_id, "agent_id": "hermes", "oss": oss_config})
|
||||
|
||||
_install_provider_deps(llm_id, embedder_id, vector_id)
|
||||
|
||||
from hermes_cli.config import save_config
|
||||
config["memory"]["provider"] = "mem0"
|
||||
save_config(config)
|
||||
|
||||
_run_connectivity_checks(oss_config)
|
||||
print("\n ✓ Mem0 configured (OSS mode)")
|
||||
print(f" LLM: {oss_config['llm']['provider']} ({oss_config['llm']['config'].get('model', '')})")
|
||||
print(f" Embedder: {oss_config['embedder']['provider']} ({oss_config['embedder']['config'].get('model', '')})")
|
||||
print(f" Vector: {vector_id}")
|
||||
if env_writes:
|
||||
print(" API keys saved to .env")
|
||||
print(" Config saved to mem0.json")
|
||||
print(" Provider set in config.yaml")
|
||||
print("\n Start a new session to activate.\n")
|
||||
|
||||
|
||||
def _prompt_api_key(label: str, env_var: str, hermes_home: str) -> str:
|
||||
"""Prompt for API key, showing masked existing value if found."""
|
||||
existing = os.environ.get(env_var, "")
|
||||
if not existing:
|
||||
env_path = Path(hermes_home) / ".env"
|
||||
if env_path.exists():
|
||||
for line in env_path.read_text().splitlines():
|
||||
if line.startswith(f"{env_var}="):
|
||||
existing = line.split("=", 1)[1].strip()
|
||||
break
|
||||
if existing:
|
||||
masked = f"...{existing[-4:]}" if len(existing) > 4 else "set"
|
||||
return getpass.getpass(f" {label} API key (current: {masked}, blank to keep): ").strip()
|
||||
return getpass.getpass(f" {label} API key: ").strip()
|
||||
|
||||
|
||||
_PGVECTOR_CONTAINER = "hermes-pgvector"
|
||||
_PGVECTOR_IMAGE = "pgvector/pgvector:pg17"
|
||||
_PGVECTOR_PASSWORD = "hermes"
|
||||
|
||||
|
||||
def _ensure_pgvector(host: str = "localhost", port: int = 5432) -> dict | None:
|
||||
"""Ensure pgvector is reachable; offer Docker setup if not.
|
||||
|
||||
Returns updated vector_config dict if Docker was started, None otherwise.
|
||||
"""
|
||||
ok, _ = _check_pgvector(host, port)
|
||||
if ok:
|
||||
print(f" ✓ PostgreSQL reachable at {host}:{port}")
|
||||
return None
|
||||
|
||||
print(f" PostgreSQL not reachable at {host}:{port}")
|
||||
|
||||
# Check if our container already exists but is stopped
|
||||
if shutil.which("docker"):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", _PGVECTOR_CONTAINER, "--format", "{{.State.Status}}"],
|
||||
capture_output=True, text=True, timeout=10, stdin=subprocess.DEVNULL,
|
||||
)
|
||||
if result.returncode == 0 and "exited" in result.stdout:
|
||||
print(f" Found stopped container '{_PGVECTOR_CONTAINER}', restarting...")
|
||||
subprocess.run(["docker", "start", _PGVECTOR_CONTAINER],
|
||||
capture_output=True, timeout=15,
|
||||
stdin=subprocess.DEVNULL)
|
||||
_wait_for_port(host, port, timeout=15)
|
||||
ok, _ = _check_pgvector(host, port)
|
||||
if ok:
|
||||
print(" ✓ PostgreSQL container restarted")
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
answer = input(" Start pgvector via Docker? [Y/n]: ").strip().lower()
|
||||
if answer in ("", "y", "yes"):
|
||||
return _start_pgvector_docker(host, port)
|
||||
else:
|
||||
print(" Skipping Docker setup. Make sure PostgreSQL with pgvector is running.")
|
||||
return None
|
||||
else:
|
||||
print(" Docker not found. Install Docker to auto-start pgvector,")
|
||||
print(" or run PostgreSQL with pgvector manually.")
|
||||
return None
|
||||
|
||||
|
||||
def _start_pgvector_docker(host: str, port: int) -> dict | None:
|
||||
"""Pull and start pgvector Docker container."""
|
||||
try:
|
||||
print(f" Pulling {_PGVECTOR_IMAGE}...")
|
||||
subprocess.run(["docker", "pull", _PGVECTOR_IMAGE],
|
||||
capture_output=True, timeout=120,
|
||||
stdin=subprocess.DEVNULL)
|
||||
|
||||
# Remove existing container if present
|
||||
subprocess.run(["docker", "rm", "-f", _PGVECTOR_CONTAINER],
|
||||
capture_output=True, timeout=10,
|
||||
stdin=subprocess.DEVNULL)
|
||||
|
||||
print(f" Starting container '{_PGVECTOR_CONTAINER}' on port {port}...")
|
||||
subprocess.run([
|
||||
"docker", "run", "-d",
|
||||
"--name", _PGVECTOR_CONTAINER,
|
||||
"-e", f"POSTGRES_PASSWORD={_PGVECTOR_PASSWORD}",
|
||||
"-p", f"{port}:5432",
|
||||
_PGVECTOR_IMAGE,
|
||||
], capture_output=True, timeout=30, check=True, stdin=subprocess.DEVNULL)
|
||||
|
||||
_wait_for_port(host, port, timeout=20)
|
||||
ok, _ = _check_pgvector(host, port)
|
||||
if ok:
|
||||
print(f" ✓ pgvector running on {host}:{port}")
|
||||
return {
|
||||
"host": host, "port": port,
|
||||
"user": "postgres", "password": _PGVECTOR_PASSWORD,
|
||||
"dbname": "postgres",
|
||||
}
|
||||
else:
|
||||
print(" Warning: Container started but PostgreSQL not yet accepting connections.")
|
||||
print(" It may need a few more seconds. Config will be saved; retry later.")
|
||||
return {
|
||||
"host": host, "port": port,
|
||||
"user": "postgres", "password": _PGVECTOR_PASSWORD,
|
||||
"dbname": "postgres",
|
||||
}
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f" Failed to start Docker container: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" Docker error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_ollama(models: list[str]) -> bool:
|
||||
"""Ensure Ollama is running and required models are pulled.
|
||||
|
||||
Returns True if Ollama is ready, False if user needs to handle it manually.
|
||||
"""
|
||||
url = "http://localhost:11434"
|
||||
ollama_bin = shutil.which("ollama")
|
||||
ok, _ = _check_ollama(url)
|
||||
|
||||
if not ok:
|
||||
if ollama_bin:
|
||||
print(" Ollama installed but not running. Starting...")
|
||||
try:
|
||||
subprocess.Popen(
|
||||
[ollama_bin, "serve"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
_wait_for_port("localhost", 11434, timeout=10)
|
||||
ok, _ = _check_ollama(url)
|
||||
if ok:
|
||||
print(" ✓ Ollama started")
|
||||
except Exception as e:
|
||||
print(f" Could not start Ollama: {e}")
|
||||
else:
|
||||
print(" Ollama not found. Install it:")
|
||||
print(" curl -fsSL https://ollama.com/install.sh | sh")
|
||||
print(" Or on macOS: brew install ollama")
|
||||
return False
|
||||
|
||||
if not ok:
|
||||
print(" Warning: Ollama not reachable. Models cannot be pulled.")
|
||||
return False
|
||||
|
||||
# Pull required models
|
||||
for model in models:
|
||||
if _ollama_has_model(url, model):
|
||||
print(f" ✓ Model '{model}' available")
|
||||
else:
|
||||
print(f" Pulling '{model}'... (this may take a few minutes)")
|
||||
try:
|
||||
subprocess.run([ollama_bin or "ollama", "pull", model], timeout=600,
|
||||
stdin=subprocess.DEVNULL)
|
||||
print(f" ✓ Model '{model}' pulled")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not pull '{model}': {e}")
|
||||
print(f" Run manually: ollama pull {model}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _ollama_has_model(url: str, model: str) -> bool:
|
||||
"""Check if Ollama already has a model pulled."""
|
||||
try:
|
||||
req = urllib.request.Request(f"{url}/api/tags", method="GET")
|
||||
resp = urllib.request.urlopen(req, timeout=5)
|
||||
data = json.loads(resp.read())
|
||||
names = [m.get("name", "") for m in data.get("models", [])]
|
||||
base_model = model.split(":")[0]
|
||||
return any(model in n or base_model in n for n in names)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_pgvector_extension(pg_config: dict) -> None:
|
||||
"""Create the pgvector extension if it doesn't exist."""
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
return
|
||||
conn_params = {
|
||||
"host": pg_config.get("host", "localhost"),
|
||||
"port": pg_config.get("port", 5432),
|
||||
"user": pg_config.get("user", "postgres"),
|
||||
"dbname": pg_config.get("dbname", "postgres"),
|
||||
}
|
||||
if pg_config.get("password"):
|
||||
conn_params["password"] = pg_config["password"]
|
||||
try:
|
||||
conn = psycopg2.connect(**conn_params)
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(" ✓ pgvector extension enabled")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not enable pgvector extension: {e}")
|
||||
|
||||
|
||||
def _wait_for_port(host: str, port: int, timeout: int = 15) -> None:
|
||||
"""Wait until a TCP port is accepting connections."""
|
||||
import time
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
sock = socket.create_connection((host, port), timeout=1)
|
||||
sock.close()
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _provider_description(v: dict) -> str:
|
||||
"""Description for LLM/embedder picker: model + URL if applicable."""
|
||||
model = v.get("default_model", "")
|
||||
url = v.get("default_url")
|
||||
if url:
|
||||
return f"{model} ({url})"
|
||||
return model
|
||||
|
||||
|
||||
def _vector_description(pid: str, v: dict) -> str:
|
||||
cfg = v.get("default_config", {})
|
||||
if pid == "qdrant":
|
||||
return cfg.get("path", "local storage")
|
||||
if pid == "pgvector":
|
||||
return f"{cfg.get('host', 'localhost')}:{cfg.get('port', 5432)}"
|
||||
return pid
|
||||
|
||||
|
||||
def _setup_oss_interactive(hermes_home: str, config: dict) -> None:
|
||||
"""Interactive OSS setup using curses pickers."""
|
||||
llm_items = [(v["label"], _provider_description(v)) for pid, v in LLM_PROVIDERS.items()]
|
||||
llm_idx = _curses_select("LLM Provider", llm_items, 0)
|
||||
llm_id = list(LLM_PROVIDERS.keys())[llm_idx]
|
||||
llm_def = LLM_PROVIDERS[llm_id]
|
||||
|
||||
env_writes: dict[str, str] = {}
|
||||
llm_model = llm_def["default_model"]
|
||||
llm_url = llm_def.get("default_url")
|
||||
if llm_def["needs_key"]:
|
||||
key = _prompt_api_key(llm_def["label"], llm_def["env_var"], hermes_home)
|
||||
if key:
|
||||
env_writes[llm_def["env_var"]] = key
|
||||
if llm_id == "ollama":
|
||||
llm_model = input(f" LLM model [{llm_def['default_model']}]: ").strip() or llm_def["default_model"]
|
||||
llm_url = input(f" Ollama URL [{llm_def['default_url']}]: ").strip() or llm_def["default_url"]
|
||||
|
||||
embedder_items = [(v["label"], _provider_description(v)) for pid, v in EMBEDDER_PROVIDERS.items()]
|
||||
embedder_idx = _curses_select("Embedder Provider", embedder_items, 0)
|
||||
embedder_id = list(EMBEDDER_PROVIDERS.keys())[embedder_idx]
|
||||
embedder_def = EMBEDDER_PROVIDERS[embedder_id]
|
||||
|
||||
embedder_model = embedder_def["default_model"]
|
||||
embedder_url = embedder_def.get("default_url")
|
||||
if embedder_def["needs_key"] and embedder_id != llm_id:
|
||||
key = _prompt_api_key(f"{embedder_def['label']} embedder", embedder_def["env_var"], hermes_home)
|
||||
if key:
|
||||
env_writes[embedder_def["env_var"]] = key
|
||||
elif embedder_def["needs_key"] and embedder_id == llm_id:
|
||||
if llm_def.get("env_var") in env_writes:
|
||||
env_writes[embedder_def["env_var"]] = env_writes[llm_def["env_var"]]
|
||||
if embedder_id == "ollama":
|
||||
embedder_model = input(f" Embedder model [{embedder_def['default_model']}]: ").strip() or embedder_def["default_model"]
|
||||
embedder_url = input(f" Ollama URL [{embedder_def['default_url']}]: ").strip() or embedder_def["default_url"]
|
||||
|
||||
vector_items = [(v["label"], _vector_description(pid, v)) for pid, v in VECTOR_PROVIDERS.items()]
|
||||
vector_idx = _curses_select("Vector Store", vector_items, 0)
|
||||
vector_id = list(VECTOR_PROVIDERS.keys())[vector_idx]
|
||||
|
||||
# Auto-setup: ensure Ollama is running and models are pulled
|
||||
ollama_models = []
|
||||
if llm_id == "ollama":
|
||||
ollama_models.append(llm_model)
|
||||
if embedder_id == "ollama":
|
||||
ollama_models.append(embedder_model)
|
||||
if ollama_models:
|
||||
_ensure_ollama(ollama_models)
|
||||
|
||||
# Auto-setup: ensure pgvector is reachable (offer Docker if not)
|
||||
pgvector_config = None
|
||||
if vector_id == "pgvector":
|
||||
pgvector_config = _ensure_pgvector()
|
||||
if not pgvector_config:
|
||||
# Native PostgreSQL — prompt for connection details
|
||||
default_user = os.getenv("USER", "postgres")
|
||||
pg_user = input(f" PostgreSQL user [{default_user}]: ").strip() or default_user
|
||||
pg_host = input(" PostgreSQL host [localhost]: ").strip() or "localhost"
|
||||
pg_port = input(" PostgreSQL port [5432]: ").strip() or "5432"
|
||||
pg_dbname = input(" PostgreSQL database [postgres]: ").strip() or "postgres"
|
||||
pg_password = getpass.getpass(" PostgreSQL password (blank if none): ").strip()
|
||||
pgvector_config = {
|
||||
"host": pg_host, "port": int(pg_port),
|
||||
"user": pg_user, "dbname": pg_dbname,
|
||||
}
|
||||
if pg_password:
|
||||
pgvector_config["password"] = pg_password
|
||||
|
||||
user_id = input(f" User ID [{os.getenv('USER', 'hermes-user')}]: ").strip()
|
||||
user_id = user_id or os.getenv("USER", "hermes-user")
|
||||
|
||||
agent_id = input(" Agent ID [hermes]: ").strip()
|
||||
agent_id = agent_id or "hermes"
|
||||
|
||||
flags = {
|
||||
"oss_llm": llm_id,
|
||||
"oss_llm_key": env_writes.get(llm_def["env_var"], "") if llm_def.get("env_var") else "",
|
||||
"oss_llm_model": llm_model,
|
||||
"oss_llm_url": llm_url or "",
|
||||
"oss_embedder": embedder_id,
|
||||
"oss_embedder_model": embedder_model,
|
||||
"oss_embedder_url": embedder_url or "",
|
||||
"oss_vector": vector_id,
|
||||
"user_id": user_id,
|
||||
}
|
||||
|
||||
if pgvector_config:
|
||||
flags["oss_vector_host"] = pgvector_config["host"]
|
||||
flags["oss_vector_port"] = str(pgvector_config["port"])
|
||||
flags["oss_vector_user"] = pgvector_config["user"]
|
||||
if pgvector_config.get("password"):
|
||||
flags["oss_vector_password"] = pgvector_config["password"]
|
||||
flags["oss_vector_dbname"] = pgvector_config["dbname"]
|
||||
|
||||
oss_config, _ = build_oss_config(flags)
|
||||
|
||||
if env_writes:
|
||||
_write_env(Path(hermes_home) / ".env", env_writes)
|
||||
_save_mem0_json(hermes_home, {"mode": "oss", "user_id": user_id, "agent_id": agent_id, "oss": oss_config})
|
||||
|
||||
_install_provider_deps(llm_id, embedder_id, vector_id)
|
||||
|
||||
if vector_id == "pgvector" and pgvector_config:
|
||||
_ensure_pgvector_extension(pgvector_config)
|
||||
|
||||
from hermes_cli.config import save_config
|
||||
config["memory"]["provider"] = "mem0"
|
||||
save_config(config)
|
||||
|
||||
_run_connectivity_checks(oss_config)
|
||||
print("\n ✓ Mem0 configured (OSS mode)")
|
||||
print(f" LLM: {oss_config['llm']['provider']} ({oss_config['llm']['config'].get('model', '')})")
|
||||
print(f" Embedder: {oss_config['embedder']['provider']} ({oss_config['embedder']['config'].get('model', '')})")
|
||||
print(f" Vector: {vector_id}")
|
||||
if env_writes:
|
||||
print(" API keys saved to .env")
|
||||
print(" Config saved to mem0.json")
|
||||
print(" Provider set in config.yaml")
|
||||
print("\n Start a new session to activate.\n")
|
||||
|
||||
|
||||
def _install_provider_deps(llm_id: str, embedder_id: str, vector_id: str) -> None:
|
||||
"""Install all optional pip deps for selected providers."""
|
||||
deps: set[str] = set()
|
||||
for registry, pid in [(LLM_PROVIDERS, llm_id), (EMBEDDER_PROVIDERS, embedder_id),
|
||||
(VECTOR_PROVIDERS, vector_id)]:
|
||||
dep = registry.get(pid, {}).get("pip_dep")
|
||||
if dep:
|
||||
deps.add(dep)
|
||||
for dep in sorted(deps):
|
||||
try:
|
||||
print(f" Installing {dep}...")
|
||||
subprocess.run(
|
||||
["uv", "pip", "install", "--python", sys.executable, dep],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
print(f" ✓ Installed {dep}")
|
||||
except Exception:
|
||||
print(f" Warning: Could not install {dep}. Install manually: uv pip install {dep}")
|
||||
if deps:
|
||||
import importlib
|
||||
importlib.invalidate_caches()
|
||||
|
||||
|
||||
def _check_qdrant_path(path: str) -> tuple[bool, str]:
|
||||
"""Check that qdrant local storage parent dir is writable."""
|
||||
p = Path(path).expanduser()
|
||||
parent = p.parent
|
||||
try:
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
return True, f"Directory writable: {parent}"
|
||||
except OSError as e:
|
||||
return False, f"Cannot write to {parent}: {e}"
|
||||
|
||||
|
||||
def _check_ollama(url: str) -> tuple[bool, str]:
|
||||
"""Check Ollama is reachable via /api/tags."""
|
||||
try:
|
||||
req = urllib.request.Request(f"{url.rstrip('/')}/api/tags", method="GET")
|
||||
urllib.request.urlopen(req, timeout=3)
|
||||
return True, "Ollama reachable"
|
||||
except Exception as e:
|
||||
return False, f"Ollama not reachable at {url}: {e}"
|
||||
|
||||
|
||||
def _check_pgvector(host: str, port: int) -> tuple[bool, str]:
|
||||
"""Check PGVector via TCP socket."""
|
||||
try:
|
||||
sock = socket.create_connection((host, port), timeout=3)
|
||||
sock.close()
|
||||
return True, f"PGVector reachable at {host}:{port}"
|
||||
except Exception as e:
|
||||
return False, f"PGVector not reachable at {host}:{port}: {e}"
|
||||
|
||||
|
||||
def _run_connectivity_checks(oss_config: dict) -> None:
|
||||
"""Run connectivity checks and print warnings."""
|
||||
vs = oss_config.get("vector_store", {})
|
||||
if vs.get("provider") == "qdrant":
|
||||
path = vs.get("config", {}).get("path")
|
||||
url = vs.get("config", {}).get("url")
|
||||
if path:
|
||||
ok, msg = _check_qdrant_path(path)
|
||||
if not ok:
|
||||
print(f" Warning: {msg}")
|
||||
elif url:
|
||||
try:
|
||||
req = urllib.request.Request(f"{url.rstrip('/')}/healthz", method="GET")
|
||||
urllib.request.urlopen(req, timeout=3)
|
||||
except Exception as e:
|
||||
print(f" Warning: Qdrant not reachable at {url}: {e}")
|
||||
elif vs.get("provider") == "pgvector":
|
||||
cfg = vs.get("config", {})
|
||||
ok, msg = _check_pgvector(cfg.get("host", "localhost"), cfg.get("port", 5432))
|
||||
if not ok:
|
||||
print(f" Warning: {msg}")
|
||||
|
||||
llm = oss_config.get("llm", {})
|
||||
if llm.get("provider") == "ollama":
|
||||
url = llm.get("config", {}).get("ollama_base_url", "http://localhost:11434")
|
||||
ok, msg = _check_ollama(url)
|
||||
if not ok:
|
||||
print(f" Warning: {msg}")
|
||||
|
||||
|
||||
def _check_min_dep_version() -> None:
|
||||
"""Ensure mem0ai meets the minimum version from plugin.yaml."""
|
||||
try:
|
||||
import mem0
|
||||
installed_ver = getattr(mem0, "__version__", None)
|
||||
if not installed_ver:
|
||||
return
|
||||
installed_parts = tuple(int(x) for x in installed_ver.split(".")[:3])
|
||||
required_parts = (2, 0, 7)
|
||||
if installed_parts < required_parts:
|
||||
req_str = ".".join(str(x) for x in required_parts)
|
||||
print(f"\n ⚠ mem0ai {installed_ver} installed but >={req_str} required.")
|
||||
print(f" Run: uv pip install --python {sys.executable} 'mem0ai>={req_str}'")
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def post_setup(hermes_home: str, config: dict) -> None:
|
||||
"""Entry point called by hermes memory setup framework.
|
||||
|
||||
Routes on --mode (platform / selfhosted / oss); with no flag it shows an
|
||||
interactive picker with all three modes. Platform keeps the framework's
|
||||
original schema-based onboarding; selfhosted points at an existing Mem0
|
||||
server; oss builds a local SDK config.
|
||||
"""
|
||||
_check_min_dep_version()
|
||||
flags = parse_flags(sys.argv[1:])
|
||||
|
||||
if flags["mode"] == "oss":
|
||||
flags["_mode_from_flag"] = True
|
||||
_setup_oss(hermes_home, config, flags)
|
||||
return
|
||||
|
||||
if flags["mode"] in ("selfhosted", "self-hosted"):
|
||||
_setup_selfhosted(hermes_home, config, flags)
|
||||
return
|
||||
|
||||
if flags["mode"] == "platform":
|
||||
_setup_platform(hermes_home, config, flags)
|
||||
return
|
||||
|
||||
# No --mode flag: show interactive picker
|
||||
mode_items = [
|
||||
("Platform", "Mem0 Cloud API (lightweight, just needs an API key)"),
|
||||
("Self-hosted server", "Connect to an existing self-hosted Mem0 server (Docker/FastAPI)"),
|
||||
("Open Source", "Run Mem0 locally (self-hosted LLM + vector store)"),
|
||||
]
|
||||
mode_idx = _curses_select(" Select mode", mode_items, 0)
|
||||
if mode_idx == 1:
|
||||
_setup_selfhosted(hermes_home, config, flags)
|
||||
elif mode_idx == 2:
|
||||
flags["_mode_from_flag"] = False
|
||||
_setup_oss(hermes_home, config, flags)
|
||||
else:
|
||||
_setup_platform(hermes_home, config, flags)
|
||||
@@ -0,0 +1,5 @@
|
||||
name: mem0
|
||||
version: 1.3.0
|
||||
description: "Mem0 — server-side LLM fact extraction with semantic search, automatic deduplication, and opt-in reranking (platform mode)."
|
||||
pip_dependencies:
|
||||
- mem0ai>=2.0.10,<3
|
||||
@@ -0,0 +1,83 @@
|
||||
# OpenViking Memory Provider
|
||||
|
||||
Context database by Volcengine (ByteDance) with filesystem-style knowledge hierarchy, tiered retrieval, and automatic memory extraction.
|
||||
|
||||
## Requirements
|
||||
|
||||
- `pip install openviking`
|
||||
- OpenViking server running (`openviking-server`)
|
||||
- Embedding + VLM model configured in `~/.openviking/ov.conf`
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "openviking"
|
||||
```
|
||||
|
||||
The setup can link to an existing `~/.openviking/ovcli.conf`, copy its current
|
||||
connection values into Hermes, or create a minimal `ovcli.conf` when one does
|
||||
not exist.
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider openviking
|
||||
echo "OPENVIKING_ENDPOINT=http://localhost:1933" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
All config via environment variables in `.env`:
|
||||
|
||||
| Env Var | Default | Description |
|
||||
|---------|---------|-------------|
|
||||
| `OPENVIKING_ENDPOINT` | `http://127.0.0.1:1933` | Server URL |
|
||||
| `OPENVIKING_API_KEY` | (none) | User/admin API key for authenticated servers |
|
||||
| `OPENVIKING_ACCOUNT` | `default` | Tenant account for local/trusted mode |
|
||||
| `OPENVIKING_USER` | `default` | Tenant user for local/trusted mode |
|
||||
| `OPENVIKING_AGENT` | `hermes` | Hermes peer ID in OpenViking, used for peer-scoped memories |
|
||||
|
||||
When `OPENVIKING_API_KEY` is set, Hermes lets OpenViking derive account/user
|
||||
identity from the key. In local or trusted deployments without an API key,
|
||||
Hermes sends `OPENVIKING_ACCOUNT` and `OPENVIKING_USER` as identity headers.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `viking_search` | Semantic search with fast/deep/auto modes |
|
||||
| `viking_read` | Read content at a viking:// URI (abstract/overview/full) |
|
||||
| `viking_browse` | Filesystem-style navigation (list/tree/stat) |
|
||||
| `viking_remember` | Store a fact directly with OpenViking `content/write` |
|
||||
| `viking_forget` | Delete one exact `viking://` memory file URI |
|
||||
| `viking_add_resource` | Ingest URLs/docs into the knowledge base |
|
||||
|
||||
## Memory Writes And Deletes
|
||||
|
||||
`viking_remember` writes directly to OpenViking with `POST /api/v1/content/write`
|
||||
and `mode=create`. It creates peer-scoped memory files under
|
||||
`viking://user/peers/${OPENVIKING_AGENT}/memories/...`; OpenViking may return a
|
||||
canonical user-scoped form such as
|
||||
`viking://user/default/peers/${OPENVIKING_AGENT}/memories/...` in API-key mode.
|
||||
Explicit remembers do not depend on session commit extraction.
|
||||
|
||||
Hermes built-in `memory` tool additions are mirrored to OpenViking after the
|
||||
local memory operation succeeds:
|
||||
|
||||
| Hermes action | OpenViking operation |
|
||||
|---------------|----------------------|
|
||||
| `add` | `content/write` with `mode=create` under the configured peer memory namespace |
|
||||
|
||||
Built-in `replace` and `remove` operations are not mirrored because Hermes
|
||||
native memory entries do not yet carry stable OpenViking file URIs. Use
|
||||
`viking_forget` when the user explicitly asks to delete a specific OpenViking
|
||||
memory URI.
|
||||
|
||||
`viking_forget` is intentionally narrow. It only accepts concrete user memory
|
||||
file URIs, such as
|
||||
`viking://user/peers/hermes/memories/preferences/mem_abc123.md` or the canonical
|
||||
`viking://user/default/peers/hermes/memories/preferences/mem_abc123.md`. Files
|
||||
directly under `memories/`, such as `viking://user/default/memories/profile.md`,
|
||||
are also allowed because OpenViking supports them. The tool rejects directories,
|
||||
resources, skills, sessions, generated summary files, and URIs with query
|
||||
strings or fragments. Use OpenViking's MCP, CLI, or admin APIs for broader
|
||||
resource and directory cleanup.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
name: openviking
|
||||
version: 2.0.0
|
||||
description: "OpenViking context database — session-managed memory with automatic extraction, tiered retrieval, and filesystem-style knowledge browsing."
|
||||
pip_dependencies:
|
||||
- httpx
|
||||
requires_env: []
|
||||
hooks:
|
||||
- on_session_end
|
||||
@@ -0,0 +1,40 @@
|
||||
# RetainDB Memory Provider
|
||||
|
||||
Cloud memory API with hybrid search (Vector + BM25 + Reranking) and 7 memory types.
|
||||
|
||||
## Requirements
|
||||
|
||||
- RetainDB account ($20/month) from [retaindb.com](https://www.retaindb.com)
|
||||
- `pip install requests`
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "retaindb"
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
hermes config set memory.provider retaindb
|
||||
echo "RETAINDB_API_KEY=your-key" >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
All config via environment variables in `.env`:
|
||||
|
||||
| Env Var | Default | Description |
|
||||
|---------|---------|-------------|
|
||||
| `RETAINDB_API_KEY` | (required) | API key |
|
||||
| `RETAINDB_BASE_URL` | `https://api.retaindb.com` | API endpoint |
|
||||
| `RETAINDB_PROJECT` | auto (profile-scoped) | Project identifier |
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `retaindb_profile` | User's stable profile |
|
||||
| `retaindb_search` | Semantic search |
|
||||
| `retaindb_context` | Task-relevant context |
|
||||
| `retaindb_remember` | Store a fact with type + importance |
|
||||
| `retaindb_forget` | Delete a memory by ID |
|
||||
@@ -0,0 +1,771 @@
|
||||
"""RetainDB memory plugin — MemoryProvider interface.
|
||||
|
||||
Cross-session memory via RetainDB cloud API.
|
||||
|
||||
Features:
|
||||
- Correct API routes for all operations
|
||||
- Durable SQLite write-behind queue (crash-safe, async ingest)
|
||||
- Semantic search + user profile retrieval
|
||||
- Context query with deduplication overlay
|
||||
- Dialectic synthesis (LLM-powered user understanding, prefetched each turn)
|
||||
- Agent self-model (persona + instructions from SOUL.md, prefetched each turn)
|
||||
- Shared file store tools (upload, list, read, ingest, delete)
|
||||
- Explicit memory tools (profile, search, context, remember, forget)
|
||||
|
||||
Config (env vars or hermes config.yaml under retaindb:):
|
||||
RETAINDB_API_KEY — API key (required)
|
||||
RETAINDB_BASE_URL — API endpoint (default: https://api.retaindb.com)
|
||||
RETAINDB_PROJECT — Project identifier (optional — defaults to "default")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import quote
|
||||
|
||||
from agent.memory_provider import MemoryProvider
|
||||
from agent.file_safety import raise_if_read_blocked
|
||||
from tools.registry import tool_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_BASE_URL = "https://api.retaindb.com"
|
||||
_ASYNC_SHUTDOWN = object()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PROFILE_SCHEMA = {
|
||||
"name": "retaindb_profile",
|
||||
"description": "Get the user's stable profile — preferences, facts, and patterns recalled from long-term memory.",
|
||||
"parameters": {"type": "object", "properties": {}, "required": []},
|
||||
}
|
||||
|
||||
SEARCH_SCHEMA = {
|
||||
"name": "retaindb_search",
|
||||
"description": "Semantic search across stored memories. Returns ranked results with relevance scores.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "What to search for."},
|
||||
"top_k": {"type": "integer", "description": "Max results (default: 8, max: 20)."},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
}
|
||||
|
||||
CONTEXT_SCHEMA = {
|
||||
"name": "retaindb_context",
|
||||
"description": "Synthesized context block — what matters most for the current task, pulled from long-term memory.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Current task or question."},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
}
|
||||
|
||||
REMEMBER_SCHEMA = {
|
||||
"name": "retaindb_remember",
|
||||
"description": "Persist an explicit fact, preference, or decision to long-term memory.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string", "description": "The fact to remember."},
|
||||
"memory_type": {
|
||||
"type": "string",
|
||||
"enum": ["factual", "preference", "goal", "instruction", "event", "opinion"],
|
||||
"description": "Category (default: factual).",
|
||||
},
|
||||
"importance": {"type": "number", "description": "Importance 0-1 (default: 0.7)."},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
}
|
||||
|
||||
FORGET_SCHEMA = {
|
||||
"name": "retaindb_forget",
|
||||
"description": "Delete a specific memory by ID.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"memory_id": {"type": "string", "description": "Memory ID to delete."},
|
||||
},
|
||||
"required": ["memory_id"],
|
||||
},
|
||||
}
|
||||
|
||||
FILE_UPLOAD_SCHEMA = {
|
||||
"name": "retaindb_upload_file",
|
||||
"description": "Upload a file to the shared RetainDB file store. Returns an rdb:// URI any agent can reference.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"local_path": {"type": "string", "description": "Local file path to upload."},
|
||||
"remote_path": {"type": "string", "description": "Destination path, e.g. /reports/q1.pdf"},
|
||||
"scope": {"type": "string", "enum": ["USER", "PROJECT", "ORG"], "description": "Access scope (default: PROJECT)."},
|
||||
"ingest": {"type": "boolean", "description": "Also extract memories from file after upload (default: false)."},
|
||||
},
|
||||
"required": ["local_path"],
|
||||
},
|
||||
}
|
||||
|
||||
FILE_LIST_SCHEMA = {
|
||||
"name": "retaindb_list_files",
|
||||
"description": "List files in the shared file store.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prefix": {"type": "string", "description": "Path prefix to filter by, e.g. /reports/"},
|
||||
"limit": {"type": "integer", "description": "Max results (default: 50)."},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
}
|
||||
|
||||
FILE_READ_SCHEMA = {
|
||||
"name": "retaindb_read_file",
|
||||
"description": "Read the text content of a stored file by its file ID.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_id": {"type": "string", "description": "File ID returned from upload or list."},
|
||||
},
|
||||
"required": ["file_id"],
|
||||
},
|
||||
}
|
||||
|
||||
FILE_INGEST_SCHEMA = {
|
||||
"name": "retaindb_ingest_file",
|
||||
"description": "Chunk, embed, and extract memories from a stored file. Makes its contents searchable.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_id": {"type": "string", "description": "File ID to ingest."},
|
||||
},
|
||||
"required": ["file_id"],
|
||||
},
|
||||
}
|
||||
|
||||
FILE_DELETE_SCHEMA = {
|
||||
"name": "retaindb_delete_file",
|
||||
"description": "Delete a stored file.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_id": {"type": "string", "description": "File ID to delete."},
|
||||
},
|
||||
"required": ["file_id"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _Client:
|
||||
def __init__(self, api_key: str, base_url: str, project: str):
|
||||
self.api_key = api_key
|
||||
self.base_url = re.sub(r"/+$", "", base_url)
|
||||
self.project = project
|
||||
|
||||
def _headers(self, path: str) -> dict:
|
||||
token = self.api_key.replace("Bearer ", "").strip()
|
||||
h = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"x-sdk-runtime": "hermes-plugin",
|
||||
}
|
||||
if path.startswith(("/v1/memory", "/v1/context")):
|
||||
h["X-API-Key"] = token
|
||||
return h
|
||||
|
||||
def request(self, method: str, path: str, *, params=None, json_body=None, timeout: float = 8.0) -> Any:
|
||||
import requests
|
||||
url = f"{self.base_url}{path}"
|
||||
resp = requests.request(
|
||||
method.upper(), url,
|
||||
params=params,
|
||||
json=json_body if method.upper() not in {"GET", "DELETE"} else None,
|
||||
headers=self._headers(path),
|
||||
timeout=timeout,
|
||||
)
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
payload = resp.text
|
||||
if not resp.ok:
|
||||
msg = ""
|
||||
if isinstance(payload, dict):
|
||||
msg = str(payload.get("message") or payload.get("error") or "")
|
||||
raise RuntimeError(f"RetainDB {method} {path} failed ({resp.status_code}): {msg or payload}")
|
||||
return payload
|
||||
|
||||
# ── Memory ────────────────────────────────────────────────────────────────
|
||||
|
||||
def query_context(self, user_id: str, session_id: str, query: str, max_tokens: int = 1200) -> dict:
|
||||
return self.request("POST", "/v1/context/query", json_body={
|
||||
"project": self.project,
|
||||
"query": query,
|
||||
"user_id": user_id,
|
||||
"session_id": session_id,
|
||||
"include_memories": True,
|
||||
"max_tokens": max_tokens,
|
||||
})
|
||||
|
||||
def search(self, user_id: str, session_id: str, query: str, top_k: int = 8) -> dict:
|
||||
return self.request("POST", "/v1/memory/search", json_body={
|
||||
"project": self.project,
|
||||
"query": query,
|
||||
"user_id": user_id,
|
||||
"session_id": session_id,
|
||||
"top_k": top_k,
|
||||
"include_pending": True,
|
||||
})
|
||||
|
||||
def get_profile(self, user_id: str) -> dict:
|
||||
try:
|
||||
return self.request("GET", f"/v1/memory/profile/{quote(user_id, safe='')}", params={"project": self.project, "include_pending": "true"})
|
||||
except Exception:
|
||||
return self.request("GET", "/v1/memories", params={"project": self.project, "user_id": user_id, "limit": "200"})
|
||||
|
||||
def add_memory(self, user_id: str, session_id: str, content: str, memory_type: str = "factual", importance: float = 0.7) -> dict:
|
||||
try:
|
||||
return self.request("POST", "/v1/memory", json_body={
|
||||
"project": self.project, "content": content, "memory_type": memory_type,
|
||||
"user_id": user_id, "session_id": session_id, "importance": importance, "write_mode": "sync",
|
||||
}, timeout=5.0)
|
||||
except Exception:
|
||||
return self.request("POST", "/v1/memories", json_body={
|
||||
"project": self.project, "content": content, "memory_type": memory_type,
|
||||
"user_id": user_id, "session_id": session_id, "importance": importance,
|
||||
}, timeout=5.0)
|
||||
|
||||
def delete_memory(self, memory_id: str) -> dict:
|
||||
try:
|
||||
return self.request("DELETE", f"/v1/memory/{quote(memory_id, safe='')}", timeout=5.0)
|
||||
except Exception:
|
||||
return self.request("DELETE", f"/v1/memories/{quote(memory_id, safe='')}", timeout=5.0)
|
||||
|
||||
def ingest_session(self, user_id: str, session_id: str, messages: list, timeout: float = 15.0) -> dict:
|
||||
return self.request("POST", "/v1/memory/ingest/session", json_body={
|
||||
"project": self.project, "session_id": session_id, "user_id": user_id,
|
||||
"messages": messages, "write_mode": "sync",
|
||||
}, timeout=timeout)
|
||||
|
||||
def ask_user(self, user_id: str, query: str, reasoning_level: str = "low") -> dict:
|
||||
return self.request("POST", f"/v1/memory/profile/{quote(user_id, safe='')}/ask", json_body={
|
||||
"project": self.project, "query": query, "reasoning_level": reasoning_level,
|
||||
}, timeout=8.0)
|
||||
|
||||
def get_agent_model(self, agent_id: str) -> dict:
|
||||
return self.request("GET", f"/v1/memory/agent/{quote(agent_id, safe='')}/model", params={"project": self.project}, timeout=4.0)
|
||||
|
||||
def seed_agent_identity(self, agent_id: str, content: str, source: str = "soul_md") -> dict:
|
||||
return self.request("POST", f"/v1/memory/agent/{quote(agent_id, safe='')}/seed", json_body={
|
||||
"project": self.project, "content": content, "source": source,
|
||||
}, timeout=20.0)
|
||||
|
||||
# ── Files ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def upload_file(self, data: bytes, filename: str, remote_path: str, mime_type: str, scope: str, project_id: str | None) -> dict:
|
||||
import io
|
||||
import requests
|
||||
url = f"{self.base_url}/v1/files"
|
||||
token = self.api_key.replace("Bearer ", "").strip()
|
||||
headers = {"Authorization": f"Bearer {token}", "x-sdk-runtime": "hermes-plugin"}
|
||||
fields = {"path": remote_path, "scope": scope.upper()}
|
||||
if project_id:
|
||||
fields["project_id"] = project_id
|
||||
resp = requests.post(url, files={"file": (filename, io.BytesIO(data), mime_type)}, data=fields, headers=headers, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def list_files(self, prefix: str | None = None, limit: int = 50) -> dict:
|
||||
params: dict = {"limit": limit}
|
||||
if prefix:
|
||||
params["prefix"] = prefix
|
||||
return self.request("GET", "/v1/files", params=params)
|
||||
|
||||
def get_file(self, file_id: str) -> dict:
|
||||
return self.request("GET", f"/v1/files/{quote(file_id, safe='')}")
|
||||
|
||||
def read_file_content(self, file_id: str) -> bytes:
|
||||
import requests
|
||||
token = self.api_key.replace("Bearer ", "").strip()
|
||||
url = f"{self.base_url}/v1/files/{quote(file_id, safe='')}/content"
|
||||
resp = requests.get(url, headers={"Authorization": f"Bearer {token}", "x-sdk-runtime": "hermes-plugin"}, timeout=30, allow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
def ingest_file(self, file_id: str, user_id: str | None = None, agent_id: str | None = None) -> dict:
|
||||
body: dict = {}
|
||||
if user_id:
|
||||
body["user_id"] = user_id
|
||||
if agent_id:
|
||||
body["agent_id"] = agent_id
|
||||
return self.request("POST", f"/v1/files/{quote(file_id, safe='')}/ingest", json_body=body, timeout=60.0)
|
||||
|
||||
def delete_file(self, file_id: str) -> dict:
|
||||
return self.request("DELETE", f"/v1/files/{quote(file_id, safe='')}", timeout=5.0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Durable write-behind queue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _WriteQueue:
|
||||
"""SQLite-backed async write queue. Survives crashes — pending rows replay on startup."""
|
||||
|
||||
def __init__(self, client: _Client, db_path: Path):
|
||||
self._client = client
|
||||
self._db_path = db_path
|
||||
self._q: queue.Queue = queue.Queue()
|
||||
self._thread = threading.Thread(target=self._loop, name="retaindb-writer", daemon=True)
|
||||
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Thread-local connection cache — one connection per thread, reused.
|
||||
self._local = threading.local()
|
||||
self._init_db()
|
||||
self._thread.start()
|
||||
# Replay any rows left from a previous crash
|
||||
for row_id, user_id, session_id, msgs_json in self._pending_rows():
|
||||
self._q.put((row_id, user_id, session_id, json.loads(msgs_json)))
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
"""Return a cached connection for the current thread."""
|
||||
conn = getattr(self._local, "conn", None)
|
||||
if conn is None:
|
||||
conn = sqlite3.connect(str(self._db_path), timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
self._local.conn = conn
|
||||
return conn
|
||||
|
||||
def _init_db(self) -> None:
|
||||
conn = self._get_conn()
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS pending (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT, session_id TEXT, messages_json TEXT,
|
||||
created_at TEXT, last_error TEXT
|
||||
)""")
|
||||
conn.commit()
|
||||
|
||||
def _pending_rows(self) -> list:
|
||||
conn = self._get_conn()
|
||||
return conn.execute("SELECT id, user_id, session_id, messages_json FROM pending ORDER BY id ASC LIMIT 200").fetchall()
|
||||
|
||||
def enqueue(self, user_id: str, session_id: str, messages: list) -> None:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conn = self._get_conn()
|
||||
cur = conn.execute(
|
||||
"INSERT INTO pending (user_id, session_id, messages_json, created_at) VALUES (?,?,?,?)",
|
||||
(user_id, session_id, json.dumps(messages, ensure_ascii=False), now),
|
||||
)
|
||||
row_id = cur.lastrowid
|
||||
conn.commit()
|
||||
self._q.put((row_id, user_id, session_id, messages))
|
||||
|
||||
def _flush_row(self, row_id: int, user_id: str, session_id: str, messages: list) -> None:
|
||||
try:
|
||||
self._client.ingest_session(user_id, session_id, messages)
|
||||
conn = self._get_conn()
|
||||
conn.execute("DELETE FROM pending WHERE id = ?", (row_id,))
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
logger.warning("RetainDB ingest failed (will retry): %s", exc)
|
||||
conn = self._get_conn()
|
||||
conn.execute("UPDATE pending SET last_error = ? WHERE id = ?", (str(exc), row_id))
|
||||
conn.commit()
|
||||
time.sleep(2)
|
||||
|
||||
def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
item = self._q.get(timeout=5)
|
||||
if item is _ASYNC_SHUTDOWN:
|
||||
break
|
||||
self._flush_row(*item)
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.error("RetainDB writer error: %s", exc)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._q.put(_ASYNC_SHUTDOWN)
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Overlay formatter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_overlay(profile: dict, query_result: dict, local_entries: list[str] | None = None) -> str:
|
||||
def _compact(s: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(s or "")).strip()[:320]
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
return re.sub(r"[^a-z0-9 ]", "", _compact(s).lower())
|
||||
|
||||
seen: list[str] = [_norm(e) for e in (local_entries or []) if _norm(e)]
|
||||
profile_items: list[str] = []
|
||||
for m in list((profile or {}).get("memories") or [])[:5]:
|
||||
c = _compact((m or {}).get("content") or "")
|
||||
n = _norm(c)
|
||||
if c and n not in seen:
|
||||
seen.append(n)
|
||||
profile_items.append(c)
|
||||
|
||||
query_items: list[str] = []
|
||||
for r in list((query_result or {}).get("results") or [])[:5]:
|
||||
c = _compact((r or {}).get("content") or "")
|
||||
n = _norm(c)
|
||||
if c and n not in seen:
|
||||
seen.append(n)
|
||||
query_items.append(c)
|
||||
|
||||
if not profile_items and not query_items:
|
||||
return ""
|
||||
|
||||
lines = ["[RetainDB Context]", "Profile:"]
|
||||
lines += [f"- {i}" for i in profile_items] or ["- None"]
|
||||
lines.append("Relevant memories:")
|
||||
lines += [f"- {i}" for i in query_items] or ["- None"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main plugin class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RetainDBMemoryProvider(MemoryProvider):
|
||||
"""RetainDB cloud memory — durable queue, semantic search, dialectic synthesis, shared files."""
|
||||
|
||||
def __init__(self):
|
||||
self._client: _Client | None = None
|
||||
self._queue: _WriteQueue | None = None
|
||||
self._user_id = "default"
|
||||
self._session_id = ""
|
||||
self._agent_id = "hermes"
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Prefetch caches
|
||||
self._context_result = ""
|
||||
self._dialectic_result = ""
|
||||
self._agent_model: dict = {}
|
||||
|
||||
# Prefetch thread tracking — prevents accumulation on rapid calls
|
||||
self._prefetch_threads: list[threading.Thread] = []
|
||||
|
||||
# ── Core identity ──────────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "retaindb"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(os.environ.get("RETAINDB_API_KEY"))
|
||||
|
||||
def get_config_schema(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{"key": "api_key", "description": "RetainDB API key", "secret": True, "required": True, "env_var": "RETAINDB_API_KEY", "url": "https://retaindb.com"},
|
||||
{"key": "base_url", "description": "API endpoint", "default": _DEFAULT_BASE_URL},
|
||||
{"key": "project", "description": "Project identifier (optional — uses 'default' project if not set)", "default": ""},
|
||||
]
|
||||
|
||||
# ── Lifecycle ──────────────────────────────────────────────────────────
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
api_key = os.environ.get("RETAINDB_API_KEY", "")
|
||||
base_url = re.sub(r"/+$", "", os.environ.get("RETAINDB_BASE_URL", _DEFAULT_BASE_URL))
|
||||
|
||||
# Project resolution: RETAINDB_PROJECT > hermes-<profile> > "default"
|
||||
# If unset, the API auto-creates and uses the "default" project — no config required.
|
||||
explicit = os.environ.get("RETAINDB_PROJECT")
|
||||
if explicit:
|
||||
project = explicit
|
||||
else:
|
||||
hermes_home = str(kwargs.get("hermes_home", ""))
|
||||
profile_name = os.path.basename(hermes_home) if hermes_home else ""
|
||||
project = f"hermes-{profile_name}" if (profile_name and profile_name not in {"", ".hermes"}) else "default"
|
||||
|
||||
self._client = _Client(api_key, base_url, project)
|
||||
self._session_id = session_id
|
||||
self._user_id = kwargs.get("user_id", "default") or "default"
|
||||
self._agent_id = kwargs.get("agent_id", "hermes") or "hermes"
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
hermes_home_path = get_hermes_home()
|
||||
db_path = hermes_home_path / "retaindb_queue.db"
|
||||
self._queue = _WriteQueue(self._client, db_path)
|
||||
|
||||
# Seed agent identity from SOUL.md in background
|
||||
soul_path = hermes_home_path / "SOUL.md"
|
||||
if soul_path.exists():
|
||||
soul_content = soul_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if soul_content:
|
||||
threading.Thread(
|
||||
target=self._seed_soul,
|
||||
args=(soul_content,),
|
||||
name="retaindb-soul-seed",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
def _seed_soul(self, content: str) -> None:
|
||||
try:
|
||||
self._client.seed_agent_identity(self._agent_id, content, source="soul_md")
|
||||
except Exception as exc:
|
||||
logger.debug("RetainDB soul seed failed: %s", exc)
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
project = self._client.project if self._client else "retaindb"
|
||||
return (
|
||||
"# RetainDB Memory\n"
|
||||
f"Active. Project: {project}.\n"
|
||||
"Use retaindb_search to find memories, retaindb_remember to store facts, "
|
||||
"retaindb_profile for a user overview, retaindb_context for current-task context."
|
||||
)
|
||||
|
||||
# ── Background prefetch (fires at turn-end, consumed next turn-start) ──
|
||||
|
||||
def queue_prefetch(self, query: str, *, session_id: str = "") -> None:
|
||||
"""Fire context + dialectic + agent model prefetches in background."""
|
||||
if not self._client:
|
||||
return
|
||||
# Wait for any still-running prefetch threads before spawning new ones.
|
||||
# Prevents thread accumulation if turns fire faster than prefetches complete.
|
||||
for t in self._prefetch_threads:
|
||||
t.join(timeout=2.0)
|
||||
threads = [
|
||||
threading.Thread(target=self._prefetch_context, args=(query,), name="retaindb-ctx", daemon=True),
|
||||
threading.Thread(target=self._prefetch_dialectic, args=(query,), name="retaindb-dialectic", daemon=True),
|
||||
threading.Thread(target=self._prefetch_agent_model, name="retaindb-agent-model", daemon=True),
|
||||
]
|
||||
self._prefetch_threads = threads
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
def _prefetch_context(self, query: str) -> None:
|
||||
try:
|
||||
query_result = self._client.query_context(self._user_id, self._session_id, query)
|
||||
profile = self._client.get_profile(self._user_id)
|
||||
overlay = _build_overlay(profile, query_result)
|
||||
with self._lock:
|
||||
self._context_result = overlay
|
||||
except Exception as exc:
|
||||
logger.debug("RetainDB context prefetch failed: %s", exc)
|
||||
|
||||
def _prefetch_dialectic(self, query: str) -> None:
|
||||
try:
|
||||
result = self._client.ask_user(self._user_id, query, reasoning_level=self._reasoning_level(query))
|
||||
answer = str(result.get("answer") or "")
|
||||
if answer:
|
||||
with self._lock:
|
||||
self._dialectic_result = answer
|
||||
except Exception as exc:
|
||||
logger.debug("RetainDB dialectic prefetch failed: %s", exc)
|
||||
|
||||
def _prefetch_agent_model(self) -> None:
|
||||
try:
|
||||
model = self._client.get_agent_model(self._agent_id)
|
||||
if model.get("memory_count", 0) > 0:
|
||||
with self._lock:
|
||||
self._agent_model = model
|
||||
except Exception as exc:
|
||||
logger.debug("RetainDB agent model prefetch failed: %s", exc)
|
||||
|
||||
@staticmethod
|
||||
def _reasoning_level(query: str) -> str:
|
||||
n = len(query)
|
||||
if n < 120:
|
||||
return "low"
|
||||
if n < 400:
|
||||
return "medium"
|
||||
return "high"
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
"""Consume prefetched results and return them as a context block."""
|
||||
with self._lock:
|
||||
context = self._context_result
|
||||
dialectic = self._dialectic_result
|
||||
agent_model = self._agent_model
|
||||
self._context_result = ""
|
||||
self._dialectic_result = ""
|
||||
self._agent_model = {}
|
||||
|
||||
parts: list[str] = []
|
||||
if context:
|
||||
parts.append(context)
|
||||
if dialectic:
|
||||
parts.append(f"[RetainDB User Synthesis]\n{dialectic}")
|
||||
if agent_model and agent_model.get("memory_count", 0) > 0:
|
||||
model_lines: list[str] = []
|
||||
if agent_model.get("persona"):
|
||||
model_lines.append(f"Persona: {agent_model['persona']}")
|
||||
if agent_model.get("persistent_instructions"):
|
||||
model_lines.append("Instructions:\n" + "\n".join(f"- {i}" for i in agent_model["persistent_instructions"]))
|
||||
if agent_model.get("working_style"):
|
||||
model_lines.append(f"Working style: {agent_model['working_style']}")
|
||||
if model_lines:
|
||||
parts.append("[RetainDB Agent Self-Model]\n" + "\n".join(model_lines))
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
# ── Turn sync ──────────────────────────────────────────────────────────
|
||||
|
||||
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
||||
"""Queue turn for async ingest. Returns immediately."""
|
||||
if not self._queue or not user_content:
|
||||
return
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
self._queue.enqueue(
|
||||
self._user_id,
|
||||
session_id or self._session_id,
|
||||
[
|
||||
{"role": "user", "content": user_content, "timestamp": now},
|
||||
{"role": "assistant", "content": assistant_content, "timestamp": now},
|
||||
],
|
||||
)
|
||||
|
||||
# ── Tools ──────────────────────────────────────────────────────────────
|
||||
|
||||
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
PROFILE_SCHEMA, SEARCH_SCHEMA, CONTEXT_SCHEMA,
|
||||
REMEMBER_SCHEMA, FORGET_SCHEMA,
|
||||
FILE_UPLOAD_SCHEMA, FILE_LIST_SCHEMA, FILE_READ_SCHEMA,
|
||||
FILE_INGEST_SCHEMA, FILE_DELETE_SCHEMA,
|
||||
]
|
||||
|
||||
def handle_tool_call(self, tool_name: str, args: dict, **kwargs) -> str:
|
||||
if not self._client:
|
||||
return tool_error("RetainDB not initialized")
|
||||
try:
|
||||
return json.dumps(self._dispatch(tool_name, args))
|
||||
except Exception as exc:
|
||||
return tool_error(str(exc))
|
||||
|
||||
def _dispatch(self, tool_name: str, args: dict) -> Any:
|
||||
c = self._client
|
||||
|
||||
if tool_name == "retaindb_profile":
|
||||
return c.get_profile(self._user_id)
|
||||
|
||||
if tool_name == "retaindb_search":
|
||||
query = args.get("query", "")
|
||||
if not query:
|
||||
return {"error": "query is required"}
|
||||
return c.search(self._user_id, self._session_id, query, top_k=min(int(args.get("top_k", 8)), 20))
|
||||
|
||||
if tool_name == "retaindb_context":
|
||||
query = args.get("query", "")
|
||||
if not query:
|
||||
return {"error": "query is required"}
|
||||
query_result = c.query_context(self._user_id, self._session_id, query)
|
||||
profile = c.get_profile(self._user_id)
|
||||
overlay = _build_overlay(profile, query_result)
|
||||
return {"context": overlay, "raw": query_result}
|
||||
|
||||
if tool_name == "retaindb_remember":
|
||||
content = args.get("content", "")
|
||||
if not content:
|
||||
return {"error": "content is required"}
|
||||
return c.add_memory(
|
||||
self._user_id, self._session_id, content,
|
||||
memory_type=args.get("memory_type", "factual"),
|
||||
importance=float(args.get("importance", 0.7)),
|
||||
)
|
||||
|
||||
if tool_name == "retaindb_forget":
|
||||
memory_id = args.get("memory_id", "")
|
||||
if not memory_id:
|
||||
return {"error": "memory_id is required"}
|
||||
return c.delete_memory(memory_id)
|
||||
|
||||
# ── File tools ──────────────────────────────────────────────────────
|
||||
|
||||
if tool_name == "retaindb_upload_file":
|
||||
local_path = args.get("local_path", "")
|
||||
if not local_path:
|
||||
return {"error": "local_path is required"}
|
||||
path_obj = Path(local_path)
|
||||
if not path_obj.exists():
|
||||
return {"error": f"File not found: {local_path}"}
|
||||
try:
|
||||
raise_if_read_blocked(str(path_obj))
|
||||
except ValueError as exc:
|
||||
return {"error": str(exc)}
|
||||
data = path_obj.read_bytes()
|
||||
import mimetypes
|
||||
mime = mimetypes.guess_type(path_obj.name)[0] or "application/octet-stream"
|
||||
remote_path = args.get("remote_path") or f"/{path_obj.name}"
|
||||
result = c.upload_file(data, path_obj.name, remote_path, mime, args.get("scope", "PROJECT"), None)
|
||||
if args.get("ingest") and result.get("file", {}).get("id"):
|
||||
ingest = c.ingest_file(result["file"]["id"], user_id=self._user_id, agent_id=self._agent_id)
|
||||
result["ingest"] = ingest
|
||||
return result
|
||||
|
||||
if tool_name == "retaindb_list_files":
|
||||
return c.list_files(prefix=args.get("prefix"), limit=int(args.get("limit", 50)))
|
||||
|
||||
if tool_name == "retaindb_read_file":
|
||||
file_id = args.get("file_id", "")
|
||||
if not file_id:
|
||||
return {"error": "file_id is required"}
|
||||
meta = c.get_file(file_id)
|
||||
file_info = meta.get("file") or {}
|
||||
mime = (file_info.get("mime_type") or "").lower()
|
||||
raw = c.read_file_content(file_id)
|
||||
if not (mime.startswith("text/") or any(file_info.get("name", "").endswith(e) for e in (".txt", ".md", ".json", ".csv", ".yaml", ".yml", ".xml", ".html"))):
|
||||
return {"file_id": file_id, "rdb_uri": file_info.get("rdb_uri"), "name": file_info.get("name"), "content": None, "note": "Binary file — use retaindb_ingest_file to extract text into memory."}
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
return {"file_id": file_id, "rdb_uri": file_info.get("rdb_uri"), "name": file_info.get("name"), "content": text[:32000], "truncated": len(text) > 32000}
|
||||
|
||||
if tool_name == "retaindb_ingest_file":
|
||||
file_id = args.get("file_id", "")
|
||||
if not file_id:
|
||||
return {"error": "file_id is required"}
|
||||
return c.ingest_file(file_id, user_id=self._user_id, agent_id=self._agent_id)
|
||||
|
||||
if tool_name == "retaindb_delete_file":
|
||||
file_id = args.get("file_id", "")
|
||||
if not file_id:
|
||||
return {"error": "file_id is required"}
|
||||
return c.delete_file(file_id)
|
||||
|
||||
return {"error": f"Unknown tool: {tool_name}"}
|
||||
|
||||
# ── Optional hooks ─────────────────────────────────────────────────────
|
||||
|
||||
def on_memory_write(self, action: str, target: str, content: str) -> None:
|
||||
"""Mirror built-in memory writes to RetainDB."""
|
||||
if action != "add" or not content or not self._client:
|
||||
return
|
||||
try:
|
||||
memory_type = "preference" if target == "user" else "factual"
|
||||
self._client.add_memory(self._user_id, self._session_id, content, memory_type=memory_type)
|
||||
except Exception as exc:
|
||||
logger.debug("RetainDB memory mirror failed: %s", exc)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for t in self._prefetch_threads:
|
||||
t.join(timeout=3.0)
|
||||
if self._queue:
|
||||
self._queue.shutdown()
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register RetainDB as a memory provider plugin."""
|
||||
ctx.register_memory_provider(RetainDBMemoryProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: retaindb
|
||||
version: 1.0.0
|
||||
description: "RetainDB — cloud memory API with hybrid search and 7 memory types."
|
||||
pip_dependencies:
|
||||
- requests
|
||||
requires_env:
|
||||
- RETAINDB_API_KEY
|
||||
@@ -0,0 +1,111 @@
|
||||
# Supermemory Memory Provider
|
||||
|
||||
Semantic long-term memory with profile recall, semantic search, explicit memory tools, and full-session conversation ingest (one ingest per session) for richer profiles.
|
||||
|
||||
## Requirements
|
||||
|
||||
- `pip install supermemory`
|
||||
- Supermemory API key from [app.supermemory.ai/integrations?connect=hermes](http://app.supermemory.ai/integrations?connect=hermes)
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
hermes memory setup # select "supermemory"
|
||||
```
|
||||
|
||||
Or manually:
|
||||
|
||||
```bash
|
||||
hermes config set memory.provider supermemory
|
||||
echo 'SUPERMEMORY_API_KEY=***' >> ~/.hermes/.env
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
Config file: `$HERMES_HOME/supermemory.json`
|
||||
|
||||
| Key | Default | Description |
|
||||
|-----|---------|-------------|
|
||||
| `container_tag` | `hermes` | Container tag used for search and writes. Supports `{identity}` template for profile-scoped tags (e.g. `hermes-{identity}` → `hermes-coder`). |
|
||||
| `auto_recall` | `true` | Inject relevant memory context before turns |
|
||||
| `auto_capture` | `true` | Store cleaned user-assistant turns after each response |
|
||||
| `max_recall_results` | `10` | Max recalled items to format into context |
|
||||
| `profile_frequency` | `50` | Include profile facts on first turn and every N turns |
|
||||
| `capture_mode` | `all` | Skip tiny or trivial turns by default |
|
||||
| `search_mode` | `hybrid` | Search mode: `hybrid` (profile + memories), `memories` (memories only), `documents` (documents only) |
|
||||
| `entity_context` | built-in default | Extraction guidance passed to Supermemory |
|
||||
| `api_timeout` | `5.0` | Timeout for SDK and ingest requests |
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `SUPERMEMORY_API_KEY` | API key (required) |
|
||||
| `SUPERMEMORY_CONTAINER_TAG` | Override container tag (takes priority over config file) |
|
||||
|
||||
## Tools
|
||||
|
||||
Kebab-case names are registered for the agent; snake_case aliases remain supported.
|
||||
|
||||
| Tool | Alias | Description |
|
||||
|------|-------|-------------|
|
||||
| `supermemory-save` | `supermemory_store` | Store an explicit memory |
|
||||
| `supermemory-search` | `supermemory_search` | Search memories by semantic similarity |
|
||||
| `supermemory-forget` | `supermemory_forget` | Forget a memory by ID or best-match query |
|
||||
| `supermemory-profile` | `supermemory_profile` | Retrieve persistent profile and recent context |
|
||||
|
||||
## Source attribution
|
||||
|
||||
All Supermemory API calls send `x-sm-source: hermes`, and document writes stamp
|
||||
`metadata.sm_source: hermes`. This is a **functional routing key, not telemetry**:
|
||||
it groups Hermes-written memories into a dedicated "Hermes" Space in the
|
||||
Supermemory app, so you can filter, browse, and bulk-manage them per source agent
|
||||
(alongside Codex, Claude Code, etc.) from the Supermemory UI.
|
||||
|
||||
## Behavior
|
||||
|
||||
When enabled, Hermes can:
|
||||
|
||||
- prefetch relevant memory context before each turn
|
||||
- buffer the full conversation and ingest it as **one session** at session end (or on `/reset`, branch, compression, or shutdown)
|
||||
- ingest the full session to the conversations endpoint for richer profile/graph updates
|
||||
- expose explicit tools for search, store, forget, and profile access
|
||||
|
||||
The session is written once via the conversations endpoint, which drives Supermemory's entity extraction and profile building while keeping a clean, retrievable full transcript.
|
||||
|
||||
## Profile-Scoped Containers
|
||||
|
||||
Use `{identity}` in the `container_tag` to scope memories per Hermes profile:
|
||||
|
||||
```json
|
||||
{
|
||||
"container_tag": "hermes-{identity}"
|
||||
}
|
||||
```
|
||||
|
||||
For a profile named `coder`, this resolves to `hermes-coder`. The default profile resolves to `hermes-default`. Without `{identity}`, all profiles share the same container.
|
||||
|
||||
## Multi-Container Mode
|
||||
|
||||
For advanced setups (e.g. OpenClaw-style multi-workspace), you can enable custom container tags so the agent can read/write across multiple named containers:
|
||||
|
||||
```json
|
||||
{
|
||||
"container_tag": "hermes",
|
||||
"enable_custom_container_tags": true,
|
||||
"custom_containers": ["project-alpha", "project-beta", "shared-knowledge"],
|
||||
"custom_container_instructions": "Use project-alpha for coding tasks, project-beta for research, and shared-knowledge for team-wide facts."
|
||||
}
|
||||
```
|
||||
|
||||
When enabled:
|
||||
- `supermemory-search`, `supermemory-save`, `supermemory-forget`, and `supermemory-profile` accept an optional `container_tag` parameter
|
||||
- The tag must be in the whitelist: primary container + `custom_containers`
|
||||
- Automatic operations (turn sync, prefetch, memory write mirroring, session ingest) always use the **primary** container only
|
||||
- Custom container instructions are injected into the system prompt
|
||||
|
||||
## Support
|
||||
|
||||
- [Supermemory Discord](https://supermemory.link/discord)
|
||||
- [support@supermemory.com](mailto:support@supermemory.com)
|
||||
- [supermemory.ai](https://supermemory.ai)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
name: supermemory
|
||||
version: 1.0.1
|
||||
description: "Supermemory semantic long-term memory with profile recall, semantic search, explicit memory tools, and session ingest."
|
||||
pip_dependencies:
|
||||
- supermemory
|
||||
Reference in New Issue
Block a user