chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env bash
|
||||
# Backfill memory artifacts for sessions imported via `agentmemory import-jsonl`.
|
||||
#
|
||||
# The import path only persists Session + Observation rows (via synthetic,
|
||||
# zero-LLM compression) and the deterministic crystal/lesson derivation.
|
||||
# It does NOT call mem::summarize, so the semantic/procedural/reflect tiers
|
||||
# of the consolidation pipeline have nothing to roll up.
|
||||
#
|
||||
# This script walks every session tagged `jsonl-import` and:
|
||||
# 1. POSTs /agentmemory/summarize per session (LLM call)
|
||||
# 2. POSTs /agentmemory/consolidate-pipeline once at the end
|
||||
#
|
||||
# Graph extraction (/agentmemory/graph/extract) is intentionally skipped —
|
||||
# its API takes a per-observation payload, which is cost-prohibitive for
|
||||
# bulk imports. `reflect` falls back to a no-graph clustering mode.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/backfill-imported-sessions.sh --dry-run
|
||||
# scripts/backfill-imported-sessions.sh --limit 5
|
||||
# scripts/backfill-imported-sessions.sh # process all
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
URL="${AGENTMEMORY_URL:-http://localhost:3111}"
|
||||
DRY_RUN=0
|
||||
LIMIT=0 # 0 = no limit
|
||||
ONLY_TAG="jsonl-import"
|
||||
SKIP_CONSOLIDATE=0
|
||||
SKIP_AGENTS=0 # drop sessions whose project starts with "agent-"
|
||||
MAX_OBS=0 # 0 = no cap; skip sessions with more observations than this
|
||||
DEBUG_ON_ERROR=0 # on failure, dump session metadata + obs to DEBUG_DIR
|
||||
DEBUG_DIR="${AGENTMEMORY_DEBUG_DIR:-./agentmemory-debug}"
|
||||
PROJECT_PATTERN="" # jq test() regex against .project; "" means no filter
|
||||
|
||||
# Cost-estimate knobs (defaults tuned for DeepSeek V4 Flash on DeepInfra:
|
||||
# $0.14 / 1M input, $0.28 / 1M output). Override via env if needed.
|
||||
COST_IN_PER_1M="${AGENTMEMORY_COST_IN_PER_1M:-0.14}"
|
||||
COST_OUT_PER_1M="${AGENTMEMORY_COST_OUT_PER_1M:-0.28}"
|
||||
# Rough token weight per compressed observation, derived from inspecting
|
||||
# real synthetic-compression payloads in the kv store (mostly 100-300 tok,
|
||||
# heavy-tailed). Override if your sessions are unusually verbose.
|
||||
TOKENS_PER_OBS="${AGENTMEMORY_TOKENS_PER_OBS:-200}"
|
||||
# Reserved per-call output budget (XML summary is small).
|
||||
TOKENS_OUT_PER_SESSION="${AGENTMEMORY_TOKENS_OUT_PER_SESSION:-500}"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--limit) LIMIT="${2:?--limit needs a number}"; shift 2 ;;
|
||||
--tag) ONLY_TAG="${2:?--tag needs a value (use empty string for all)}"; shift 2 ;;
|
||||
--skip-consolidate) SKIP_CONSOLIDATE=1; shift ;;
|
||||
--skip-agents) SKIP_AGENTS=1; shift ;;
|
||||
--max-obs) MAX_OBS="${2:?--max-obs needs a number}"; shift 2 ;;
|
||||
--debug-on-error) DEBUG_ON_ERROR=1; shift ;;
|
||||
--project-pattern) PROJECT_PATTERN="${2:?--project-pattern needs a regex}"; shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,28p' "$0"
|
||||
exit 0 ;;
|
||||
*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
for bin in curl jq; do
|
||||
command -v "$bin" >/dev/null || { echo "missing dependency: $bin" >&2; exit 1; }
|
||||
done
|
||||
|
||||
# Curl timeout profiles. Metadata reads (livez, sessions list, observations
|
||||
# pull for debug dumps) should fail fast and retry transient blips. The LLM
|
||||
# work calls (summarize, consolidate) intentionally have no --retry and a
|
||||
# wide --max-time: each call can legitimately take minutes for chunked
|
||||
# summarize on large sessions, and retrying a half-finished LLM job is
|
||||
# expensive both in dollars and in duplicated server-side work.
|
||||
META_CURL_OPTS=(--connect-timeout 10 --max-time 30 --retry 2 --retry-delay 1)
|
||||
WORK_CURL_OPTS=(--connect-timeout 10 --max-time 1800)
|
||||
|
||||
echo "agentmemory backfill — server: $URL"
|
||||
[[ "$DRY_RUN" == 1 ]] && echo "DRY RUN: no POSTs will be made."
|
||||
|
||||
# --- liveness ---
|
||||
if ! curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/livez" >/dev/null; then
|
||||
echo "server not reachable at $URL (try: npx @agentmemory/agentmemory)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- collect session ids ---
|
||||
sessions_json="$(curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/sessions")"
|
||||
filter='.sessions[] | select(.status=="completed")'
|
||||
if [[ -n "$ONLY_TAG" ]]; then
|
||||
filter+=" | select((.tags // []) | index(\"$ONLY_TAG\"))"
|
||||
fi
|
||||
if [[ "$SKIP_AGENTS" == 1 ]]; then
|
||||
filter+=' | select((.project // "") | startswith("agent-") | not)'
|
||||
fi
|
||||
if [[ -n "$PROJECT_PATTERN" ]]; then
|
||||
# jq's test() applies a regex against the project string.
|
||||
filter+=" | select((.project // \"\") | test(\"$PROJECT_PATTERN\"))"
|
||||
fi
|
||||
if [[ "$MAX_OBS" -gt 0 ]]; then
|
||||
filter+=" | select((.observationCount // 0) <= $MAX_OBS)"
|
||||
fi
|
||||
filter+=' | "\(.id)\t\(.observationCount // 0)\t\(.project // "")"'
|
||||
|
||||
rows=()
|
||||
while IFS= read -r line; do
|
||||
rows+=("$line")
|
||||
done < <(echo "$sessions_json" | jq -r "$filter")
|
||||
total="${#rows[@]}"
|
||||
|
||||
if [[ "$total" -eq 0 ]]; then
|
||||
echo "no sessions matched (tag='$ONLY_TAG'); nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$LIMIT" -gt 0 && "$LIMIT" -lt "$total" ]]; then
|
||||
rows=("${rows[@]:0:$LIMIT}")
|
||||
fi
|
||||
|
||||
echo "matched $total session(s); will process ${#rows[@]}."
|
||||
total_obs=0
|
||||
for row in "${rows[@]}"; do
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
total_obs=$(( total_obs + obs ))
|
||||
done
|
||||
est_in=$(( total_obs * TOKENS_PER_OBS + ${#rows[@]} * 500 ))
|
||||
est_out=$(( ${#rows[@]} * TOKENS_OUT_PER_SESSION ))
|
||||
est_cost="$(awk -v i="$est_in" -v o="$est_out" -v ci="$COST_IN_PER_1M" -v co="$COST_OUT_PER_1M" \
|
||||
'BEGIN { printf "%.2f", (i*ci + o*co) / 1000000 }')"
|
||||
|
||||
echo "≈ ${#rows[@]} summarize LLM calls (one per session, covering $total_obs observations)"
|
||||
printf '≈ %d input tok + %d output tok → $%s (rates: in=$%s/1M out=$%s/1M, %s tok/obs)\n' \
|
||||
"$est_in" "$est_out" "$est_cost" "$COST_IN_PER_1M" "$COST_OUT_PER_1M" "$TOKENS_PER_OBS"
|
||||
echo
|
||||
|
||||
if [[ "$DRY_RUN" == 1 ]]; then
|
||||
printf '%-40s %10s %s\n' "session" "obs" "project"
|
||||
for row in "${rows[@]}"; do
|
||||
id="$(cut -f1 <<<"$row")"
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
proj="$(cut -f3 <<<"$row")"
|
||||
printf '%-40s %10s %s\n' "$id" "$obs" "$proj"
|
||||
done
|
||||
echo
|
||||
echo "(dry run) next steps if you re-run without --dry-run:"
|
||||
echo " for each session above: POST $URL/agentmemory/summarize {sessionId}"
|
||||
if [[ "$SKIP_CONSOLIDATE" == 0 ]]; then
|
||||
echo " then: POST $URL/agentmemory/consolidate-pipeline {}"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- summarize loop ---
|
||||
if [[ "$DEBUG_ON_ERROR" == 1 ]]; then
|
||||
mkdir -p "$DEBUG_DIR"
|
||||
echo "debug mode: failed calls will dump to $DEBUG_DIR/"
|
||||
echo
|
||||
fi
|
||||
|
||||
dump_failure() {
|
||||
local id="$1" obs="$2" resp="$3"
|
||||
# Replace anything outside [A-Za-z0-9._-] with `_` before joining with
|
||||
# DEBUG_DIR. Session IDs from the API are UUIDs in practice, but the
|
||||
# server doesn't enforce that — a hostile or buggy id containing `/` or
|
||||
# `..` would otherwise escape the debug directory.
|
||||
local safe_id
|
||||
safe_id="$(printf '%s' "$id" | tr -c 'A-Za-z0-9._-' '_')"
|
||||
local file="$DEBUG_DIR/${safe_id}.json"
|
||||
# Pull the raw observations (what would have gone into the prompt) so the
|
||||
# operator can reconstruct the upstream payload locally. We also compute
|
||||
# narrative size stats so size-related rejections are immediately visible.
|
||||
# Stream observations through stdin (avoids exec-arg overflow on
|
||||
# multi-thousand-obs sessions — macOS argv ceiling is ~256k).
|
||||
# `--get --data-urlencode` percent-encodes the session id so special
|
||||
# characters can't corrupt the query string.
|
||||
curl -fsS "${META_CURL_OPTS[@]}" --get \
|
||||
--data-urlencode "sessionId=$id" \
|
||||
"$URL/agentmemory/observations" \
|
||||
| jq \
|
||||
--arg id "$id" \
|
||||
--argjson obsCount "$obs" \
|
||||
--arg url "$URL/agentmemory/summarize" \
|
||||
--argjson response "$resp" \
|
||||
'. as $root
|
||||
| .observations as $obs
|
||||
| {
|
||||
sessionId: $id,
|
||||
observationCount: $obsCount,
|
||||
request: { url: $url, method: "POST", body: { sessionId: $id } },
|
||||
response: $response,
|
||||
observations: $obs,
|
||||
stats: {
|
||||
totalNarrativeBytes: ($obs | map(.narrative // "" | length) | add // 0),
|
||||
maxNarrativeBytes: ($obs | map(.narrative // "" | length) | max // 0),
|
||||
titleHistogram: ($obs | group_by(.title) | map({title: .[0].title, count: length}) | sort_by(-.count))
|
||||
}
|
||||
}' >"$file"
|
||||
echo " → $file"
|
||||
}
|
||||
|
||||
ok=0; skipped=0; failed=0
|
||||
i=0
|
||||
for row in "${rows[@]}"; do
|
||||
i=$(( i + 1 ))
|
||||
id="$(cut -f1 <<<"$row")"
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
|
||||
body="$(jq -nc --arg id "$id" '{sessionId:$id}')"
|
||||
resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/summarize" \
|
||||
-H 'content-type: application/json' --data "$body" || echo '{"success":false,"error":"curl_failed"}')"
|
||||
# iii's HTTP layer occasionally returns non-JSON (HTML 5xx, empty body
|
||||
# on timeout, etc.). Validate before parsing so `set -e` doesn't abort
|
||||
# the whole backfill loop on a single bad response.
|
||||
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||||
status="$(jq -r '.success // false' <<<"$resp")"
|
||||
err="$(jq -r '.error // ""' <<<"$resp")"
|
||||
title="$(jq -r '.summary.title // ""' <<<"$resp")"
|
||||
else
|
||||
status="false"
|
||||
err="invalid_json_response"
|
||||
title=""
|
||||
fi
|
||||
|
||||
if [[ "$status" == "true" ]]; then
|
||||
ok=$(( ok + 1 ))
|
||||
printf '[%3d/%3d] OK %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$title"
|
||||
elif [[ "$err" == "no_observations" || "$err" == "no_provider" ]]; then
|
||||
skipped=$(( skipped + 1 ))
|
||||
printf '[%3d/%3d] SKIP %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
|
||||
else
|
||||
failed=$(( failed + 1 ))
|
||||
printf '[%3d/%3d] FAIL %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
|
||||
[[ "$DEBUG_ON_ERROR" == 1 ]] && dump_failure "$id" "$obs" "$resp"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "summarize: ok=$ok skipped=$skipped failed=$failed"
|
||||
|
||||
# --- consolidate ---
|
||||
if [[ "$SKIP_CONSOLIDATE" == 1 ]]; then
|
||||
echo "skipping consolidate-pipeline (--skip-consolidate)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$ok" -eq 0 ]]; then
|
||||
echo "no summaries produced; skipping consolidate-pipeline."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "running consolidate-pipeline …"
|
||||
resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/consolidate-pipeline" \
|
||||
-H 'content-type: application/json' --data '{}' || echo '{"success":false,"error":"curl_failed"}')"
|
||||
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||||
echo "$resp" | jq .
|
||||
else
|
||||
echo "consolidate-pipeline returned non-JSON (likely a timeout or upstream error):"
|
||||
printf '%s\n' "$resp" | head -c 500
|
||||
echo
|
||||
fi
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env node
|
||||
//
|
||||
// Sync-check: every env var read by `src/` MUST be documented in
|
||||
// `.env.example`. Runs in CI as a soft guard rail — keeps `.env.example`
|
||||
// from drifting behind real config-surface additions.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/check-env-example.mjs
|
||||
//
|
||||
// Returns 0 when in sync, 1 with a diff when out of sync.
|
||||
|
||||
import { readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const ROOT = new URL("..", import.meta.url).pathname;
|
||||
const SRC = join(ROOT, "src");
|
||||
const ENV_FILE = join(ROOT, ".env.example");
|
||||
|
||||
// Env vars read by the runtime but NOT user-facing config — these are
|
||||
// either process-injected (HOME, PATH, USERPROFILE), set by the build /
|
||||
// wrapper (NODE_*, npm_*), or set by tests (VITEST, *_TEST_*). Skipping
|
||||
// them keeps `.env.example` a documented config surface rather than an
|
||||
// inventory of every getenv anywhere in the codebase.
|
||||
const RUNTIME_ONLY = new Set([
|
||||
"HOME",
|
||||
"PATH",
|
||||
"USERPROFILE",
|
||||
"NODE_ENV",
|
||||
"AGENTMEMORY_SDK_CHILD",
|
||||
]);
|
||||
|
||||
// Walk src/ for .ts / .mts / .mjs / .js files (excluding `.d.ts` declarations
|
||||
// and dotfile dirs / node_modules). test/ lives outside src/ so it never enters.
|
||||
function walk(dir) {
|
||||
const out = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const s = statSync(full);
|
||||
if (s.isDirectory()) {
|
||||
if (entry === "node_modules" || entry.startsWith(".")) continue;
|
||||
out.push(...walk(full));
|
||||
} else if (/\.(ts|mts|mjs|js)$/.test(entry) && !entry.endsWith(".d.ts")) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Multiple patterns:
|
||||
// process.env["KEY"] — direct access
|
||||
// env["KEY"] — local alias inside detectProvider, etc.
|
||||
// getEnvVar("KEY") — helper from src/config.ts
|
||||
// env: ProcessEnv → env.KEY — caught as `env["KEY"]` only; if you add
|
||||
// a dotted-access path, extend the regex.
|
||||
const PATTERNS = [
|
||||
// Direct map index: process.env["KEY"], env["KEY"], getMergedEnv()["KEY"].
|
||||
// The trailing `]\s*` form covers `…)["KEY"]` and `…env["KEY"]`.
|
||||
/\[\s*"([A-Z][A-Z0-9_]+)"\s*\]/g,
|
||||
/getEnvVar\(\s*"([A-Z][A-Z0-9_]+)"\s*\)/g,
|
||||
];
|
||||
const used = new Set();
|
||||
for (const file of walk(SRC)) {
|
||||
const text = readFileSync(file, "utf8");
|
||||
for (const pat of PATTERNS) {
|
||||
pat.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = pat.exec(text)) !== null) {
|
||||
const name = m[1];
|
||||
if (!RUNTIME_ONLY.has(name)) used.add(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const envText = readFileSync(ENV_FILE, "utf8");
|
||||
const documented = new Set();
|
||||
for (const line of envText.split("\n")) {
|
||||
const m = line.match(/^#?\s*([A-Z][A-Z0-9_]+)=/);
|
||||
if (m) documented.add(m[1]);
|
||||
}
|
||||
|
||||
const missing = [...used].filter((k) => !documented.has(k)).sort();
|
||||
const orphan = [...documented].filter((k) => !used.has(k)).sort();
|
||||
|
||||
if (missing.length === 0 && orphan.length === 0) {
|
||||
console.log(`env-example: in sync (${used.size} keys documented)`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
console.error(
|
||||
`env-example: MISSING from .env.example — add documentation for these keys:`,
|
||||
);
|
||||
for (const k of missing) console.error(` - ${k}`);
|
||||
}
|
||||
if (orphan.length > 0) {
|
||||
console.error(
|
||||
`env-example: ORPHAN in .env.example — no longer read by src/, remove or move to runtime-only allowlist:`,
|
||||
);
|
||||
for (const k of orphan) console.error(` - ${k}`);
|
||||
}
|
||||
process.exit(1);
|
||||
@@ -0,0 +1,80 @@
|
||||
import { readFileSync, readdirSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, "..", "..");
|
||||
const SKILLS = join(ROOT, "plugin", "skills");
|
||||
|
||||
const errors: string[] = [];
|
||||
const MAX_LINES = 100;
|
||||
const DUP_MARKER = "/plugin list";
|
||||
|
||||
function parseFrontmatter(text: string): Record<string, string> | null {
|
||||
if (!text.startsWith("---")) return null;
|
||||
const end = text.indexOf("\n---", 3);
|
||||
if (end === -1) return null;
|
||||
const body = text.slice(3, end);
|
||||
const out: Record<string, string> = {};
|
||||
for (const line of body.split("\n")) {
|
||||
const m = /^([a-zA-Z_-]+):\s*(.*)$/.exec(line.trim());
|
||||
if (m) out[m[1]] = m[2];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const dirs = readdirSync(SKILLS, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory() && !e.name.startsWith("_"))
|
||||
.map((e) => e.name);
|
||||
|
||||
for (const name of dirs) {
|
||||
const skillFile = join(SKILLS, name, "SKILL.md");
|
||||
if (!existsSync(skillFile)) {
|
||||
errors.push(`${name}: missing SKILL.md`);
|
||||
continue;
|
||||
}
|
||||
const text = readFileSync(skillFile, "utf8");
|
||||
const rel = `plugin/skills/${name}/SKILL.md`;
|
||||
|
||||
const fm = parseFrontmatter(text);
|
||||
if (!fm) {
|
||||
errors.push(`${rel}: missing or malformed frontmatter`);
|
||||
} else {
|
||||
if (!fm.name) errors.push(`${rel}: frontmatter missing 'name'`);
|
||||
if (fm.name && fm.name !== name) errors.push(`${rel}: frontmatter name '${fm.name}' != dir '${name}'`);
|
||||
if (!fm.description) errors.push(`${rel}: frontmatter missing 'description'`);
|
||||
else {
|
||||
if (!/use when/i.test(fm.description)) errors.push(`${rel}: description must contain a "Use when ..." trigger sentence`);
|
||||
if (fm.description.length > 1024) errors.push(`${rel}: description exceeds 1024 chars`);
|
||||
}
|
||||
}
|
||||
|
||||
const lines = text.split("\n").length;
|
||||
if (lines > MAX_LINES) errors.push(`${rel}: ${lines} lines (max ${MAX_LINES}); move detail into REFERENCE.md or EXAMPLES.md`);
|
||||
|
||||
const body = fm ? text.slice(text.indexOf("\n---", 3) + 4) : text;
|
||||
if (body.includes(DUP_MARKER)) {
|
||||
errors.push(`${rel}: inlines the shared troubleshooting block; reference ../_shared/TROUBLESHOOTING.md instead`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!existsSync(join(SKILLS, "_shared", "TROUBLESHOOTING.md"))) {
|
||||
errors.push(`_shared/TROUBLESHOOTING.md is missing`);
|
||||
}
|
||||
|
||||
const pluginJson = join(ROOT, "plugin", "plugin.json");
|
||||
if (existsSync(pluginJson)) {
|
||||
const desc = (JSON.parse(readFileSync(pluginJson, "utf8")).description as string) ?? "";
|
||||
const m = /(\d+)\s+skills/.exec(desc);
|
||||
if (!m) errors.push(`plugin/plugin.json: description should state the skill count as "N skills"`);
|
||||
else if (Number(m[1]) !== dirs.length) {
|
||||
errors.push(`plugin/plugin.json: description says ${m[1]} skills but ${dirs.length} skill dirs exist`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.error("Skill lint failed:");
|
||||
for (const e of errors) console.error(` - ${e}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`Skill lint passed: ${dirs.length} skills checked.`);
|
||||
@@ -0,0 +1,170 @@
|
||||
import { readFileSync, writeFileSync, readdirSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { getAllTools, ESSENTIAL_TOOLS } from "../../src/mcp/tools-registry.js";
|
||||
import { ADAPTERS } from "../../src/cli/connect/index.js";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, "..", "..");
|
||||
const SKILLS = join(ROOT, "plugin", "skills");
|
||||
const SRC = join(ROOT, "src");
|
||||
|
||||
const check = process.argv.includes("--check");
|
||||
|
||||
function clean(s: string): string {
|
||||
return s.replace(/\s*[—–]\s*/g, ", ");
|
||||
}
|
||||
|
||||
function block(key: string, body: string): { open: string; close: string; full: string } {
|
||||
const open = `<!-- AUTOGEN:${key} START - generated by scripts/skills/generate.ts, do not edit by hand -->`;
|
||||
const close = `<!-- AUTOGEN:${key} END -->`;
|
||||
return { open, close, full: `${open}\n${body.trim()}\n${close}` };
|
||||
}
|
||||
|
||||
function applyBlock(file: string, key: string, body: string): void {
|
||||
const { open, close, full } = block(key, body);
|
||||
const existing = existsSync(file) ? readFileSync(file, "utf8") : "";
|
||||
let next: string;
|
||||
if (existing.includes(open) && existing.includes(close)) {
|
||||
const start = existing.indexOf(open);
|
||||
const end = existing.indexOf(close) + close.length;
|
||||
next = existing.slice(0, start) + full + existing.slice(end);
|
||||
} else if (existing.trim()) {
|
||||
next = `${existing.trimEnd()}\n\n${full}\n`;
|
||||
} else {
|
||||
next = `${full}\n`;
|
||||
}
|
||||
if (check) {
|
||||
if (existing !== next) {
|
||||
console.error(`DRIFT: ${file.replace(ROOT + "/", "")} (AUTOGEN:${key} out of date — run \`npm run skills:gen\`)`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (existing !== next) {
|
||||
writeFileSync(file, next);
|
||||
console.log(`wrote AUTOGEN:${key} -> ${file.replace(ROOT + "/", "")}`);
|
||||
}
|
||||
}
|
||||
|
||||
function walk(dir: string, out: string[] = []): string[] {
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = join(dir, e.name);
|
||||
if (e.isDirectory()) {
|
||||
if (e.name === "node_modules" || e.name === "dist") continue;
|
||||
walk(full, out);
|
||||
} else if (e.name.endsWith(".ts")) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function mdEscape(s: string): string {
|
||||
return clean(s.replace(/\|/g, "\\|").replace(/\n/g, " ")).trim();
|
||||
}
|
||||
|
||||
function tools(): string {
|
||||
const all = getAllTools();
|
||||
const lines = [
|
||||
`agentmemory exposes ${all.length} MCP tools. ${ESSENTIAL_TOOLS.size} are in the lean core set (\`--tools core\` or \`AGENTMEMORY_TOOLS=core\`); the rest load with \`--tools all\` (default).`,
|
||||
"",
|
||||
"| Tool | Core | Parameters | Purpose |",
|
||||
"| --- | --- | --- | --- |",
|
||||
];
|
||||
for (const t of all.sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const required = new Set(t.inputSchema.required ?? []);
|
||||
const params = Object.entries(t.inputSchema.properties ?? {})
|
||||
.map(([n, s]) => `\`${n}\`${required.has(n) ? "*" : ""}: ${s.type}`)
|
||||
.join(", ") || "none";
|
||||
const core = ESSENTIAL_TOOLS.has(t.name) ? "yes" : "";
|
||||
lines.push(`| \`${t.name}\` | ${core} | ${mdEscape(params)} | ${mdEscape(t.description)} |`);
|
||||
}
|
||||
lines.push("", "`*` marks required parameters.");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function rest(): string {
|
||||
const text = readFileSync(join(SRC, "triggers", "api.ts"), "utf8");
|
||||
const found: { path: string; method: string }[] = [];
|
||||
const re = /api_path:\s*"([^"]+)"/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
const path = m[1];
|
||||
const win = text.slice(Math.max(0, m.index - 140), m.index + 140);
|
||||
const mm = /http_method:\s*"([A-Z]+)"/.exec(win);
|
||||
found.push({ path, method: mm ? mm[1] : "POST" });
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const rows = found
|
||||
.filter((e) => (seen.has(e.path) ? false : (seen.add(e.path), true)))
|
||||
.sort((a, b) => a.path.localeCompare(b.path));
|
||||
const lines = [
|
||||
`The REST API is the primary surface. All paths are under \`http://localhost:3111\` (override with \`--port\`). When \`AGENTMEMORY_SECRET\` is set, send \`Authorization: Bearer $AGENTMEMORY_SECRET\`; localhost is otherwise open.`,
|
||||
"",
|
||||
`${rows.length} registered endpoints:`,
|
||||
"",
|
||||
"| Method | Path |",
|
||||
"| --- | --- |",
|
||||
...rows.map((e) => `| ${e.method} | \`${e.path}\` |`),
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function env(): string {
|
||||
const files = walk(SRC);
|
||||
const vars = new Set<string>();
|
||||
for (const f of files) {
|
||||
const text = readFileSync(f, "utf8");
|
||||
const re = /AGENTMEMORY_[A-Z0-9_]+/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
if (!m[0].endsWith("__")) vars.add(m[0]);
|
||||
}
|
||||
}
|
||||
const sorted = [...vars].sort();
|
||||
const lines = [
|
||||
`Configuration is read from the environment and from \`~/.agentmemory/.env\` (no \`export\` prefix). ${sorted.length} recognized variables:`,
|
||||
"",
|
||||
...sorted.map((v) => `- \`${v}\``),
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function agents(): string {
|
||||
const lines = [
|
||||
`\`agentmemory connect <agent>\` wires the memory server into a host agent. ${ADAPTERS.length} adapters:`,
|
||||
"",
|
||||
"| Agent | Name | Protocol |",
|
||||
"| --- | --- | --- |",
|
||||
];
|
||||
for (const a of [...ADAPTERS].sort((x, y) => x.name.localeCompare(y.name))) {
|
||||
const note = (a.protocolNote ?? "").replace(/^[→\s]+/, "");
|
||||
lines.push(`| ${mdEscape(a.displayName)} | \`${a.name}\` | ${mdEscape(note) || "MCP"} |`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function hooks(): string {
|
||||
const file = join(ROOT, "plugin", "hooks", "hooks.json");
|
||||
const json = JSON.parse(readFileSync(file, "utf8")) as { hooks?: Record<string, unknown> };
|
||||
const events = Object.keys(json.hooks ?? {}).sort();
|
||||
const lines = [
|
||||
`The Claude Code plugin registers hooks on ${events.length} lifecycle events to capture observations automatically:`,
|
||||
"",
|
||||
...events.map((e) => `- \`${e}\``),
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
applyBlock(join(SKILLS, "agentmemory-mcp-tools", "REFERENCE.md"), "tools", tools());
|
||||
applyBlock(join(SKILLS, "agentmemory-rest-api", "REFERENCE.md"), "rest", rest());
|
||||
applyBlock(join(SKILLS, "agentmemory-config", "REFERENCE.md"), "env", env());
|
||||
applyBlock(join(SKILLS, "agentmemory-agents", "REFERENCE.md"), "agents", agents());
|
||||
applyBlock(join(SKILLS, "agentmemory-hooks", "REFERENCE.md"), "hooks", hooks());
|
||||
|
||||
if (check && process.exitCode) {
|
||||
console.error("\nSkill reference docs are stale. Run: npm run skills:gen");
|
||||
} else if (!check) {
|
||||
console.log("skills reference generation complete");
|
||||
}
|
||||
Reference in New Issue
Block a user