chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env bash
|
||||
# Backfill memory artifacts for sessions imported via `agentmemory import-jsonl`.
|
||||
#
|
||||
# The import path only persists Session + Observation rows (via synthetic,
|
||||
# zero-LLM compression) and the deterministic crystal/lesson derivation.
|
||||
# It does NOT call mem::summarize, so the semantic/procedural/reflect tiers
|
||||
# of the consolidation pipeline have nothing to roll up.
|
||||
#
|
||||
# This script walks every session tagged `jsonl-import` and:
|
||||
# 1. POSTs /agentmemory/summarize per session (LLM call)
|
||||
# 2. POSTs /agentmemory/consolidate-pipeline once at the end
|
||||
#
|
||||
# Graph extraction (/agentmemory/graph/extract) is intentionally skipped —
|
||||
# its API takes a per-observation payload, which is cost-prohibitive for
|
||||
# bulk imports. `reflect` falls back to a no-graph clustering mode.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/backfill-imported-sessions.sh --dry-run
|
||||
# scripts/backfill-imported-sessions.sh --limit 5
|
||||
# scripts/backfill-imported-sessions.sh # process all
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
URL="${AGENTMEMORY_URL:-http://localhost:3111}"
|
||||
DRY_RUN=0
|
||||
LIMIT=0 # 0 = no limit
|
||||
ONLY_TAG="jsonl-import"
|
||||
SKIP_CONSOLIDATE=0
|
||||
SKIP_AGENTS=0 # drop sessions whose project starts with "agent-"
|
||||
MAX_OBS=0 # 0 = no cap; skip sessions with more observations than this
|
||||
DEBUG_ON_ERROR=0 # on failure, dump session metadata + obs to DEBUG_DIR
|
||||
DEBUG_DIR="${AGENTMEMORY_DEBUG_DIR:-./agentmemory-debug}"
|
||||
PROJECT_PATTERN="" # jq test() regex against .project; "" means no filter
|
||||
|
||||
# Cost-estimate knobs (defaults tuned for DeepSeek V4 Flash on DeepInfra:
|
||||
# $0.14 / 1M input, $0.28 / 1M output). Override via env if needed.
|
||||
COST_IN_PER_1M="${AGENTMEMORY_COST_IN_PER_1M:-0.14}"
|
||||
COST_OUT_PER_1M="${AGENTMEMORY_COST_OUT_PER_1M:-0.28}"
|
||||
# Rough token weight per compressed observation, derived from inspecting
|
||||
# real synthetic-compression payloads in the kv store (mostly 100-300 tok,
|
||||
# heavy-tailed). Override if your sessions are unusually verbose.
|
||||
TOKENS_PER_OBS="${AGENTMEMORY_TOKENS_PER_OBS:-200}"
|
||||
# Reserved per-call output budget (XML summary is small).
|
||||
TOKENS_OUT_PER_SESSION="${AGENTMEMORY_TOKENS_OUT_PER_SESSION:-500}"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--limit) LIMIT="${2:?--limit needs a number}"; shift 2 ;;
|
||||
--tag) ONLY_TAG="${2:?--tag needs a value (use empty string for all)}"; shift 2 ;;
|
||||
--skip-consolidate) SKIP_CONSOLIDATE=1; shift ;;
|
||||
--skip-agents) SKIP_AGENTS=1; shift ;;
|
||||
--max-obs) MAX_OBS="${2:?--max-obs needs a number}"; shift 2 ;;
|
||||
--debug-on-error) DEBUG_ON_ERROR=1; shift ;;
|
||||
--project-pattern) PROJECT_PATTERN="${2:?--project-pattern needs a regex}"; shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,28p' "$0"
|
||||
exit 0 ;;
|
||||
*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
for bin in curl jq; do
|
||||
command -v "$bin" >/dev/null || { echo "missing dependency: $bin" >&2; exit 1; }
|
||||
done
|
||||
|
||||
# Curl timeout profiles. Metadata reads (livez, sessions list, observations
|
||||
# pull for debug dumps) should fail fast and retry transient blips. The LLM
|
||||
# work calls (summarize, consolidate) intentionally have no --retry and a
|
||||
# wide --max-time: each call can legitimately take minutes for chunked
|
||||
# summarize on large sessions, and retrying a half-finished LLM job is
|
||||
# expensive both in dollars and in duplicated server-side work.
|
||||
META_CURL_OPTS=(--connect-timeout 10 --max-time 30 --retry 2 --retry-delay 1)
|
||||
WORK_CURL_OPTS=(--connect-timeout 10 --max-time 1800)
|
||||
|
||||
echo "agentmemory backfill — server: $URL"
|
||||
[[ "$DRY_RUN" == 1 ]] && echo "DRY RUN: no POSTs will be made."
|
||||
|
||||
# --- liveness ---
|
||||
if ! curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/livez" >/dev/null; then
|
||||
echo "server not reachable at $URL (try: npx @agentmemory/agentmemory)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- collect session ids ---
|
||||
sessions_json="$(curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/sessions")"
|
||||
filter='.sessions[] | select(.status=="completed")'
|
||||
if [[ -n "$ONLY_TAG" ]]; then
|
||||
filter+=" | select((.tags // []) | index(\"$ONLY_TAG\"))"
|
||||
fi
|
||||
if [[ "$SKIP_AGENTS" == 1 ]]; then
|
||||
filter+=' | select((.project // "") | startswith("agent-") | not)'
|
||||
fi
|
||||
if [[ -n "$PROJECT_PATTERN" ]]; then
|
||||
# jq's test() applies a regex against the project string.
|
||||
filter+=" | select((.project // \"\") | test(\"$PROJECT_PATTERN\"))"
|
||||
fi
|
||||
if [[ "$MAX_OBS" -gt 0 ]]; then
|
||||
filter+=" | select((.observationCount // 0) <= $MAX_OBS)"
|
||||
fi
|
||||
filter+=' | "\(.id)\t\(.observationCount // 0)\t\(.project // "")"'
|
||||
|
||||
rows=()
|
||||
while IFS= read -r line; do
|
||||
rows+=("$line")
|
||||
done < <(echo "$sessions_json" | jq -r "$filter")
|
||||
total="${#rows[@]}"
|
||||
|
||||
if [[ "$total" -eq 0 ]]; then
|
||||
echo "no sessions matched (tag='$ONLY_TAG'); nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$LIMIT" -gt 0 && "$LIMIT" -lt "$total" ]]; then
|
||||
rows=("${rows[@]:0:$LIMIT}")
|
||||
fi
|
||||
|
||||
echo "matched $total session(s); will process ${#rows[@]}."
|
||||
total_obs=0
|
||||
for row in "${rows[@]}"; do
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
total_obs=$(( total_obs + obs ))
|
||||
done
|
||||
est_in=$(( total_obs * TOKENS_PER_OBS + ${#rows[@]} * 500 ))
|
||||
est_out=$(( ${#rows[@]} * TOKENS_OUT_PER_SESSION ))
|
||||
est_cost="$(awk -v i="$est_in" -v o="$est_out" -v ci="$COST_IN_PER_1M" -v co="$COST_OUT_PER_1M" \
|
||||
'BEGIN { printf "%.2f", (i*ci + o*co) / 1000000 }')"
|
||||
|
||||
echo "≈ ${#rows[@]} summarize LLM calls (one per session, covering $total_obs observations)"
|
||||
printf '≈ %d input tok + %d output tok → $%s (rates: in=$%s/1M out=$%s/1M, %s tok/obs)\n' \
|
||||
"$est_in" "$est_out" "$est_cost" "$COST_IN_PER_1M" "$COST_OUT_PER_1M" "$TOKENS_PER_OBS"
|
||||
echo
|
||||
|
||||
if [[ "$DRY_RUN" == 1 ]]; then
|
||||
printf '%-40s %10s %s\n' "session" "obs" "project"
|
||||
for row in "${rows[@]}"; do
|
||||
id="$(cut -f1 <<<"$row")"
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
proj="$(cut -f3 <<<"$row")"
|
||||
printf '%-40s %10s %s\n' "$id" "$obs" "$proj"
|
||||
done
|
||||
echo
|
||||
echo "(dry run) next steps if you re-run without --dry-run:"
|
||||
echo " for each session above: POST $URL/agentmemory/summarize {sessionId}"
|
||||
if [[ "$SKIP_CONSOLIDATE" == 0 ]]; then
|
||||
echo " then: POST $URL/agentmemory/consolidate-pipeline {}"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- summarize loop ---
|
||||
if [[ "$DEBUG_ON_ERROR" == 1 ]]; then
|
||||
mkdir -p "$DEBUG_DIR"
|
||||
echo "debug mode: failed calls will dump to $DEBUG_DIR/"
|
||||
echo
|
||||
fi
|
||||
|
||||
dump_failure() {
|
||||
local id="$1" obs="$2" resp="$3"
|
||||
# Replace anything outside [A-Za-z0-9._-] with `_` before joining with
|
||||
# DEBUG_DIR. Session IDs from the API are UUIDs in practice, but the
|
||||
# server doesn't enforce that — a hostile or buggy id containing `/` or
|
||||
# `..` would otherwise escape the debug directory.
|
||||
local safe_id
|
||||
safe_id="$(printf '%s' "$id" | tr -c 'A-Za-z0-9._-' '_')"
|
||||
local file="$DEBUG_DIR/${safe_id}.json"
|
||||
# Pull the raw observations (what would have gone into the prompt) so the
|
||||
# operator can reconstruct the upstream payload locally. We also compute
|
||||
# narrative size stats so size-related rejections are immediately visible.
|
||||
# Stream observations through stdin (avoids exec-arg overflow on
|
||||
# multi-thousand-obs sessions — macOS argv ceiling is ~256k).
|
||||
# `--get --data-urlencode` percent-encodes the session id so special
|
||||
# characters can't corrupt the query string.
|
||||
curl -fsS "${META_CURL_OPTS[@]}" --get \
|
||||
--data-urlencode "sessionId=$id" \
|
||||
"$URL/agentmemory/observations" \
|
||||
| jq \
|
||||
--arg id "$id" \
|
||||
--argjson obsCount "$obs" \
|
||||
--arg url "$URL/agentmemory/summarize" \
|
||||
--argjson response "$resp" \
|
||||
'. as $root
|
||||
| .observations as $obs
|
||||
| {
|
||||
sessionId: $id,
|
||||
observationCount: $obsCount,
|
||||
request: { url: $url, method: "POST", body: { sessionId: $id } },
|
||||
response: $response,
|
||||
observations: $obs,
|
||||
stats: {
|
||||
totalNarrativeBytes: ($obs | map(.narrative // "" | length) | add // 0),
|
||||
maxNarrativeBytes: ($obs | map(.narrative // "" | length) | max // 0),
|
||||
titleHistogram: ($obs | group_by(.title) | map({title: .[0].title, count: length}) | sort_by(-.count))
|
||||
}
|
||||
}' >"$file"
|
||||
echo " → $file"
|
||||
}
|
||||
|
||||
ok=0; skipped=0; failed=0
|
||||
i=0
|
||||
for row in "${rows[@]}"; do
|
||||
i=$(( i + 1 ))
|
||||
id="$(cut -f1 <<<"$row")"
|
||||
obs="$(cut -f2 <<<"$row")"
|
||||
|
||||
body="$(jq -nc --arg id "$id" '{sessionId:$id}')"
|
||||
resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/summarize" \
|
||||
-H 'content-type: application/json' --data "$body" || echo '{"success":false,"error":"curl_failed"}')"
|
||||
# iii's HTTP layer occasionally returns non-JSON (HTML 5xx, empty body
|
||||
# on timeout, etc.). Validate before parsing so `set -e` doesn't abort
|
||||
# the whole backfill loop on a single bad response.
|
||||
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||||
status="$(jq -r '.success // false' <<<"$resp")"
|
||||
err="$(jq -r '.error // ""' <<<"$resp")"
|
||||
title="$(jq -r '.summary.title // ""' <<<"$resp")"
|
||||
else
|
||||
status="false"
|
||||
err="invalid_json_response"
|
||||
title=""
|
||||
fi
|
||||
|
||||
if [[ "$status" == "true" ]]; then
|
||||
ok=$(( ok + 1 ))
|
||||
printf '[%3d/%3d] OK %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$title"
|
||||
elif [[ "$err" == "no_observations" || "$err" == "no_provider" ]]; then
|
||||
skipped=$(( skipped + 1 ))
|
||||
printf '[%3d/%3d] SKIP %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
|
||||
else
|
||||
failed=$(( failed + 1 ))
|
||||
printf '[%3d/%3d] FAIL %s obs=%-5s %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
|
||||
[[ "$DEBUG_ON_ERROR" == 1 ]] && dump_failure "$id" "$obs" "$resp"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "summarize: ok=$ok skipped=$skipped failed=$failed"
|
||||
|
||||
# --- consolidate ---
|
||||
if [[ "$SKIP_CONSOLIDATE" == 1 ]]; then
|
||||
echo "skipping consolidate-pipeline (--skip-consolidate)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$ok" -eq 0 ]]; then
|
||||
echo "no summaries produced; skipping consolidate-pipeline."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "running consolidate-pipeline …"
|
||||
resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/consolidate-pipeline" \
|
||||
-H 'content-type: application/json' --data '{}' || echo '{"success":false,"error":"curl_failed"}')"
|
||||
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||||
echo "$resp" | jq .
|
||||
else
|
||||
echo "consolidate-pipeline returned non-JSON (likely a timeout or upstream error):"
|
||||
printf '%s\n' "$resp" | head -c 500
|
||||
echo
|
||||
fi
|
||||
Reference in New Issue
Block a user