chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:10:23 +08:00
commit fbab2c6005
567 changed files with 114434 additions and 0 deletions
+75
View File
@@ -0,0 +1,75 @@
# Scripts Documentation
## export_docs.py
Consolidates markdown documentation files for use with ChatGPT or other platforms with file upload limits.
### What It Does
- Scans all subdirectories in the `docs/` folder
- For each subdirectory, combines all `.md` files (excluding `index.md` files)
- Creates one consolidated markdown file per subdirectory
- Saves all exported files to `doc_exports/` in the project root
### Usage
```bash
# Using Makefile (recommended)
make export-docs
# Or run directly with uv
uv run python scripts/export_docs.py
# Or run with standard Python
python scripts/export_docs.py
```
### Output
The script creates `doc_exports/` directory with consolidated files like:
- `getting-started.md` - All getting-started documentation
- `user-guide.md` - All user guide content
- `features.md` - All feature documentation
- `development.md` - All development documentation
- etc.
Each exported file includes:
- A main header with the folder name
- Section headers for each source file
- Source file attribution
- The complete content from each markdown file
- Visual separators between sections
### Example Output Structure
```markdown
# Getting Started
This document consolidates all content from the getting-started documentation folder.
---
## Installation
*Source: installation.md*
[Full content of installation.md]
---
## Quick Start
*Source: quick-start.md*
[Full content of quick-start.md]
---
```
### Notes
- The `doc_exports/` directory is gitignored and safe to regenerate anytime
- Index files (`index.md`) are automatically excluded
- Files are sorted alphabetically for consistent output
- The script handles subdirectories only (ignores files in the root `docs/` folder)
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Check that relative markdown links point to files that exist.
Scans every tracked *.md file in the repo and validates relative link targets
(external URLs, anchors and mailto links are skipped; code blocks and inline
code spans are ignored). Exits 1 if any broken link is found.
Run locally: python3 scripts/check_md_links.py
"""
import os
import re
import subprocess
import sys
LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)")
FENCED_CODE_RE = re.compile(r"```.*?```", re.DOTALL)
INLINE_CODE_RE = re.compile(r"`[^`\n]*`")
SKIP_PREFIXES = ("http://", "https://", "mailto:", "#", "<")
def repo_root() -> str:
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def tracked_markdown_files(root: str) -> list[str]:
out = subprocess.check_output(["git", "ls-files", "*.md"], text=True, cwd=root)
return [line for line in out.splitlines() if line]
def main() -> int:
root = repo_root()
broken: list[str] = []
for rel in tracked_markdown_files(root):
path = os.path.join(root, rel)
try:
with open(path, encoding="utf-8") as fh:
text = fh.read()
except OSError:
continue
text = FENCED_CODE_RE.sub("", text)
text = INLINE_CODE_RE.sub("", text)
for match in LINK_RE.finditer(text):
target = match.group(1)
if target.startswith(SKIP_PREFIXES):
continue
file_part = target.split("#")[0].split("?")[0]
if not file_part:
continue
if file_part.startswith("/"):
resolved = os.path.join(root, file_part.lstrip("/"))
else:
resolved = os.path.normpath(
os.path.join(os.path.dirname(path), file_part)
)
if not os.path.exists(resolved):
broken.append(f"{rel}: {target}")
if broken:
print(f"{len(broken)} broken relative link(s):")
for entry in broken:
print(f" {entry}")
return 1
print("All relative markdown links resolve.")
return 0
if __name__ == "__main__":
sys.exit(main())
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""
Export documentation by consolidating markdown files from each docs folder.
This script:
1. Scans all subdirectories in the docs/ folder
2. For each subdirectory, concatenates all .md files (except index.md)
3. Generates a Table of Contents for easy navigation
4. Saves the consolidated content to doc_exports/{folder_name}.md
"""
import logging
from pathlib import Path
from typing import List
# Configure logging
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
logger = logging.getLogger(__name__)
def get_markdown_files(folder: Path) -> List[Path]:
"""Get all markdown files in a folder, excluding index.md files."""
md_files = [f for f in folder.glob("*.md") if f.name.lower() != "index.md"]
return sorted(md_files) # Sort for consistent ordering
def consolidate_folder(folder: Path, output_dir: Path) -> None:
"""Consolidate all markdown files from a folder into a single file with a TOC."""
md_files = get_markdown_files(folder)
if not md_files:
logger.info(f" Skipping {folder.name} - no markdown files found")
return
output_file = output_dir / f"{folder.name}.md"
with output_file.open("w", encoding="utf-8") as outf:
# Write header
folder_title = folder.name.replace("-", " ").title()
outf.write(f"# {folder_title}\n\n")
outf.write(
f"This document consolidates all content from the {folder.name} documentation folder.\n\n"
)
# Generate a Table of Contents dynamically
outf.write("## Table of Contents\n\n")
for md_file in md_files:
section_title = md_file.stem.replace("-", " ").title()
# Convert title to a markdown-friendly anchor link
# (lowercase, hyphens instead of spaces)
anchor = section_title.lower().replace(" ", "-")
outf.write(f"* [{section_title}](#{anchor})\n")
outf.write("\n---\n\n")
# Process each markdown file
for md_file in md_files:
logger.info(f" Adding {md_file.name}")
# Add section header with filename
outf.write(f"## {md_file.stem.replace('-', ' ').title()}\n\n")
outf.write(f"*Source: {md_file.name}*\n\n")
# Add file content
content = md_file.read_text(encoding="utf-8")
outf.write(content)
outf.write("\n\n---\n\n")
logger.info(f" ✓ Created {output_file.name} ({len(md_files)} files)")
def main():
"""Main function to export documentation."""
# Define paths
docs_dir = Path("docs")
output_dir = Path("doc_exports")
# Validate docs directory exists
if not docs_dir.exists():
logger.error(f"Documentation directory '{docs_dir}' not found")
return
# Create output directory
output_dir.mkdir(exist_ok=True)
logger.info(f"Output directory: {output_dir.absolute()}")
# Get all subdirectories in docs/
subdirs = [
d for d in docs_dir.iterdir() if d.is_dir() and not d.name.startswith(".")
]
if not subdirs:
logger.warning("No subdirectories found in docs/")
return
logger.info(f"Found {len(subdirs)} documentation folders\n")
# Process each subdirectory
for subdir in sorted(subdirs):
logger.info(f"Processing {subdir.name}...")
consolidate_folder(subdir, output_dir)
logger.info("\n✓ Documentation export complete!")
logger.info(f"Exported files are in: {output_dir.absolute()}")
if __name__ == "__main__":
main()
@@ -0,0 +1,43 @@
services:
surrealdb:
image: surrealdb/surrealdb:v2
command: start --log info --user root --pass root rocksdb:/mydata/test.db
user: root
volumes:
- ${DATA_DIR:?set DATA_DIR}/surreal:/mydata
app:
image: ${APP_IMAGE:?set APP_IMAGE}
ports:
- "127.0.0.1:${API_PORT:?set API_PORT}:5055"
- "127.0.0.1:${FE_PORT:?set FE_PORT}:8502"
environment:
# Without an explicit API_URL the frontend's /config points the BROWSER
# at host:5055 — on a dev machine that is the development API, not this
# stack (silent data crossover between environments!)
- API_URL=${RC_API_URL:-}
- OPEN_NOTEBOOK_ENCRYPTION_KEY=${RC_ENCRYPTION_KEY:-release-test-key}
- SURREAL_URL=ws://surrealdb:8000/rpc
- SURREAL_USER=root
- SURREAL_PASSWORD=root
- SURREAL_NAMESPACE=${RC_SURREAL_NS:-open_notebook}
- SURREAL_DATABASE=${RC_SURREAL_DB:-open_notebook}
volumes:
- ${DATA_DIR}/notebook:/app/data
# Reach host services (e.g. Ollama on localhost:11434) via
# host.docker.internal on Linux too; Docker Desktop resolves it natively.
# Credentials pointing at local services must use
# http://host.docker.internal:<port> when the app runs in a container.
extra_hosts:
- "host.docker.internal:host-gateway"
depends_on:
- surrealdb
proxy:
image: nginx:alpine
ports:
- "127.0.0.1:${PROXY_PORT:?set PROXY_PORT}:80"
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
- app
+18
View File
@@ -0,0 +1,18 @@
# Reverse proxy in front of the frontend with default buffering ON — the SSE
# streaming fix must work anyway because the API sends X-Accel-Buffering: no,
# which nginx honors. Host is passed through verbatim to exercise the
# frontend's Host validation with a real proxy in the path.
server {
listen 80;
server_name _;
location / {
proxy_pass http://app:8502;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 300s;
}
}
+75
View File
@@ -0,0 +1,75 @@
#!/bin/bash
# Browsable release-candidate stack on the local machine: runs a pushed (or
# local) image, optionally with a COPY of your dev data, for manual release
# verification without touching the dev environment.
#
# Usage:
# rc-stack.sh up <tag> [dump.surql] # start (imports the dump if given)
# rc-stack.sh down <tag> # stop and remove everything
#
# To produce a data copy from a running dev SurrealDB (originals untouched):
# docker exec <surreal-container> /surreal export \
# --conn http://localhost:8000 --user root --pass root \
# --ns open_notebook --db <your-db> /dev/stdout > /tmp/dev-dump.surql
#
# Ports: UI http://localhost:18502 · via nginx http://localhost:18080 · API 15055
set -euo pipefail
DIR="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$DIR/../.." && pwd)"
TAG="${2:?usage: rc-stack.sh <up|down> <tag> [dump.surql]}"
DUMP="${3:-}"
RC_DATA=/tmp/onrel-rc-data
# Reuse the dev encryption key so copied credentials decrypt; the DB name must
# match the one the dump came from (defaults to the dev .env's database).
KEY=$(grep '^OPEN_NOTEBOOK_ENCRYPTION_KEY' "$REPO/.env" 2>/dev/null | cut -d= -f2- | tr -d '"' || true)
DB=$(grep '^SURREAL_DATABASE' "$REPO/.env" 2>/dev/null | cut -d= -f2- | tr -d '"' || true)
compose() {
APP_IMAGE="lfnovo/open_notebook:$TAG" DATA_DIR="$RC_DATA" \
API_PORT=15055 FE_PORT=18502 PROXY_PORT=18080 \
RC_API_URL="http://localhost:15055" \
RC_ENCRYPTION_KEY="${KEY:-release-test-key}" RC_SURREAL_DB="${DB:-open_notebook}" \
docker compose -p onrelrc -f "$DIR/docker-compose.release-test.yml" "$@"
}
case "$1" in
down)
compose down -v
rm -rf "$RC_DATA"
echo "RC stack removed."
;;
up)
compose down -v >/dev/null 2>&1 || true
rm -rf "$RC_DATA"; mkdir -p "$RC_DATA/surreal" "$RC_DATA/notebook"
if [ -n "$DUMP" ]; then
# SurrealDB import quirks, learned in production:
# - OVERWRITE goes AFTER the type keyword (DEFINE FIELD OVERWRITE ...),
# needed because RELATION tables auto-define in/out fields
# - the exporter can leak its own log line into the dump (starts with ESC)
sed -E $'/^\x1b/d; s/^DEFINE (TABLE|FIELD|INDEX|ANALYZER|FUNCTION|EVENT|PARAM|ACCESS) /DEFINE \\1 OVERWRITE /' "$DUMP" > "$RC_DATA/surreal/dump.surql"
fi
compose up -d surrealdb
sleep 4
if [ -n "$DUMP" ]; then
docker exec onrelrc-surrealdb-1 /surreal import --conn http://localhost:8000 \
--user root --pass root --ns open_notebook --db "${DB:-open_notebook}" \
/mydata/dump.surql
fi
compose up -d
echo "Waiting for API..."
for i in $(seq 1 30); do
curl -sf -m 5 -o /dev/null http://localhost:15055/docs && break
sleep 5
done
NB=$(curl -s http://localhost:15055/api/notebooks | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null || echo "?")
echo "RC stack up — image lfnovo/open_notebook:$TAG"
echo " UI: http://localhost:18502"
echo " via nginx: http://localhost:18080"
echo " API: http://localhost:15055"
echo " notebooks: $NB"
echo
echo "NOTE: credentials pointing at host services (e.g. Ollama) need"
echo " base_url http://host.docker.internal:<port> inside the container."
;;
esac
+150
View File
@@ -0,0 +1,150 @@
#!/bin/bash
# Release image gate: fresh-install and upgrade tests against built Docker images.
#
# Usage: release-image-test.sh <fresh|upgrade|all> <new-image> [old-image]
# e.g. release-image-test.sh all lfnovo/open_notebook:1.12.0 lfnovo/open_notebook:1.11.0
#
# IMPORTANT: `make docker-build-local` tags the build with the CURRENT pyproject
# version. If that version matches a published release, `docker pull` the
# genuine old tag first or the upgrade test will compare the new build against
# itself.
set -u
DIR="$(cd "$(dirname "$0")" && pwd)"
COMPOSE="$DIR/docker-compose.release-test.yml"
NEW_IMAGE="${2:?usage: release-image-test.sh <fresh|upgrade|all> <new-image> [old-image]}"
OLD_IMAGE="${3:-}"
PASS=0; FAIL=0
ok() { echo "$1"; PASS=$((PASS+1)); }
bad() { echo "$1"; FAIL=$((FAIL+1)); }
check() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (expected=$2, got=$3)"; fi; }
# Each phase gets its own ports so a leaked container can never answer for the
# wrong stack (learned the hard way: a leftover fresh-install app once served
# the whole upgrade test).
set_ports() { API_PORT=$1; FE_PORT=$2; PROXY_PORT=$3; API="http://localhost:$1"; FE="http://localhost:$2"; PROXY="http://localhost:$3"; }
compose_env() { env APP_IMAGE="$1" DATA_DIR="$2" API_PORT="$API_PORT" FE_PORT="$FE_PORT" PROXY_PORT="$PROXY_PORT" docker compose -p "$3" -f "$COMPOSE" "${@:4}"; }
compose_up() { # <image> <datadir> <project>
local OUT
OUT=$(compose_env "$1" "$2" "$3" up -d --quiet-pull 2>&1)
if echo "$OUT" | grep -qi "error"; then
bad "compose up ($3): $(echo "$OUT" | grep -i error | head -1)"
return 1
fi
local RUNNING
RUNNING=$(docker inspect "$3-app-1" --format '{{.State.Running}}' 2>/dev/null)
check "container $3-app-1 running" "true" "$RUNNING"
}
compose_down() { # <project> <datadir-to-remove (optional)>
compose_env unused /tmp/unused "$1" down -v >/dev/null 2>&1
if docker ps --format '{{.Names}}' | grep -q "^$1-"; then
bad "teardown of $1 left containers running"
fi
[ -n "${2:-}" ] && rm -rf "$2"
}
wait_api() { # up to 150s (startup has ~50s of DB-wait backoff + migrations)
for i in $(seq 1 30); do
if curl -sf -m 5 -o /dev/null "$API/docs"; then return 0; fi
sleep 5
done
return 1
}
seed_and_verify() {
NB=$(curl -s -X POST "$API/api/notebooks" -H "Content-Type: application/json" \
-d '{"name":"release-probe","description":"release test seed"}' | python3 -c "import json,sys; print(json.load(sys.stdin)['id'])" 2>/dev/null)
[ -z "$NB" ] && { bad "create notebook"; return 1; }
ok "create notebook ($NB)"
SRC=$(curl -s -X POST "$API/api/sources" -F "type=text" -F "notebooks=[\"$NB\"]" \
-F "content=Release test content. The Turing test evaluates machine intelligence." \
-F "title=release-probe-source" -F "async_processing=true" -F "embed=false" \
| python3 -c "import json,sys; print(json.load(sys.stdin)['id'])" 2>/dev/null)
[ -z "$SRC" ] && { bad "create source"; return 1; }
ok "create source ($SRC)"
ST=""
for i in $(seq 1 24); do
ST=$(curl -s "$API/api/sources/$SRC/status" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status',''))" 2>/dev/null)
[ "$ST" = "completed" ] && break
sleep 5
done
check "in-image worker processed source" "completed" "$ST"
FT=$(curl -s "$API/api/sources/$SRC" | python3 -c "import json,sys; d=json.load(sys.stdin); print('yes' if d.get('full_text') else 'no')" 2>/dev/null)
check "full_text present" "yes" "$FT"
}
fresh_test() {
echo "═══ FRESH INSTALL — $NEW_IMAGE"
set_ports 15055 18502 18080
local DD; DD=$(mktemp -d /tmp/onrel-fresh-XXXX)
compose_up "$NEW_IMAGE" "$DD" onrelfresh || { compose_down onrelfresh "$DD"; return 1; }
if wait_api; then ok "API up (migrations ran on boot)"; else
bad "API did not come up in 150s"; docker logs onrelfresh-app-1 2>&1 | tail -20
compose_down onrelfresh "$DD"; return 1
fi
N=$(curl -s "$API/api/notebooks" | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null)
check "GET /api/notebooks on a virgin database" "0" "$N"
seed_and_verify
for f in type title created updated insights_count embedded; do
S=$(curl -s -o /dev/null -w '%{http_code}' "$API/api/sources?sort_by=$f&limit=2")
check "sort_by=$f" "200" "$S"
done
S=$(curl -s -o /dev/null -w '%{http_code}' -m 15 "$FE")
if [ "$S" = "200" ] || [ "$S" = "307" ]; then ok "frontend responds ($S)"; else bad "frontend ($S)"; fi
C=$(curl -s -m 10 "$PROXY/config")
echo "$C" | grep -q apiUrl && ok "/config via nginx: $C" || bad "/config via nginx: $C"
# Malformed Host: nginx may reject with 400 (correct) or the app falls back
# (200). What must never happen is a 5xx.
C=$(curl -s -m 10 -H 'Host: bad<host>!' "$PROXY/config" -o /dev/null -w '%{http_code}')
if [ "$C" = "200" ] || [ "$C" = "400" ]; then ok "/config via nginx with malformed Host -> $C (no 5xx)"; else bad "/config malformed Host -> $C"; fi
compose_down onrelfresh "$DD"
echo
}
upgrade_test() {
if [ -z "$OLD_IMAGE" ]; then echo "═══ UPGRADE skipped: no old image given"; return 0; fi
echo "═══ UPGRADE — $OLD_IMAGE -> $NEW_IMAGE"
set_ports 25055 28502 28080
local DD; DD=$(mktemp -d /tmp/onrel-upg-XXXX)
echo "-- phase 1: boot $OLD_IMAGE and seed"
compose_up "$OLD_IMAGE" "$DD" onrelupg || { compose_down onrelupg "$DD"; return 1; }
IMG=$(docker inspect onrelupg-app-1 --format '{{.Config.Image}}' 2>/dev/null)
check "phase 1 runs the old image" "$OLD_IMAGE" "$IMG"
if wait_api; then ok "old image up"; else
bad "old image did not come up"; compose_down onrelupg "$DD"; return 1
fi
seed_and_verify
echo "-- phase 2: swap to the new image on the SAME volume"
compose_env unused /tmp/unused onrelupg stop app >/dev/null 2>&1
compose_env unused /tmp/unused onrelupg rm -f app >/dev/null 2>&1
compose_up "$NEW_IMAGE" "$DD" onrelupg || { compose_down onrelupg "$DD"; return 1; }
IMG=$(docker inspect onrelupg-app-1 --format '{{.Config.Image}}' 2>/dev/null)
check "phase 2 runs the new image" "$NEW_IMAGE" "$IMG"
if wait_api; then ok "new image up on old data"; else
bad "new image did not come up on old data"; docker logs onrelupg-app-1 2>&1 | tail -20
compose_down onrelupg "$DD"; return 1
fi
FOUND=$(curl -s "$API/api/notebooks" | python3 -c "
import json,sys
nbs=json.load(sys.stdin)
print('yes' if any(n.get('name')=='release-probe' for n in nbs) else 'no')" 2>/dev/null)
check "notebook seeded on old image survived the upgrade" "yes" "$FOUND"
S=$(curl -s -o /dev/null -w '%{http_code}' "$API/api/sources?sort_by=title&limit=2")
check "sort_by=title after upgrade" "200" "$S"
echo " phase 2 migration log:"
docker logs onrelupg-app-1 2>&1 | grep -i "migrat" | tail -5 | sed 's/^/ /'
compose_down onrelupg "$DD"
echo
}
case "${1:-all}" in
fresh) fresh_test ;;
upgrade) upgrade_test ;;
all) fresh_test; upgrade_test ;;
esac
echo "═══ RESULT: $PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
# Wait for the API to be healthy before starting the frontend
# This prevents the "Unable to Connect to API Server" error during startup
# POSIX-compliant so it runs with /bin/sh (dash) in slim images
API_URL="${INTERNAL_API_URL:-http://localhost:5055}"
MAX_RETRIES=60
RETRY_INTERVAL=5
i=0
echo "Waiting for API to be ready at ${API_URL}/health..."
while [ $i -lt $MAX_RETRIES ]; do
if curl -s -f "${API_URL}/health" > /dev/null 2>&1; then
echo "API is ready! Starting frontend..."
exit 0
fi
i=$((i + 1))
echo "Attempt $i/$MAX_RETRIES: API not ready yet, waiting ${RETRY_INTERVAL}s..."
sleep $RETRY_INTERVAL
done
echo "ERROR: API did not become ready within $((MAX_RETRIES * RETRY_INTERVAL)) seconds"
echo "Starting frontend anyway - users may see connection errors initially"
exit 0 # Exit 0 so frontend still starts (better than nothing)