wehub/lfnovo--open-notebook
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:10:23 +08:00
commit fbab2c6005
567 changed files with 114434 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
open_notebook/database/async_migrate.py
+295
View File
@@ -0,0 +1,295 @@
"""
Async migration system for SurrealDB using the official Python client.
Based on patterns from sblpy migration system.
"""
from typing import List
from loguru import logger
from .repository import db_connection, repo_query
class AsyncMigration:
"""
Handles individual migration operations with async support.
"""
def __init__(self, sql: str) -> None:
"""Initialize migration with SQL content."""
self.sql = sql
@classmethod
def from_file(cls, file_path: str) -> "AsyncMigration":
"""Create migration from SQL file."""
with open(file_path, "r", encoding="utf-8") as file:
raw_content = file.read()
# Clean up SQL content
lines = []
for line in raw_content.split("\n"):
line = line.strip()
if line and not line.startswith("--"):
lines.append(line)
sql = " ".join(lines)
return cls(sql)
async def run(self, bump: bool = True) -> None:
"""Run the migration."""
try:
async with db_connection() as connection:
await connection.query(self.sql)
if bump:
await bump_version()
else:
await lower_version()
except Exception as e:
logger.error(f"Migration failed: {str(e)}")
raise
class AsyncMigrationRunner:
"""
Handles running multiple migrations in sequence.
"""
def __init__(
self,
up_migrations: List[AsyncMigration],
down_migrations: List[AsyncMigration],
) -> None:
"""Initialize runner with migration lists."""
self.up_migrations = up_migrations
self.down_migrations = down_migrations
async def run_all(self) -> None:
"""Run all pending up migrations."""
current_version = await get_latest_version()
for i in range(current_version, len(self.up_migrations)):
logger.info(f"Running migration {i + 1}")
await self.up_migrations[i].run(bump=True)
async def run_one_up(self) -> None:
"""Run one up migration."""
current_version = await get_latest_version()
if current_version < len(self.up_migrations):
logger.info(f"Running migration {current_version + 1}")
await self.up_migrations[current_version].run(bump=True)
async def run_one_down(self) -> None:
"""Run one down migration."""
current_version = await get_latest_version()
if current_version > 0:
logger.info(f"Rolling back migration {current_version}")
await self.down_migrations[current_version - 1].run(bump=False)
class AsyncMigrationManager:
"""
Main migration manager with async support.
"""
def __init__(self):
"""Initialize migration manager."""
self.up_migrations = [
AsyncMigration.from_file("open_notebook/database/migrations/1.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/2.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/3.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/4.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/5.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/6.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/7.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/8.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/9.surrealql"),
AsyncMigration.from_file("open_notebook/database/migrations/10.surrealql"),
AsyncMigration.from_file(
"open_notebook/database/migrations/11.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/12.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/13.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/14.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/15.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/16.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/17.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/18.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/19.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/20.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/21.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/22.surrealql"
),
]
self.down_migrations = [
AsyncMigration.from_file(
"open_notebook/database/migrations/1_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/2_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/3_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/4_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/5_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/6_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/7_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/8_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/9_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/10_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/11_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/12_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/13_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/14_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/15_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/16_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/17_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/18_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/19_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/20_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/21_down.surrealql"
),
AsyncMigration.from_file(
"open_notebook/database/migrations/22_down.surrealql"
),
]
self.runner = AsyncMigrationRunner(
up_migrations=self.up_migrations,
down_migrations=self.down_migrations,
)
async def get_current_version(self) -> int:
"""Get current database version."""
return await get_latest_version()
async def ping(self) -> None:
"""Check whether SurrealDB is reachable for migration startup."""
async with db_connection() as connection:
await connection.query("RETURN true;")
# Also exercise the migration version path. get_current_version() already
# treats a missing migrations table as version 0 for fresh databases.
await self.get_current_version()
async def needs_migration(self) -> bool:
"""Check if migration is needed."""
current_version = await self.get_current_version()
return current_version < len(self.up_migrations)
async def run_migration_up(self):
"""Run all pending migrations."""
current_version = await self.get_current_version()
logger.info(f"Current version before migration: {current_version}")
if await self.needs_migration():
try:
await self.runner.run_all()
new_version = await self.get_current_version()
logger.info(f"Migration successful. New version: {new_version}")
except Exception as e:
logger.error(f"Migration failed: {str(e)}")
raise
else:
logger.info("Database is already at the latest version")
# Database version management functions
async def get_latest_version() -> int:
"""Get the latest version from the migrations table."""
try:
versions = await get_all_versions()
if not versions:
return 0
return max(version["version"] for version in versions)
except Exception:
# If migrations table doesn't exist, we're at version 0
return 0
async def get_all_versions() -> List[dict]:
"""Get all versions from the migrations table."""
try:
result = await repo_query("SELECT * FROM _sbl_migrations ORDER BY version;")
return result
except Exception:
# If table doesn't exist, return empty list
return []
async def bump_version() -> None:
"""Bump the version by adding a new entry to migrations table."""
current_version = await get_latest_version()
new_version = current_version + 1
await repo_query(
"CREATE type::thing('_sbl_migrations', $version) SET version = $version, applied_at = time::now();",
{"version": new_version},
)
async def lower_version() -> None:
"""Lower the version by removing the latest entry from migrations table."""
current_version = await get_latest_version()
if current_version > 0:
await repo_query(
"DELETE type::thing('_sbl_migrations', $version);",
{"version": current_version},
)
open_notebook/database/migrate.py
+26
View File
@@ -0,0 +1,26 @@
import asyncio
from .async_migrate import AsyncMigrationManager
class MigrationManager:
"""
Synchronous wrapper around AsyncMigrationManager for backward compatibility.
"""
def __init__(self):
"""Initialize with async migration manager."""
self._async_manager = AsyncMigrationManager()
def get_current_version(self) -> int:
"""Get current database version (sync wrapper)."""
return asyncio.run(self._async_manager.get_current_version())
@property
def needs_migration(self) -> bool:
"""Check if migration is needed (sync wrapper)."""
return asyncio.run(self._async_manager.needs_migration())
def run_migration_up(self):
"""Run migrations (sync wrapper)."""
asyncio.run(self._async_manager.run_migration_up())
open_notebook/database/migrations/1.surrealql
+178
View File
@@ -0,0 +1,178 @@
DEFINE TABLE IF NOT EXISTS source SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS
asset
ON TABLE source
FLEXIBLE TYPE option<object>;
DEFINE FIELD IF NOT EXISTS title ON TABLE source TYPE option<string>;
DEFINE FIELD IF NOT EXISTS topics ON TABLE source TYPE option<array<string>>;
DEFINE FIELD IF NOT EXISTS full_text ON TABLE source TYPE option<string>;
DEFINE FIELD IF NOT EXISTS created ON source DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON source DEFAULT time::now() VALUE time::now();
DEFINE TABLE IF NOT EXISTS source_embedding SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS source ON TABLE source_embedding TYPE record<source>;
DEFINE FIELD IF NOT EXISTS order ON TABLE source_embedding TYPE int;
DEFINE FIELD IF NOT EXISTS content ON TABLE source_embedding TYPE string;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_embedding TYPE array<float>;
DEFINE TABLE IF NOT EXISTS source_insight SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS source ON TABLE source_insight TYPE record<source>;
DEFINE FIELD IF NOT EXISTS insight_type ON TABLE source_insight TYPE string;
DEFINE FIELD IF NOT EXISTS content ON TABLE source_insight TYPE string;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_insight TYPE array<float>;
DEFINE EVENT IF NOT EXISTS source_delete ON TABLE source WHEN ($after == NONE) THEN {
delete source_embedding where source == $before.id;
delete source_insight where source == $before.id;
};
DEFINE TABLE IF NOT EXISTS note SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS title ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS summary ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS content ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE note TYPE array<float>;
DEFINE FIELD IF NOT EXISTS created ON note DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON note DEFAULT time::now() VALUE time::now();
DEFINE TABLE IF NOT EXISTS notebook SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS name ON TABLE notebook TYPE option<string>;
DEFINE FIELD IF NOT EXISTS description ON TABLE notebook TYPE option<string>;
DEFINE FIELD IF NOT EXISTS archived ON TABLE notebook TYPE option<bool> DEFAULT False;
DEFINE FIELD IF NOT EXISTS created ON notebook DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON notebook DEFAULT time::now() VALUE time::now();
DEFINE TABLE IF NOT EXISTS reference
TYPE RELATION
FROM source TO notebook;
DEFINE TABLE IF NOT EXISTS artifact
TYPE RELATION
FROM note TO notebook;
DEFINE TABLE IF NOT EXISTS podcast_config SCHEMALESS;
-- entender o analyzer
DEFINE ANALYZER IF NOT EXISTS my_analyzer TOKENIZERS blank,class,camel,punct FILTERS snowball(english), lowercase;
DEFINE INDEX IF NOT EXISTS idx_source_title ON TABLE source COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_full_text ON TABLE source COLUMNS full_text SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_embed_chunk ON TABLE source_embedding COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_insight ON TABLE source_insight COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_note ON TABLE note COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_note_title ON TABLE note COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_title_search =
IF $sources {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM source
WHERE title @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_embedding_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source_embedding
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_full_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source
WHERE full_text @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source_insight
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $note_title_search =
IF $show_notes {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM note
WHERE title @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM note
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
let $source_asset_results = array::union($source_title_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_asset_results );
let $note_results = array::union($note_title_search, $note_content_search );
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results
group by item_id ORDER BY relevance DESC LIMIT $match_count);
};
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_embedding_search =
IF $sources {(
SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding LIMIT $match_count)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight LIMIT $match_count)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM note LIMIT $match_count)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_insight_search);
let $note_results = $note_content_search;
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results
group by item_id ORDER BY similarity DESC LIMIT $match_count);
};
IF array::len(select * from open_notebook:default_models) == 0 THEN
CREATE open_notebook:default_models SET
default_chat_model= ""
END;
open_notebook/database/migrations/10.surrealql
+13
View File
@@ -0,0 +1,13 @@
-- Migration 10: Add indexes for source_insight and source_embedding source field
-- These indexes significantly improve performance of source listing queries
-- that count insights and check embedding existence per source
DEFINE INDEX IF NOT EXISTS idx_source_insight_source ON source_insight FIELDS source CONCURRENTLY;
DEFINE INDEX IF NOT EXISTS idx_source_embedding_source ON source_embedding FIELDS source CONCURRENTLY;
DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE option<array<float>>;
DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE option<array<float>>;
-- delete orphan records
DELETE from source_embedding WHERE source.id=NONE;
DELETE from source_insight WHERE source.id=NONE;
open_notebook/database/migrations/10_down.surrealql
+4
View File
@@ -0,0 +1,4 @@
-- Rollback Migration 10: Remove source field indexes
REMOVE INDEX IF EXISTS idx_source_insight_source ON TABLE source_insight;
REMOVE INDEX IF EXISTS idx_source_embedding_source ON TABLE source_embedding;
open_notebook/database/migrations/11.surrealql
+10
View File
@@ -0,0 +1,10 @@
-- Migration 11: Create provider configuration singleton record
-- This record stores multiple API key configurations per provider
-- The data is managed by the ProviderConfig RecordModel class
-- Create the provider configs singleton record for multi-config support
-- This record stores multiple API key configurations per provider
-- The data is managed by the ProviderConfig RecordModel class
UPSERT open_notebook:provider_configs CONTENT {
credentials: {}
};
open_notebook/database/migrations/11_down.surrealql
+4
View File
@@ -0,0 +1,4 @@
-- Rollback Migration 11: Remove provider configuration records
-- Remove provider configs singleton (if exists)
DELETE open_notebook:provider_configs;
open_notebook/database/migrations/12.surrealql
+29
View File
@@ -0,0 +1,29 @@
-- Migration 12: Create credential table and add credential link to model table
-- Individual credential records replace the ProviderConfig singleton
-- Each credential stores API key and provider-specific configuration
DEFINE TABLE credential SCHEMAFULL;
DEFINE FIELD name ON credential TYPE string;
DEFINE FIELD provider ON credential TYPE string;
DEFINE FIELD modalities ON credential TYPE array DEFAULT [];
DEFINE FIELD modalities.* ON credential TYPE string;
DEFINE FIELD api_key ON credential TYPE option<string>;
DEFINE FIELD base_url ON credential TYPE option<string>;
DEFINE FIELD endpoint ON credential TYPE option<string>;
DEFINE FIELD api_version ON credential TYPE option<string>;
DEFINE FIELD endpoint_llm ON credential TYPE option<string>;
DEFINE FIELD endpoint_embedding ON credential TYPE option<string>;
DEFINE FIELD endpoint_stt ON credential TYPE option<string>;
DEFINE FIELD endpoint_tts ON credential TYPE option<string>;
DEFINE FIELD project ON credential TYPE option<string>;
DEFINE FIELD location ON credential TYPE option<string>;
DEFINE FIELD credentials_path ON credential TYPE option<string>;
DEFINE FIELD created ON credential TYPE option<datetime> DEFAULT time::now();
DEFINE FIELD updated ON credential TYPE option<datetime> DEFAULT time::now();
-- Index for fast provider lookups
DEFINE INDEX idx_credential_provider ON credential FIELDS provider;
-- Add optional credential link to model table
DEFINE FIELD credential ON model TYPE option<record<credential>>;
open_notebook/database/migrations/12_down.surrealql
+5
View File
@@ -0,0 +1,5 @@
-- Rollback Migration 12: Remove credential table and credential field from model
REMOVE FIELD credential ON TABLE model;
REMOVE INDEX idx_credential_provider ON credential;
REMOVE TABLE credential;
open_notebook/database/migrations/13.surrealql
+3
View File
@@ -0,0 +1,3 @@
DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE option<array<float>>;
DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE option<array<float>>;
open_notebook/database/migrations/13_down.surrealql
+2
View File
@@ -0,0 +1,2 @@
DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE array<float>;
DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE array<float>;
open_notebook/database/migrations/14.surrealql
+27
View File
@@ -0,0 +1,27 @@
-- Migration 14: Podcast profiles model registry integration
-- Adds record<model> references to replace loose provider/model strings
-- Adds language field to episode_profile
-- Adds per-speaker TTS override support
-- EPISODE PROFILE
-- Legacy fields: make optional (app ignores, preserved for data migration)
DEFINE FIELD OVERWRITE outline_provider ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE outline_model ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE transcript_provider ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE transcript_model ON TABLE episode_profile TYPE option<string>;
-- New fields: reference to Model registry
DEFINE FIELD IF NOT EXISTS outline_llm ON TABLE episode_profile TYPE option<record<model>>;
DEFINE FIELD IF NOT EXISTS transcript_llm ON TABLE episode_profile TYPE option<record<model>>;
DEFINE FIELD IF NOT EXISTS language ON TABLE episode_profile TYPE option<string>;
-- SPEAKER PROFILE
-- Legacy fields: make optional
DEFINE FIELD OVERWRITE tts_provider ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD OVERWRITE tts_model ON TABLE speaker_profile TYPE option<string>;
-- New field: reference to Model registry (profile-level)
DEFINE FIELD IF NOT EXISTS voice_model ON TABLE speaker_profile TYPE option<record<model>>;
-- Per-speaker TTS override
DEFINE FIELD IF NOT EXISTS speakers.*.voice_model ON TABLE speaker_profile TYPE option<record<model>>;
open_notebook/database/migrations/14_down.surrealql
+20
View File
@@ -0,0 +1,20 @@
-- Migration 14 rollback: Remove model registry fields from podcast profiles
-- Remove new fields from episode_profile
REMOVE FIELD IF EXISTS outline_llm ON TABLE episode_profile;
REMOVE FIELD IF EXISTS transcript_llm ON TABLE episode_profile;
REMOVE FIELD IF EXISTS language ON TABLE episode_profile;
-- Restore episode_profile legacy fields as required strings
DEFINE FIELD OVERWRITE outline_provider ON TABLE episode_profile TYPE string;
DEFINE FIELD OVERWRITE outline_model ON TABLE episode_profile TYPE string;
DEFINE FIELD OVERWRITE transcript_provider ON TABLE episode_profile TYPE string;
DEFINE FIELD OVERWRITE transcript_model ON TABLE episode_profile TYPE string;
-- Remove new fields from speaker_profile
REMOVE FIELD IF EXISTS voice_model ON TABLE speaker_profile;
REMOVE FIELD IF EXISTS speakers.*.voice_model ON TABLE speaker_profile;
-- Restore speaker_profile legacy fields as required strings
DEFINE FIELD OVERWRITE tts_provider ON TABLE speaker_profile TYPE string;
DEFINE FIELD OVERWRITE tts_model ON TABLE speaker_profile TYPE string;
open_notebook/database/migrations/15.surrealql
+6
View File
@@ -0,0 +1,6 @@
-- Migration 15: Add a flexible config object to the credential table.
-- Stores provider-specific tuning options (e.g. Ollama num_ctx) without a
-- schema change per option. FLEXIBLE allows arbitrary keys inside the object
-- on this SCHEMAFULL table; validation stays in the Pydantic Credential model.
DEFINE FIELD config ON credential FLEXIBLE TYPE option<object>;
open_notebook/database/migrations/15_down.surrealql
+3
View File
@@ -0,0 +1,3 @@
-- Migration 15 rollback: remove the flexible config field from credential
REMOVE FIELD IF EXISTS config ON TABLE credential;
open_notebook/database/migrations/16.surrealql
+7
View File
@@ -0,0 +1,7 @@
-- Migration 16: Add optional max_tokens override to episode_profile.
-- Lets a profile cap (or raise) the output tokens used for outline and
-- transcript generation, which is passed through to podcast_creator.
-- The episode_profile table is SCHEMAFULL, so the field must be declared
-- here for the value to persist.
DEFINE FIELD IF NOT EXISTS max_tokens ON TABLE episode_profile TYPE option<int>;
open_notebook/database/migrations/16_down.surrealql
+3
View File
@@ -0,0 +1,3 @@
-- Migration 16 rollback: Remove max_tokens field from episode_profile.
REMOVE FIELD IF EXISTS max_tokens ON TABLE episode_profile;
open_notebook/database/migrations/17.surrealql
+3
View File
@@ -0,0 +1,3 @@
-- Migration 17: Per-transformation model selection
DEFINE FIELD IF NOT EXISTS model_id ON TABLE transformation TYPE option<record<model>>;
open_notebook/database/migrations/17_down.surrealql
+3
View File
@@ -0,0 +1,3 @@
-- Migration 17 rollback: Remove per-transformation model selection
REMOVE FIELD IF EXISTS model_id ON TABLE transformation;
open_notebook/database/migrations/18.surrealql
+10
View File
@@ -0,0 +1,10 @@
-- Migration 18: Recently viewed timestamps
-- Adds optional last-viewed timestamps for notebooks and sources.
DEFINE FIELD IF NOT EXISTS last_viewed_at ON TABLE notebook TYPE option<datetime>;
DEFINE FIELD IF NOT EXISTS last_viewed_at ON TABLE source TYPE option<datetime>;
-- Indexes backing the recently-viewed query (ORDER BY last_viewed_at DESC LIMIT)
-- so it does not degrade into a full table scan as data grows.
DEFINE INDEX IF NOT EXISTS idx_notebook_last_viewed ON TABLE notebook FIELDS last_viewed_at CONCURRENTLY;
DEFINE INDEX IF NOT EXISTS idx_source_last_viewed ON TABLE source FIELDS last_viewed_at CONCURRENTLY;
open_notebook/database/migrations/18_down.surrealql
+6
View File
@@ -0,0 +1,6 @@
-- Migration 18 rollback: Remove recently viewed timestamps
REMOVE INDEX IF EXISTS idx_notebook_last_viewed ON TABLE notebook;
REMOVE INDEX IF EXISTS idx_source_last_viewed ON TABLE source;
REMOVE FIELD IF EXISTS last_viewed_at ON TABLE notebook;
REMOVE FIELD IF EXISTS last_viewed_at ON TABLE source;
open_notebook/database/migrations/19.surrealql
+8
View File
@@ -0,0 +1,8 @@
-- Migration 19: Timestamp source_insight records
-- source_insight is SCHEMAFULL but never defined created/updated, so new
-- insights were stored without timestamps (and clients saw "None").
-- Mirrors the created/updated definitions used by source, note and notebook.
-- Existing rows are left untouched (no backfill); the API returns null for them.
DEFINE FIELD IF NOT EXISTS created ON source_insight DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON source_insight DEFAULT time::now() VALUE time::now();
open_notebook/database/migrations/19_down.surrealql
+4
View File
@@ -0,0 +1,4 @@
-- Migration 19 rollback: Remove source_insight timestamp fields
REMOVE FIELD IF EXISTS created ON TABLE source_insight;
REMOVE FIELD IF EXISTS updated ON TABLE source_insight;
open_notebook/database/migrations/1_down.surrealql
+24
View File
@@ -0,0 +1,24 @@
REMOVE TABLE IF EXISTS source;
REMOVE TABLE IF EXISTS source_embedding;
REMOVE TABLE IF EXISTS source_insight;
REMOVE TABLE IF EXISTS note;
REMOVE TABLE IF EXISTS notebook;
REMOVE TABLE IF EXISTS reference;
REMOVE TABLE IF EXISTS artifact;
REMOVE TABLE IF EXISTS podcast_config;
REMOVE EVENT IF EXISTS source_delete ON TABLE source;
REMOVE ANALYZER IF EXISTS my_analyzer;
REMOVE INDEX IF EXISTS idx_source_title ON TABLE source;
REMOVE INDEX IF EXISTS idx_source_full_text ON TABLE source;
REMOVE INDEX IF EXISTS idx_source_embed_chunk ON TABLE source_embedding;
REMOVE INDEX IF EXISTS idx_source_insight ON TABLE source_insight;
REMOVE INDEX IF EXISTS idx_note ON TABLE note;
REMOVE INDEX IF EXISTS idx_note_title ON TABLE note;
REMOVE FUNCTION IF EXISTS fn::text_search;
REMOVE FUNCTION IF EXISTS fn::vector_search;
DELETE open_notebook:default_models;
open_notebook/database/migrations/2.surrealql
+1
View File
@@ -0,0 +1 @@
DEFINE FIELD IF NOT EXISTS note_type ON TABLE note TYPE option<string>;
open_notebook/database/migrations/20.surrealql
+24
View File
@@ -0,0 +1,24 @@
-- Migration 20: episode_profile.speaker_config references speaker_profile by record ID (#630)
-- Previously speaker_config stored the speaker profile NAME (a plain string),
-- so renaming a speaker profile silently broke every episode profile that
-- referenced it. This converts the field to a record link.
--
-- Steps:
-- 1. Widen the field type so both the legacy strings and the new record
-- links are valid during the data conversion (the table is SCHEMAFULL).
-- 2. Convert existing rows: look up the speaker_profile with a matching
-- name and store its record ID. Orphan references (a name that no longer
-- matches any speaker_profile) become NONE - the app treats a missing
-- speaker as "needs setup" and the UI asks the user to pick one.
-- 3. Tighten the type to option<record<speaker_profile>> so only
-- speaker_profile record links (or NONE) can be stored from now on.
-- Note: SurrealDB record links do NOT prevent dangling references -
-- deleting a speaker profile still leaves the link in place, so the
-- app must keep resolving speaker_config defensively (it treats an
-- unresolvable reference the same as NONE: "needs setup").
DEFINE FIELD OVERWRITE speaker_config ON TABLE episode_profile TYPE option<string | record<speaker_profile>>;
UPDATE episode_profile SET speaker_config = (SELECT VALUE id FROM ONLY speaker_profile WHERE name = $parent.speaker_config LIMIT 1) WHERE type::is::string(speaker_config);
DEFINE FIELD OVERWRITE speaker_config ON TABLE episode_profile TYPE option<record<speaker_profile>>;
open_notebook/database/migrations/20_down.surrealql
+12
View File
@@ -0,0 +1,12 @@
-- Migration 20 rollback: restore episode_profile.speaker_config to the speaker profile name
-- Widen the type first (SCHEMAFULL table), convert record links back to the
-- linked profile's name, then settle on option<string>. The original type was
-- a required string, but rows whose reference was orphaned by the up
-- migration (or whose linked speaker_profile was deleted since) have no name
-- to restore, so the rollback type must tolerate NONE.
DEFINE FIELD OVERWRITE speaker_config ON TABLE episode_profile TYPE option<string | record<speaker_profile>>;
UPDATE episode_profile SET speaker_config = speaker_config.name WHERE type::is::record(speaker_config);
DEFINE FIELD OVERWRITE speaker_config ON TABLE episode_profile TYPE option<string>;
open_notebook/database/migrations/21.surrealql
+29
View File
@@ -0,0 +1,29 @@
-- Migration 21: store episode.audio_file relative to PODCASTS_FOLDER (#1030)
-- audio_file used to hold the absolute path returned by podcast-creator
-- (str(Path.resolve())), sometimes with a legacy file:// prefix. From this
-- migration on the app stores the path RELATIVE to PODCASTS_FOLDER
-- (e.g. "episodes/<uuid>/audio/<uuid>.mp3") and resolves + contains it at a
-- single choke point (open_notebook/podcasts/audio_paths.py).
--
-- Conversion is by known-prefix stripping only, in two steps:
-- 1. Normalize plain "file:///" URIs to absolute paths. Only URIs without
-- percent-encoding are handled (SurrealQL cannot URL-decode) and only
-- the empty-host "file:///" form (a "file://host/" URI is not a local
-- path); anything else keeps its scheme and stays legacy-invalid.
-- 2. Strip EXACTLY ONE known root prefix per row (a single IF/ELSE chain,
-- so a remainder that happens to start with another prefix is never
-- stripped twice):
-- "/app/data/podcasts/" the shipped Docker layout (WORKDIR /app,
-- DATA_FOLDER "./data" resolved at write time)
-- "/data/podcasts/" older Docker layout with data mounted at /data
-- "./data/podcasts/" CWD-relative forms written by source
-- "data/podcasts/" checkouts running from the repo root
--
-- Rows under any OTHER root (e.g. a source checkout at a custom absolute
-- path) stay untouched by design (settled in #1030): the resolve helper
-- treats any non-relative value as legacy-invalid, preserving the 403/404
-- behavior #1018's guards give those rows today. No nulling, no flags.
UPDATE episode SET audio_file = string::slice(audio_file, 7) WHERE type::is::string(audio_file) AND string::starts_with(audio_file, "file:///") AND !string::contains(audio_file, "%");
UPDATE episode SET audio_file = IF string::starts_with(audio_file, "/app/data/podcasts/") { string::slice(audio_file, 19) } ELSE IF string::starts_with(audio_file, "/data/podcasts/") { string::slice(audio_file, 15) } ELSE IF string::starts_with(audio_file, "./data/podcasts/") { string::slice(audio_file, 16) } ELSE IF string::starts_with(audio_file, "data/podcasts/") { string::slice(audio_file, 14) } ELSE { audio_file } WHERE type::is::string(audio_file);
open_notebook/database/migrations/21_down.surrealql
+10
View File
@@ -0,0 +1,10 @@
-- Migration 21 rollback: restore absolute audio_file paths (best effort).
-- The up migration collapsed several historical prefixes (file://,
-- /app/data/podcasts/, /data/podcasts/, ./data/podcasts/, data/podcasts/)
-- into one relative form, so the original spelling of each row is not
-- recoverable. Rows are restored to the shipped Docker layout, which is
-- what pre-1030 code resolved them to in the standard deployment. Rows the
-- up migration left untouched (unknown roots) are already absolute and are
-- skipped.
UPDATE episode SET audio_file = string::concat("/app/data/podcasts/", audio_file) WHERE type::is::string(audio_file) AND audio_file != "" AND !string::starts_with(audio_file, "/") AND !string::starts_with(audio_file, "file://");
open_notebook/database/migrations/22.surrealql
+42
View File
@@ -0,0 +1,42 @@
-- Migration 22: drop legacy provider/model string fields from podcast profiles (#1107)
-- Phase 4 of #584. The app has ignored these strings since migration 14
-- introduced the Model registry references (outline_llm, transcript_llm,
-- voice_model); their only remaining role was as input to the startup data
-- migration retry loop (open_notebook/podcasts/migration.py), which this
-- migration replaces and retires.
--
-- Steps:
-- 1. Best-effort mapping: for profiles whose reference fields are still
-- empty but whose legacy provider+model strings are set, populate the
-- reference from a matching `model` record (provider + name + type).
-- This mirrors the retired Python data migration MINUS the auto-create
-- path: a migration has no business touching credentials, so profiles
-- with no matching model record simply stay unresolved. Accepted
-- trade-off (#1107): those profiles were already non-functional (the app
-- ignores legacy strings) and the UI already flags them as needing model
-- selection - the user re-picks models in the profile form once.
-- 2. Clear the stored legacy values (UNSET while the fields are still
-- defined, so rows don't keep stale data the schema no longer knows).
-- 3. Drop the 6 legacy field definitions.
-- Step 1: best-effort mapping (no-op on rows already migrated or with no
-- legacy strings; a subquery with no match yields NONE, leaving the row
-- unresolved, which is the accepted outcome).
UPDATE episode_profile SET outline_llm = (SELECT VALUE id FROM ONLY model WHERE provider = $parent.outline_provider AND name = $parent.outline_model AND type = "language" LIMIT 1) WHERE outline_llm = NONE AND outline_provider != NONE AND outline_provider != "" AND outline_model != NONE AND outline_model != "";
UPDATE episode_profile SET transcript_llm = (SELECT VALUE id FROM ONLY model WHERE provider = $parent.transcript_provider AND name = $parent.transcript_model AND type = "language" LIMIT 1) WHERE transcript_llm = NONE AND transcript_provider != NONE AND transcript_provider != "" AND transcript_model != NONE AND transcript_model != "";
UPDATE speaker_profile SET voice_model = (SELECT VALUE id FROM ONLY model WHERE provider = $parent.tts_provider AND name = $parent.tts_model AND type = "text_to_speech" LIMIT 1) WHERE voice_model = NONE AND tts_provider != NONE AND tts_provider != "" AND tts_model != NONE AND tts_model != "";
-- Step 2: clear stored legacy values before dropping the definitions.
UPDATE episode_profile UNSET outline_provider, outline_model, transcript_provider, transcript_model;
UPDATE speaker_profile UNSET tts_provider, tts_model;
-- Step 3: drop the legacy field definitions.
REMOVE FIELD IF EXISTS outline_provider ON TABLE episode_profile;
REMOVE FIELD IF EXISTS outline_model ON TABLE episode_profile;
REMOVE FIELD IF EXISTS transcript_provider ON TABLE episode_profile;
REMOVE FIELD IF EXISTS transcript_model ON TABLE episode_profile;
REMOVE FIELD IF EXISTS tts_provider ON TABLE speaker_profile;
REMOVE FIELD IF EXISTS tts_model ON TABLE speaker_profile;
open_notebook/database/migrations/22_down.surrealql
+14
View File
@@ -0,0 +1,14 @@
-- Migration 22 rollback: re-define the legacy provider/model string fields
-- (best effort, documented in #1107). The up migration UNSET the stored
-- values before dropping the definitions, so the legacy strings themselves
-- are NOT recoverable - this rollback only restores the schema shape that
-- migration 14 left (optional strings, app-ignored). Profiles keep working
-- through the Model registry references (outline_llm, transcript_llm,
-- voice_model), which the up migration only ever populated, never cleared.
DEFINE FIELD OVERWRITE outline_provider ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE outline_model ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE transcript_provider ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE transcript_model ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD OVERWRITE tts_provider ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD OVERWRITE tts_model ON TABLE speaker_profile TYPE option<string>;
open_notebook/database/migrations/2_down.surrealql
+1
View File
@@ -0,0 +1 @@
REMOVE FIELD IF EXISTS note_type ON TABLE note;
open_notebook/database/migrations/3.surrealql
+145
View File
@@ -0,0 +1,145 @@
DEFINE TABLE IF NOT EXISTS chat_session SCHEMALESS;
DEFINE TABLE IF NOT EXISTS refers_to
TYPE RELATION
FROM chat_session TO notebook;
REMOVE FUNCTION IF EXISTS fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) {
let $source_embedding_search =
IF $sources {(
SELECT
id,
source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT
id,
insight_type + ' - ' + source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT
id,
title,
content,
id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM note
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $all_results = array::union(
array::union($source_embedding_search, $source_insight_search),
$note_content_search
);
RETURN (
SELECT
id, title, content, parent_id,
math::max(similarity) as similarity
FROM $all_results
GROUP BY id
ORDER BY similarity DESC
LIMIT $match_count
);
};
REMOVE FUNCTION IF EXISTS fn::text_search;
DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_title_search =
IF $sources {(
SELECT id, title,
search::highlight('`', '`', 1) as content,
id as parent_id,
math::max(search::score(1)) AS relevance
FROM source
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_embedding_search =
IF $sources {(
SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source_embedding
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_full_search =
IF $sources {(
SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source
WHERE full_text @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source_insight
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_title_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
let $source_asset_results = array::union($source_title_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_asset_results );
let $note_results = array::union($note_title_search, $note_content_search );
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results
where id is not None
group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count);
};
open_notebook/database/migrations/3_down.surrealql
+110
View File
@@ -0,0 +1,110 @@
REMOVE TABLE IF EXISTS chat_session;
REMOVE TABLE IF EXISTS refers_to;
REMOVE FUNCTION fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_embedding_search =
IF $sources {(
SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding LIMIT $match_count)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight LIMIT $match_count)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
FROM note LIMIT $match_count)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_insight_search);
let $note_results = $note_content_search;
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results
group by item_id ORDER BY similarity DESC LIMIT $match_count);
};
REMOVE FUNCTION fn::text_search;
DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_title_search =
IF $sources {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM source
WHERE title @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_embedding_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source_embedding
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_full_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source
WHERE full_text @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT source as item_id, math::max(search::score(1)) AS relevance
FROM source_insight
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $note_title_search =
IF $show_notes {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM note
WHERE title @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id as item_id, math::max(search::score(1)) AS relevance
FROM note
WHERE content @1@ $query_text
GROUP BY item_id)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
let $source_asset_results = array::union($source_title_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_asset_results );
let $note_results = array::union($note_title_search, $note_content_search );
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results
group by item_id ORDER BY relevance DESC LIMIT $match_count);
};
open_notebook/database/migrations/4.surrealql
+134
View File
@@ -0,0 +1,134 @@
REMOVE FUNCTION IF EXISTS fn::text_search;
DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_title_search =
IF $sources {(
SELECT id, title,
search::highlight('`', '`', 1) as content,
id as parent_id,
math::max(search::score(1)) AS relevance
FROM source
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_embedding_search =
IF $sources {(
SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source_embedding
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_full_search =
IF $sources {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM source
WHERE full_text @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT id, insight_type + " - " + (source.title OR '') as title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM source_insight
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_title_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
let $source_asset_results = array::union($source_title_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_asset_results );
let $note_results = array::union($note_title_search, $note_content_search );
let $final_results = array::union($source_results, $note_results );
RETURN (select id, parent_id, title, math::max(relevance) as relevance
from $final_results where id is not None
group by id, parent_id, title ORDER BY relevance DESC LIMIT $match_count);
};
REMOVE FUNCTION IF EXISTS fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) {
let $source_embedding_search =
IF $sources {(
SELECT
source.id as id,
source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT
id,
insight_type + ' - ' + (source.title OR '') as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT
id,
title,
content,
id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM note
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $all_results = array::union(
array::union($source_embedding_search, $source_insight_search),
$note_content_search
);
RETURN (select id, parent_id, title, math::max(similarity) as similarity,
array::flatten(content) as matches
from $all_results where id is not None
group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count);
};
open_notebook/database/migrations/4_down.surrealql
+139
View File
@@ -0,0 +1,139 @@
REMOVE FUNCTION IF EXISTS fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) {
let $source_embedding_search =
IF $sources {(
SELECT
id,
source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT
id,
insight_type + ' - ' + source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT
id,
title,
content,
id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM note
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $all_results = array::union(
array::union($source_embedding_search, $source_insight_search),
$note_content_search
);
RETURN (
SELECT
id, title, content, parent_id,
math::max(similarity) as similarity
FROM $all_results
GROUP BY id
ORDER BY similarity DESC
LIMIT $match_count
);
};
REMOVE FUNCTION IF EXISTS fn::text_search;
DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
let $source_title_search =
IF $sources {(
SELECT id, title,
search::highlight('`', '`', 1) as content,
id as parent_id,
math::max(search::score(1)) AS relevance
FROM source
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_embedding_search =
IF $sources {(
SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source_embedding
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_full_search =
IF $sources {(
SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source
WHERE full_text @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance
FROM source_insight
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_title_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE title @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance
FROM note
WHERE content @1@ $query_text
GROUP BY id)}
ELSE { [] };
let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
let $source_asset_results = array::union($source_title_search, $source_insight_search);
let $source_results = array::union($source_chunk_results, $source_asset_results );
let $note_results = array::union($note_title_search, $note_content_search );
let $final_results = array::union($source_results, $note_results );
RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results
where id is not None
group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count);
};
open_notebook/database/migrations/5.surrealql
+169
View File
@@ -0,0 +1,169 @@
-- remove old transformation defaults
DELETE open_notebook:default_transformations;
-- set up the default transformations
DEFINE TABLE IF NOT EXISTS transformation SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS name ON TABLE transformation TYPE string;
DEFINE FIELD IF NOT EXISTS title ON TABLE transformation TYPE string;
DEFINE FIELD IF NOT EXISTS description ON TABLE transformation TYPE string;
DEFINE FIELD IF NOT EXISTS prompt ON TABLE transformation TYPE string;
DEFINE FIELD IF NOT EXISTS apply_default ON TABLE transformation TYPE bool DEFAULT False;
DEFINE FIELD IF NOT EXISTS created ON transformation DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON transformation DEFAULT time::now() VALUE time::now();
insert into transformation [
{
name: "Analyze Paper",
title: "Paper Analysis",
description: "Analyses a technical/scientific paper",
prompt:"# IDENTITY and PURPOSE
You are an insightful and analytical reader of academic papers, extracting the key components, significance, and broader implications. Your focus is to uncover the core contributions, practical applications, methodological strengths or weaknesses, and any surprising findings. You are especially attuned to the clarity of arguments, the relevance to existing literature, and potential impacts on both the specific field and broader contexts.
# STEPS
1. **READ AND UNDERSTAND THE PAPER**: Thoroughly read the paper, identifying its main focus, arguments, methods, results, and conclusions.
2. **IDENTIFY CORE ELEMENTS**:
- **Purpose**: What is the main goal or research question?
- **Contribution**: What new knowledge or innovation does this paper bring to the field?
- **Methods**: What methods are used, and are they novel or particularly effective?
- **Key Findings**: What are the most critical results, and why do they matter?
- **Limitations**: Are there any notable limitations or areas for further research?
3. **SYNTHESIZE THE MAIN POINTS**:
- Extract the key elements and organize them into insightful observations.
- Highlight the broader impact and potential applications.
- Note any aspects that challenge established views or introduce new questions.
# OUTPUT INSTRUCTIONS
- Structure the output as follows:
- **PURPOSE**: A concise summary of the main research question or goal (1-2 sentences).
- **CONTRIBUTION**: A bullet list of 2-3 points that describe what the paper adds to the field.
- **KEY FINDINGS**: A bullet list of 2-3 points summarizing the critical outcomes of the study.
- **IMPLICATIONS**: A bullet list of 2-3 points discussing the significance or potential impact of the findings on the field or broader context.
- **LIMITATIONS**: A bullet list of 1-2 points identifying notable limitations or areas for future work.
- **Bullet Points** should be between 15-20 words.
- Avoid starting each bullet point with the same word to maintain variety.
- Use clear and concise language that conveys the key ideas effectively.
- Do not include warnings, disclaimers, or personal opinions.
- Output only the requested sections with their respective labels.",
apply_default: False
},
{
name: "Key Insights",
title: "Key Insights",
description: "Extracts important insights and actionable items",
prompt:"# IDENTITY and PURPOSE
You extract surprising, powerful, and interesting insights from text content. You are interested in insights related to the purpose and meaning of life, human flourishing, the role of technology in the future of humanity, artificial intelligence and its affect on humans, memes, learning, reading, books, continuous improvement, and similar topics.
You create 15 word bullet points that capture the most important insights from the input.
Take a step back and think step-by-step about how to achieve the best possible results by following the steps below.
# STEPS
- Extract 20 to 50 of the most surprising, insightful, and/or interesting ideas from the input in a section called IDEAS, and write them on a virtual whiteboard in your mind using 15 word bullets. If there are less than 50 then collect all of them. Make sure you extract at least 20.
- From those IDEAS, extract the most powerful and insightful of them and write them in a section called INSIGHTS. Make sure you extract at least 10 and up to 25.
# OUTPUT INSTRUCTIONS
- INSIGHTS are essentially higher-level IDEAS that are more abstracted and wise.
- Output the INSIGHTS section only.
- Each bullet should be about 15 words in length.
- Do not give warnings or notes; only output the requested sections.
- You use bulleted lists for output, not numbered lists.
- Do not start items with the same opening words.
- Ensure you follow ALL these instructions when creating your output.
",
apply_default: False
},
{
name: "Dense Summary",
title: "Dense Summary",
description: "Creates a rich, deep summary of the content",
prompt:"# MISSION
You are a Sparse Priming Representation (SPR) writer. An SPR is a particular kind of use of language for advanced NLP, NLU, and NLG tasks, particularly useful for the latest generation of Large Language Models (LLMs). You will be given information by the USER which you are to render as an SPR.
# THEORY
LLMs are a kind of deep neural network. They have been demonstrated to embed knowledge, abilities, and concepts, ranging from reasoning to planning, and even to theory of mind. These are called latent abilities and latent content, collectively referred to as latent space. The latent space of an LLM can be activated with the correct series of words as inputs, which will create a useful internal state of the neural network. This is not unlike how the right shorthand cues can prime a human mind to think in a certain way. Like human minds, LLMs are associative, meaning you only need to use the correct associations to 'prime' another model to think in the same way.
# METHODOLOGY
Render the input as a distilled list of succinct statements, assertions, associations, concepts, analogies, and metaphors. The idea is to capture as much, conceptually, as possible but with as few words as possible. Write it in a way that makes sense to you, as the future audience will be another language model, not a human. Use complete sentences.
",
apply_default: True
},
{
name: "Reflections",
title: "Reflection Questions",
description: "Generates reflection questions from the document to help explore it further",
prompt:"# IDENTITY and PURPOSE
You extract deep, thought-provoking, and meaningful reflections from text content. You are especially focused on themes related to the human experience, such as the purpose of life, personal growth, the intersection of technology and humanity, artificial intelligence's societal impact, human potential, collective evolution, and transformative learning. Your reflections aim to provoke new ways of thinking, challenge assumptions, and provide a thoughtful synthesis of the content.
# STEPS
- Extract 3 to 5 of the most profound, thought-provoking, and/or meaningful ideas from the input in a section called REFLECTIONS.
- Each reflection should aim to explore underlying implications, connections to broader human experiences, or highlight a transformative perspective.
- Take a step back and consider the deeper significance or questions that arise from the content.
# OUTPUT INSTRUCTIONS
- The output section should be labeled as REFLECTIONS.
- Each bullet point should be between 20-25 words.
- Avoid repetition in the phrasing and ensure variety in sentence structure.
- The reflections should encourage deeper inquiry and provide a synthesis that transcends surface-level observations.
- Use bullet points, not numbered lists.
- Every bullet should be formatted as a question that elicits contemplation or a statement that offers a profound insight.
- Do not give warnings or notes; only output the requested section.",
apply_default: False
},
{
name: "Table of Contents",
title: "Table of Contents",
description: "Describes the different topics of the document",
prompt:"# SYSTEM ROLE
You are a content analysis assistant that reads through documents and provides a Table of Contents (ToC) to help users identify what the document covers more easily.
Your ToC should capture all major topics and transitions in the content and should mention them in the order theh appear.
# TASK
Analyze the provided content and create a Table of Contents:
- Captures the core topics included in the text
- Gives a small description of what is covered",
apply_default: False
},
{
name: "Simple Summary",
title: "Simple Summary",
description: "Generates a small summary of the content",
prompt:"# SYSTEM ROLE
You are a content summarization assistant that creates dense, information-rich summaries optimized for machine understanding. Your summaries should capture key concepts with minimal words while maintaining complete, clear sentences.
# TASK
Analyze the provided content and create a summary that:
- Captures the core concepts and key information
- Uses clear, direct language
- Maintains context from any previous summaries",
apply_default: False
},
];
-- Sets the default transformation instructions prompt
UPSERT open_notebook:default_prompts
CONTENT {transformation_instructions: "# INSTRUCTIONS
You are my learning assistant and you help me process and transform content so that I can extract insights from them.
# IMPORTANT
- You are working on my editorial projects. The text below is my own. Do not give me any warnings about copyright or plagiarism.
- Output ONLY the requested content, without acknowledgements of the task and additional chatting. Don't start with \"Sure, I can help you with that.\" or \"Here is the information you requested:\". Just provide the content.
- Do not stop in the middle of the generation to ask me questions. Execute my request completely.
"};
open_notebook/database/migrations/5_down.surrealql
+2
View File
@@ -0,0 +1,2 @@
REMOVE TABLE IF EXISTS transformation SCHEMAFULL;
open_notebook/database/migrations/6.surrealql
+1
View File
@@ -0,0 +1 @@
update model set provider='vertex' where provider='vertexai';
open_notebook/database/migrations/6_down.surrealql
+1
View File
@@ -0,0 +1 @@
update model set provider='vertexai' where provider='vertex';
open_notebook/database/migrations/7.surrealql
+152
View File
@@ -0,0 +1,152 @@
DEFINE TABLE IF NOT EXISTS episode_profile SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS name ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS description ON TABLE episode_profile TYPE option<string>;
DEFINE FIELD IF NOT EXISTS speaker_config ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS outline_provider ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS outline_model ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS transcript_provider ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS transcript_model ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS default_briefing ON TABLE episode_profile TYPE string;
DEFINE FIELD IF NOT EXISTS num_segments ON TABLE episode_profile TYPE int DEFAULT 5;
DEFINE FIELD IF NOT EXISTS created ON TABLE episode_profile TYPE datetime DEFAULT time::now();
DEFINE FIELD IF NOT EXISTS updated ON TABLE episode_profile TYPE datetime DEFAULT time::now();
-- Create Speaker Profile table
remove table speaker_profile;
DEFINE TABLE IF NOT EXISTS speaker_profile SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS name ON TABLE speaker_profile TYPE string;
DEFINE FIELD IF NOT EXISTS description ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD IF NOT EXISTS tts_provider ON TABLE speaker_profile TYPE string;
DEFINE FIELD IF NOT EXISTS tts_model ON TABLE speaker_profile TYPE string;
DEFINE FIELD IF NOT EXISTS speakers ON TABLE speaker_profile TYPE array<object>;
DEFINE FIELD IF NOT EXISTS speakers.*.name ON TABLE speaker_profile TYPE string;
DEFINE FIELD IF NOT EXISTS speakers.*.voice_id ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD IF NOT EXISTS speakers.*.backstory ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD IF NOT EXISTS speakers.*.personality ON TABLE speaker_profile TYPE option<string>;
DEFINE FIELD IF NOT EXISTS created ON TABLE speaker_profile TYPE datetime DEFAULT time::now();
DEFINE FIELD IF NOT EXISTS updated ON TABLE speaker_profile TYPE datetime DEFAULT time::now();
-- Enhance PodcastEpisode table
DEFINE TABLE IF NOT EXISTS episode SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS created ON episode DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON episode DEFAULT time::now() VALUE time::now();
DEFINE FIELD IF NOT EXISTS name ON TABLE episode TYPE string;
DEFINE FIELD IF NOT EXISTS briefing ON TABLE episode TYPE option<string>;
DEFINE FIELD IF NOT EXISTS episode_profile ON TABLE episode FLEXIBLE TYPE object;
DEFINE FIELD IF NOT EXISTS speaker_profile ON TABLE episode FLEXIBLE TYPE object;
DEFINE FIELD IF NOT EXISTS transcript ON TABLE episode FLEXIBLE TYPE option<object>;
DEFINE FIELD IF NOT EXISTS outline ON TABLE episode FLEXIBLE TYPE option<object>;
DEFINE FIELD IF NOT EXISTS command ON TABLE episode TYPE option<record<command>>;
DEFINE FIELD IF NOT EXISTS content ON TABLE episode TYPE option<string>;
DEFINE FIELD IF NOT EXISTS audio_file ON TABLE episode TYPE option<string>;
-- Create indexes for better performance
DEFINE INDEX IF NOT EXISTS idx_episode_profile_name ON TABLE episode_profile COLUMNS name UNIQUE CONCURRENTLY;
DEFINE INDEX IF NOT EXISTS idx_speaker_profile_name ON TABLE speaker_profile COLUMNS name UNIQUE CONCURRENTLY;
DEFINE INDEX IF NOT EXISTS idx_episode_profile ON TABLE episode COLUMNS episode_profile CONCURRENTLY;
DEFINE INDEX IF NOT EXISTS idx_episode_command ON TABLE episode COLUMNS command CONCURRENTLY;
--Sample data
insert into episode_profile
[
{
name: "tech_discussion",
description: "Technical discussion between 2 experts",
speaker_config: "tech_experts",
outline_provider: "openai",
outline_model: "gpt-5-mini",
transcript_provider: "openai",
transcript_model: "gpt-5-mini",
default_briefing: "Create an engaging technical discussion about the provided content. Focus on practical insights, real-world applications, and detailed explanations that would interest developers and technical professionals.",
num_segments: 5
},
{
name: "solo_expert",
description: "Single expert explaining complex topics",
speaker_config: "solo_expert",
outline_provider: "openai",
outline_model: "gpt-5-mini",
transcript_provider: "openai",
transcript_model: "gpt-5-mini",
default_briefing: "Create an educational explanation of the provided content. Break down complex concepts into digestible segments, use analogies and examples, and maintain an engaging teaching style.",
"num_segments":4 },
{
name: "business_analysis",
description: "Business-focused analysis and discussion",
speaker_config: "business_panel",
outline_provider: "openai",
outline_model: "gpt-5-mini",
transcript_provider: "openai",
transcript_model: "gpt-5-mini",
default_briefing: "Analyze the provided content from a business perspective. Discuss market implications, strategic insights, competitive advantages, and actionable business intelligence.",
"num_segments":6 }
];
insert into speaker_profile
[
{
name: "tech_experts",
description: "Two technical experts for tech discussions",
tts_provider: "openai",
tts_model: "gpt-4o-mini-tts",
speakers: [
{
name: "Dr. Alex Chen",
voice_id: "nova",
backstory: "Senior AI researcher and former tech lead at major companies. Specializes in making complex technical concepts accessible.",
personality: "Analytical, clear communicator, asks probing questions to dig deeper into technical details"
},
{
name: "Jamie Rodriguez",
voice_id: "alloy",
backstory: "Full-stack engineer and tech entrepreneur. Loves practical applications and real-world implementations.",
personality: "Enthusiastic, practical-minded, great at explaining implementation details and trade-offs"
}
]
},
{
name: "solo_expert",
description: "Single expert for educational content",
tts_provider: "openai",
tts_model: "gpt-4o-mini-tts",
speakers: [
{
name: "Professor Sarah Kim",
voice_id: "nova",
backstory: "Distinguished professor and researcher. Has a gift for making complex topics accessible to broad audiences.",
personality: "Patient teacher, uses analogies and examples, breaks down complex concepts step by step"
}
]
},
{
name: "business_panel",
description: "Business analysis panel with diverse perspectives",
tts_provider: "openai",
tts_model: "gpt-4o-mini-tts",
speakers: [
{
name: "Marcus Thompson",
voice_id: "echo",
backstory: "Former McKinsey consultant, now startup advisor. Expert in strategic analysis and market dynamics.",
personality: "Strategic thinker, data-driven, excellent at identifying key insights and implications"
},
{
name: "Elena Vasquez",
voice_id: "shimmer",
backstory: "Serial entrepreneur and investor. Focuses on practical implementation and execution.",
personality: "Action-oriented, pragmatic, brings startup experience and execution focus"
},
{
name: "Johny Bing",
voice_id: "ash",
backstory: "Youtube celebrity and business mogul. Focuses on practical implementation and execution.",
personality: "Controversial, likes to question ideas and concepts. He brings a fresh perspective and always has a point to make."
}
]
}
];
open_notebook/database/migrations/7_down.surrealql
+3
View File
@@ -0,0 +1,3 @@
REMOVE TABLE IF EXISTS episode_profile;
REMOVE TABLE IF EXISTS speaker_profile;
REMOVE TABLE IF EXISTS episode;
open_notebook/database/migrations/8.surrealql
+12
View File
@@ -0,0 +1,12 @@
-- Migration 8: Support chat sessions for both notebooks and sources
-- This migration allows chat_session to refer to either a notebook or a source
DEFINE TABLE OVERWRITE refers_to
TYPE RELATION
FROM chat_session TO notebook|source;
-- Add model_override field to chat_session for per-session model selection
DEFINE FIELD model_override ON chat_session TYPE option<string>;
DEFINE FIELD command ON source TYPE option<record<command>>;
open_notebook/database/migrations/8_down.surrealql
+11
View File
@@ -0,0 +1,11 @@
-- Rollback Migration 8: Revert to notebook-only chat sessions
DEFINE TABLE OVERWRITE refers_to
TYPE RELATION
FROM chat_session TO notebook;
-- Remove model_override field from chat_session
REMOVE FIELD model_override ON chat_session;
REMOVE FIELD command ON source;
open_notebook/database/migrations/9.surrealql
+66
View File
@@ -0,0 +1,66 @@
REMOVE FUNCTION IF EXISTS fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) {
let $source_embedding_search =
IF $sources {(
SELECT
source.id as id,
source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding
WHERE embedding != none and array::len(embedding)=array::len($query) AND
vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT
id,
insight_type + ' - ' + (source.title OR '') as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight
WHERE embedding != none and array::len(embedding)=array::len($query) AND
vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT
id,
title,
content,
id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM note
WHERE embedding != none and array::len(embedding)=array::len($query) AND
vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $all_results = array::union(
array::union($source_embedding_search, $source_insight_search),
$note_content_search
);
RETURN (select id, parent_id, title, math::max(similarity) as similarity,
array::flatten(content) as matches
from $all_results where id is not None
group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count);
};
open_notebook/database/migrations/9_down.surrealql
+63
View File
@@ -0,0 +1,63 @@
REMOVE FUNCTION IF EXISTS fn::vector_search;
DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) {
let $source_embedding_search =
IF $sources {(
SELECT
source.id as id,
source.title as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_embedding
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $source_insight_search =
IF $sources {(
SELECT
id,
insight_type + ' - ' + (source.title OR '') as title,
content,
source.id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM source_insight
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $note_content_search =
IF $show_notes {(
SELECT
id,
title,
content,
id as parent_id,
vector::similarity::cosine(embedding, $query) as similarity
FROM note
WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity
ORDER BY similarity DESC
LIMIT $match_count
)}
ELSE { [] };
let $all_results = array::union(
array::union($source_embedding_search, $source_insight_search),
$note_content_search
);
RETURN (select id, parent_id, title, math::max(similarity) as similarity,
array::flatten(content) as matches
from $all_results where id is not None
group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count);
};
open_notebook/database/repository.py
+237
View File
@@ -0,0 +1,237 @@
import os
import re
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, TypeVar, Union
from loguru import logger
from surrealdb import AsyncSurreal, RecordID # type: ignore
from surrealdb.data.types.table import Table # type: ignore
T = TypeVar("T", Dict[str, Any], List[Dict[str, Any]])
# Bare SurrealDB table/relation identifier: no ':', whitespace, or query
# syntax. Used to validate the parts of RELATE/UPSERT/UPDATE that name a
# table or edge-relation and therefore can't be bound as a query parameter
# (SurrealQL only allows binding record/table *values*, not identifiers in
# that position).
_IDENTIFIER_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
def _ensure_safe_identifier(value: str, kind: str) -> str:
"""Validate a table/relationship name before it is interpolated into a query."""
if not isinstance(value, str) or not _IDENTIFIER_RE.match(value):
raise ValueError(f"Invalid {kind} name: {value!r}")
return value
def _get_env_or_default(name: str, default: str) -> str:
value = os.getenv(name)
return value if value else default
def get_database_url() -> str:
"""Get database URL with backward compatibility"""
surreal_url = os.getenv("SURREAL_URL")
if surreal_url:
return surreal_url
# Fallback to old format - WebSocket URL format
address = os.getenv("SURREAL_ADDRESS", "localhost")
port = os.getenv("SURREAL_PORT", "8000")
return f"ws://{address}/rpc:{port}"
def get_database_password() -> str:
"""Get password with backward compatibility"""
return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS") or "root"
def get_database_namespace() -> str:
"""Get configured SurrealDB namespace."""
return _get_env_or_default("SURREAL_NAMESPACE", "open_notebook")
def get_database_name() -> str:
"""Get configured SurrealDB database name."""
return _get_env_or_default("SURREAL_DATABASE", "open_notebook")
def parse_record_ids(obj: Any) -> Any:
"""Recursively parse and convert RecordIDs into strings."""
if isinstance(obj, dict):
return {k: parse_record_ids(v) for k, v in obj.items()}
elif isinstance(obj, list):
return [parse_record_ids(item) for item in obj]
elif isinstance(obj, RecordID):
return str(obj)
return obj
def ensure_record_id(value: Union[str, RecordID]) -> RecordID:
"""Ensure a value is a RecordID."""
if isinstance(value, RecordID):
return value
return RecordID.parse(value)
@asynccontextmanager
async def db_connection():
db = AsyncSurreal(get_database_url())
await db.signin(
{
"username": os.environ.get("SURREAL_USER"),
"password": get_database_password(),
}
)
await db.use(get_database_namespace(), get_database_name())
try:
yield db
finally:
await db.close()
async def repo_query(
query_str: str, vars: Optional[Dict[str, Any]] = None
) -> List[Dict[str, Any]]:
"""Execute a SurrealQL query and return the results"""
async with db_connection() as connection:
try:
result = parse_record_ids(await connection.query(query_str, vars))
if isinstance(result, str):
raise RuntimeError(result)
return result
except RuntimeError as e:
# RuntimeError is raised for retriable transaction conflicts - log at debug to avoid noise
logger.debug(str(e))
raise
except Exception as e:
logger.exception(e)
raise
async def repo_create(table: str, data: Dict[str, Any]) -> Dict[str, Any]:
"""Create a new record in the specified table"""
# Remove 'id' attribute if it exists in data
data.pop("id", None)
data["created"] = datetime.now(timezone.utc)
data["updated"] = datetime.now(timezone.utc)
try:
async with db_connection() as connection:
result = parse_record_ids(await connection.insert(table, data))
# SurrealDB may return a string error message instead of the expected record
if isinstance(result, str):
raise RuntimeError(result)
return result
except RuntimeError as e:
logger.error(str(e))
raise
except Exception as e:
logger.exception(e)
raise RuntimeError("Failed to create record")
async def repo_relate(
source: Union[str, RecordID],
relationship: str,
target: Union[str, RecordID],
data: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
"""Create a relationship between two records with optional data"""
if data is None:
data = {}
# relationship is an edge-table name, not a record value, so it can't be
# bound as a query parameter; validate it against an identifier allowlist
# instead of trusting the caller. source/target are always bound.
_ensure_safe_identifier(relationship, "relationship")
query = f"RELATE $source->{relationship}->$target CONTENT $data;"
# logger.debug(f"Relate query: {query}")
return await repo_query(
query,
{
"source": ensure_record_id(source),
"target": ensure_record_id(target),
"data": data,
},
)
async def repo_upsert(
table: str, id: Optional[str], data: Dict[str, Any], add_timestamp: bool = False
) -> List[Dict[str, Any]]:
"""Create or update a record in the specified table"""
data.pop("id", None)
if add_timestamp:
data["updated"] = datetime.now(timezone.utc)
_ensure_safe_identifier(table, "table")
target: Union[RecordID, Table] = ensure_record_id(id) if id else Table(table)
query = "UPSERT $target MERGE $data;"
return await repo_query(query, {"target": target, "data": data})
async def repo_update(
table: str, id: Union[str, RecordID], data: Dict[str, Any]
) -> List[Dict[str, Any]]:
"""Update an existing record by table and id"""
# If id already contains the table name, use it as is
try:
_ensure_safe_identifier(table, "table")
if isinstance(id, RecordID):
record_id: RecordID = id
elif ":" in id and id.startswith(f"{table}:"):
record_id = ensure_record_id(id)
else:
record_id = RecordID(table, id)
data.pop("id", None)
if "created" in data and isinstance(data["created"], str):
data["created"] = datetime.fromisoformat(data["created"])
data["updated"] = datetime.now(timezone.utc)
query = "UPDATE $target MERGE $data;"
# logger.debug(f"Update query: {query}")
result = await repo_query(query, {"target": record_id, "data": data})
# if isinstance(result, list):
# return [_return_data(item) for item in result]
return parse_record_ids(result)
except Exception as e:
raise RuntimeError(f"Failed to update record: {str(e)}")
async def repo_delete(record_id: Union[str, RecordID]):
"""Delete a record by record id"""
try:
async with db_connection() as connection:
return await connection.delete(ensure_record_id(record_id))
except Exception as e:
logger.exception(e)
raise RuntimeError(f"Failed to delete record: {str(e)}")
async def repo_insert(
table: str, data: List[Dict[str, Any]], ignore_duplicates: bool = False
) -> List[Dict[str, Any]]:
"""Create a new record in the specified table"""
try:
async with db_connection() as connection:
result = parse_record_ids(await connection.insert(table, data))
# SurrealDB may return a string error message instead of the expected records
if isinstance(result, str):
raise RuntimeError(result)
return result
except RuntimeError as e:
if ignore_duplicates and "already contains" in str(e):
return []
# Log transaction conflicts at debug level (they are expected during concurrent operations)
error_str = str(e).lower()
if "transaction" in error_str or "conflict" in error_str:
logger.debug(str(e))
else:
logger.error(str(e))
raise
except Exception as e:
if ignore_duplicates and "already contains" in str(e):
return []
logger.exception(e)
raise RuntimeError("Failed to create record")