commit fbab2c6005de37f758e455a3c2f58df6adde2b32 Author: wehub-resource-sync Date: Mon Jul 13 12:10:23 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b512aa6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,52 @@ +# Git +.git +.gitignore + +# Python +__pycache__ +*.pyc +*.pyo +*.pyd +.venv +venv +ENV +env +.pytest_cache +.mypy_cache +.ruff_cache + +# Frontend +frontend/node_modules +frontend/.next +frontend/dist +frontend/out +frontend/.env* +frontend/*.log + +# Project data +.antigravity +.gemini +tmp +data +mydata +notebook_data +surreal_data +surreal-data +surreal_single_data +*.db +*.log +docker.env +.env +docker-compose* + +# Documentation & CI (not needed in image) +docs +.github + +# IDE and OS files +.vscode +.idea +*.swp +*.swo +*~ +.DS_Store \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4f8fe6e --- /dev/null +++ b/.env.example @@ -0,0 +1,68 @@ +# Open Notebook Configuration +# Copy this file to .env and customize as needed + +# ============================================================================= +# REQUIRED +# ============================================================================= + +# Encryption key for storing API credentials securely in the database +# Change this to any secret string (minimum 16 characters recommended) +OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string + +# ============================================================================= +# DATABASE (Default values work with docker-compose.yml) +# ============================================================================= + +SURREAL_URL=ws://surrealdb:8000/rpc +# root:root is fine for local use. Change these before exposing the instance to a +# network โ€” docker-compose.yml reads them for both the SurrealDB server and the +# app, so the two stay in sync. +SURREAL_USER=root +SURREAL_PASSWORD=root +SURREAL_NAMESPACE=open_notebook +SURREAL_DATABASE=open_notebook + +# ============================================================================= +# OPTIONAL: AI Provider API Keys +# ============================================================================= +# You can configure these via the UI (Settings โ†’ API Keys) or set them here +# UI configuration is recommended for better security and flexibility + +# OpenAI +# OPENAI_API_KEY=sk-... + +# Anthropic +# ANTHROPIC_API_KEY=sk-ant-... + +# Google AI +# GOOGLE_API_KEY=... + +# Groq +# GROQ_API_KEY=gsk_... + +# ============================================================================= +# OPTIONAL: Advanced Configuration +# ============================================================================= + +# External API URL (for webhooks, callbacks, etc.) +# API_URL=http://localhost:5055 + +# Ollama endpoint (if running locally) +# OLLAMA_BASE_URL=http://ollama:11434 + +# Content processing +# CHUNK_SIZE=1500 +# CHUNK_OVERLAP=150 + +# Maximum upload/request body size in MB (default: 100). Raise this if you +# need to upload larger audio/video files. A fronting reverse proxy's own +# limit (e.g. nginx client_max_body_size) still applies and should be raised +# to match. +# OPEN_NOTEBOOK_MAX_UPLOAD_SIZE_MB=100 + +# Security +# BASIC_AUTH_USERNAME=admin +# BASIC_AUTH_PASSWORD=secret + +# For more configuration options, see: +# https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/environment-reference.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..369f4ef --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Ensure shell scripts always use LF so they run in Linux containers (e.g. Docker) +*.sh text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..97f4a94 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,104 @@ +name: ๐Ÿ› Bug Report +description: Report a bug or unexpected behavior (app is running but misbehaving) +title: "[Bug]: " +labels: ["bug", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug! Please fill out the information below to help us understand and fix the issue. + + **Note**: If you're having installation or setup issues, please use the "Installation Issue" template instead. + + - type: textarea + id: what-happened + attributes: + label: What did you do when it broke? + description: Describe the steps you took that led to the bug + placeholder: | + 1. I went to the Notebooks page + 2. I clicked on "Create New Notebook" + 3. I filled in the form and clicked "Save" + 4. Then the error occurred... + validations: + required: true + + - type: textarea + id: how-broke + attributes: + label: How did it break? + description: What happened that was unexpected? What did you expect to happen instead? + placeholder: | + Expected: The notebook should be created and I should see it in the list + Actual: I got an error message saying "Failed to create notebook" + validations: + required: true + + - type: textarea + id: logs-screenshots + attributes: + label: Logs or Screenshots + description: | + Please provide any error messages, logs, or screenshots that might help us understand the issue. + + **How to get logs:** + - Docker: `docker compose logs -f open_notebook` + - Check browser console (F12 โ†’ Console tab) + placeholder: | + Paste logs here or drag and drop screenshots. + + Error messages, stack traces, or browser console errors are very helpful! + validations: + required: false + + - type: dropdown + id: version + attributes: + label: Open Notebook Version + description: Which version are you using? + options: + - v1-latest (Docker) + - v1-latest-single (Docker, deprecated) + - Latest from main branch + - Other (please specify in additional context) + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: What environment are you running in? + placeholder: | + - OS: Ubuntu 22.04 / Windows 11 / macOS 14 + - Browser: Chrome 120 + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Any other information that might be helpful + placeholder: "This started happening after I upgraded to v1.5.0..." + validations: + required: false + + - type: checkboxes + id: willing-to-contribute + attributes: + label: Contribution + description: Would you like to work on fixing this bug? + options: + - label: I am a developer and would like to work on fixing this issue (pending maintainer approval) + required: false + + - type: markdown + attributes: + value: | + --- + **Next Steps:** + 1. A maintainer will review your bug report + 2. If you checked the box above and want to fix it, please propose your solution approach + 3. Wait for assignment before starting development + 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..630377f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: ๐Ÿ’ฌ Discord Community + url: https://discord.gg/37XJPXfz2w + about: Get help from the community and share ideas + - name: ๐Ÿค– Installation Assistant (ChatGPT) + url: https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant + about: CustomGPT that knows all our docs. Really useful. Try it. + - name: ๐Ÿ“š Documentation + url: https://github.com/lfnovo/open-notebook/tree/main/docs + about: Browse our comprehensive documentation diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..912f1f1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,65 @@ +name: โœจ Feature Suggestion +description: Suggest a new feature or improvement for Open Notebook +title: "[Feature]: " +labels: ["enhancement", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to suggest a feature! Your ideas help make Open Notebook better for everyone. + + - type: textarea + id: feature-description + attributes: + label: Feature Description + description: What feature would you like to see added or improved? + placeholder: "I would like to be able to..." + validations: + required: true + + - type: textarea + id: why-helpful + attributes: + label: Why would this be helpful? + description: Explain how this feature would benefit you and other users + placeholder: "This would help because..." + validations: + required: true + + - type: textarea + id: proposed-solution + attributes: + label: Proposed Solution (Optional) + description: If you have ideas on how to implement this feature, please share them + placeholder: "This could be implemented by..." + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Any other context, screenshots, or examples that might be helpful + placeholder: "For example, other tools do this by..." + validations: + required: false + + - type: checkboxes + id: willing-to-contribute + attributes: + label: Contribution + description: Would you like to work on implementing this feature? + options: + - label: I am a developer and would like to work on implementing this feature (pending maintainer approval) + required: false + + - type: markdown + attributes: + value: | + --- + **Next Steps:** + 1. A maintainer will review your feature request + 2. If approved and you checked the box above, the issue will be assigned to you + 3. Please wait for assignment before starting development + 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details + diff --git a/.github/ISSUE_TEMPLATE/installation_issue.yml b/.github/ISSUE_TEMPLATE/installation_issue.yml new file mode 100644 index 0000000..3ee32f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/installation_issue.yml @@ -0,0 +1,148 @@ +name: ๐Ÿ”ง Installation Issue +description: Report problems with installation, setup, or connectivity +title: "[Install]: " +labels: ["installation", "needs-triage"] +body: + - type: markdown + attributes: + value: | + ## โš ๏ธ Before You Continue + + **Please try these resources first:** + + 1. ๐Ÿค– **[Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)** - Our AI assistant can help you troubleshoot most installation issues instantly! + + 2. ๐Ÿ“š **[Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md)** - Comprehensive setup instructions + + 3. ๐Ÿ‹ **[Docker Deployment Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/docker.md)** - Detailed Docker setup + + 4. ๐Ÿฆ™ **Ollama Issues?** Read our [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md) first + + 5. ๐Ÿ’ฌ **[Discord Community](https://discord.gg/37XJPXfz2w)** - Get real-time help from the community + + --- + + If you've tried the above and still need help, please fill out the form below with as much detail as possible. + + - type: dropdown + id: installation-method + attributes: + label: Installation Method + description: How are you trying to install Open Notebook? + options: + - Docker (docker-compose - recommended) + - Docker (single container - v1-latest-single, deprecated) + - Local development (make start-all) + - Other (please specify below) + validations: + required: true + + - type: textarea + id: issue-description + attributes: + label: What is the issue? + description: Describe the installation or setup problem you're experiencing + placeholder: | + Example: "I can't connect to the database" or "The container won't start" or "Getting 404 errors when accessing the UI" + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs + description: | + Please provide relevant logs. **This is very important for diagnosing issues!** + + **How to get logs:** + - Docker single container: `docker logs open-notebook` + - Docker Compose: `docker compose logs -f` + - Specific service: `docker compose logs -f open_notebook` + placeholder: | + Paste your logs here. Include the full error message and stack trace if available. + render: shell + validations: + required: false + + - type: textarea + id: docker-compose + attributes: + label: Docker Compose Configuration + description: | + If using Docker Compose, please paste your `docker-compose.yml` file here. + + **โš ๏ธ IMPORTANT: Redact any sensitive information (API keys, passwords, etc.)** + placeholder: | + services: + open_notebook: + image: lfnovo/open_notebook:v1-latest-single + ports: + - "8502:8502" + - "5055:5055" + environment: + - OPENAI_API_KEY=sk-***REDACTED*** + ... + render: yaml + validations: + required: false + + - type: textarea + id: env-file + attributes: + label: Environment File + description: | + If using an `.env` or `docker.env` file, please paste it here. + + **โš ๏ธ IMPORTANT: REDACT ALL API KEYS AND PASSWORDS!** + placeholder: | + SURREAL_URL=ws://surrealdb:8000/rpc + SURREAL_USER=root + SURREAL_PASSWORD=***REDACTED*** + OPENAI_API_KEY=sk-***REDACTED*** + ANTHROPIC_API_KEY=sk-ant-***REDACTED*** + render: shell + validations: + required: false + + - type: textarea + id: system-info + attributes: + label: System Information + description: Tell us about your setup + placeholder: | + - Operating System: Ubuntu 22.04 / Windows 11 / macOS 14 + - Docker version: `docker --version` + - Docker Compose version: `docker compose version` + - Architecture: amd64 / arm64 (Apple Silicon) + - Available disk space: `df -h` + - Available memory: `free -h` (Linux) or Activity Monitor (Mac) + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Any other information that might be helpful + placeholder: | + - Are you behind a corporate proxy or firewall? + - Are you using a VPN? + - Have you made any custom modifications? + - Did this work before and suddenly break? + validations: + required: false + + - type: checkboxes + id: checklist + attributes: + label: Pre-submission Checklist + description: Please confirm you've tried these steps + options: + - label: I tried the [Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) + required: false + - label: I read the relevant documentation ([Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md) or [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md)) + required: false + - label: I searched existing issues to see if this was already reported + required: true + - label: I redacted all sensitive information (API keys, passwords, etc.) + required: true diff --git a/.github/RELEASE_PROCESS.md b/.github/RELEASE_PROCESS.md new file mode 100644 index 0000000..2be6e30 --- /dev/null +++ b/.github/RELEASE_PROCESS.md @@ -0,0 +1,174 @@ +# Release Process + +Open Notebook uses a flow-driven release process. Work moves from `ready` +issues into pull requests, pull requests merge to `main`, and maintainers cut a +version when the branch has enough validated change to ship. + +This document covers both the **mechanics** (how to cut, build and publish) and +the **confidence process** (how we know a release is good before users get it). +It was redesigned during the v1.11.0 release ([ADR-005](../docs/7-DEVELOPMENT/decisions/ADR-005-release-confidence-process.md)). + +## Release Model + +- Patch releases ship backwards-compatible fixes. +- Minor releases ship backwards-compatible features and improvements. +- Major releases are planned with a milestone when they include breaking + changes or migrations that need user coordination. +- Use the `in-dev-build` label for changes available in development images and + `released` for shipped work. (The `released` label was recreated during + v1.12.0 โ€” it had been dropped from the curated label taxonomy while this + document still required it. If a label this document references is missing, + recreate it rather than skipping the step.) + +## Normal Flow + +1. Triage issues into `ready` once the scope and design are clear. +2. Implement each change in a focused pull request linked to the approved issue. +3. Merge the pull request after review and required checks pass. +4. Let the development build publish the `v1-dev` image from `main`. +5. Cut a stable release when `main` has a coherent set of changes ready for + users โ€” following the confidence process below. + +## The Confidence Process + +Releases keep getting bigger; ad-hoc verification does not scale. Before +cutting, run this sequence: + +### 0. Changelog audit + +Diff `git log ..main` against the `[Unreleased]` section of the +CHANGELOG. Every merged PR must be represented (entries reference the issue +number when one exists, the PR number otherwise). The changelog is the input +for both the test plan and the release notes โ€” close the gaps first, via PR. + +### 1. Risk-based test matrix + +Build a matrix from the actual release diff: each change โ†’ what it can break +and for whom โ†’ which bucket tests it. Pay special attention to +**"does the protection break legitimate use?"** for security changes (e.g. an +SSRF guard vs. self-hosted Ollama on localhost) and to anything a reverse +proxy, an upgrade, or a big upload would exercise. + +Buckets: + +- **A โ€” automated, high confidence, run now**: full backend suite, frontend + lint/tests/production build, the smoke-e2e agent (full API happy path + UI + verification), targeted regression probes for the release's specific risks, + dependency audit. +- **B โ€” automatable with investment**: decide per item whether to build the + muscle now (it compounds: the image gate below started as a bucket-B item) + or verify manually this once. +- **C โ€” needs the release owner**: real provider credentials, real TTS podcast + generation, visual/UX judgment, and the final check of the pushed image. + +### 2. The image gate โ€” test the artifact, not the repo + +A green suite on `main` is not a working image. Run: + +```bash +make docker-build-local # builds + local tags +make release-test TAG= OLD_TAG= +``` + +This runs two scenarios against real containers (`scripts/release-test/`): + +- **Fresh install**: empty DB โ†’ migrations on boot โ†’ in-image worker processes + a source โ†’ API/frontend/nginx-proxied checks. +- **Upgrade**: boot the *published* previous image, seed data, swap to the new + image on the same volume โ†’ migrations apply, data survives. + +Caveat: `docker-build-local` tags with the current `pyproject.toml` version โ€” +`docker pull` the genuine previous tag before the upgrade test so you are not +comparing the new build against itself. + +### 3. Fix loop with a re-test policy + +Findings become focused PRs through the normal review flow. After each merge: +the cheap suite always re-runs; smoke/image gates re-run only if the fix +touches what they cover; manual verification is not repeated unless the fix +touches what was manually verified. Pre-existing bugs found along the way that +are not release regressions become backlog issues instead of scope creep. + +## Cutting A Stable Release + +1. Confirm `main` is green and the confidence process above has run. +2. Open the **cut PR**: bump `pyproject.toml`, date the `[Unreleased]` section + as `[] - `. +3. After merge: `make tag`. +4. Build and push version images **via CI** (it holds the registry + credentials): trigger the *Build and Release* workflow with + `push_latest=false`. Local `make docker-push` also works but requires + `docker login` on both registries. +5. **Verify the pushed image** (bucket C, final gate): run it locally with + `make release-stack TAG= [DUMP=]` โ€” a browsable, + isolated stack, optionally with a copy of real data โ€” and walk the core + flows in the browser. +6. Publish the GitHub release. A non-prerelease publication triggers the + workflow again and pushes the `v1-latest` tags automatically. +7. Verify the `v1-latest` manifests on Docker Hub and GHCR (both arches, both + variants), and mark shipped issues with `released`. + +## Communication + +Release notes follow this structure (see v1.11.0 as the reference): + +1. One-line verdict + upgrade recommendation. +2. Sections: Security, Features, Performance, Notable fixes. +3. **Behavior changes for self-hosters** โ€” anything that can require a config + tweak on upgrade gets an explicit callout. +4. **Thanks** โ€” credit every contributor by handle with what they shipped + (collect via `git log ..` + `gh pr view` for handles), plus + the issue reporters collectively. Never skip this section. + +Announce on Discord after `v1-latest` is live. + +## Retro + +Close every release by asking: what should improve in this process? Apply the +accepted improvements immediately โ€” update this document, the scripts under +`scripts/release-test/`, and the decision log while the context is fresh. + +## Docker Image Publishing (reference) + +| Command | What it does | Updates latest? | +|---------|--------------|-----------------| +| `make docker-build-local` | Build for current platform only (tags `` + `local`) | No registry push | +| CI *Build and Release* (`push_latest=false`) | Push version tags via CI credentials | โŒ No | +| GitHub release published (non-prerelease) | CI pushes version + `v1-latest` | โœ… Yes | +| `make docker-push` / `docker-push-latest` | Local equivalents (need `docker login`) | โŒ / โœ… | +| `make tag` | Create and push a git tag matching `pyproject.toml` | โ€” | + +- **Platforms:** `linux/amd64`, `linux/arm64` +- **Registries:** Docker Hub + GitHub Container Registry +- **Image variants:** regular + single-container (`-single`). Both are built + from the same `Dockerfile`: regular is the default/`runtime` target, single + is `--target single` +- **Version source:** `pyproject.toml` +- Build issues: `docker builder prune`, then `make docker-buildx-reset` + +## Known Gotchas + +- **RC stack on non-default ports needs `API_URL`** or the browser talks to + `host:5055` โ€” on a dev machine that is the development API (data crossover). + `rc-stack.sh` sets it; remember this for any custom setup. +- **Containerized app + host services**: credentials pointing at local + services (Ollama, LM Studio) need `http://host.docker.internal:`. +- **SurrealDB import**: `OVERWRITE` goes after the type keyword + (`DEFINE FIELD OVERWRITE โ€ฆ`), and the exporter can leak a log line into the + dump โ€” `rc-stack.sh` handles both. +- **Multiple local SurrealDB instances**: check which one the dev `.env` + actually points at (`SURREAL_URL`) before exporting data. +- **Dev-machine ports may belong to other projects**: check who owns + 3000/5055/8000 (`lsof -nP -iTCP: -sTCP:LISTEN` + the process cwd) + before starting or killing anything. The frontend runs fine on an alternate + port for smoke testing (`PORT=3001 npm run dev`) โ€” pass the URL to the + smoke agent. +- **Manual error-path checklist items must be validated against the code + first**: some "missing configuration" scenarios are deliberate fallbacks, + not errors (e.g. transformation and tools defaults fall back to the chat + default). Confirm the expected behavior in the provisioning code before + putting "should show an error" on the bucket-C checklist. +- **The test suite runs against the live dev database** when a developer + `.env` is loaded. During bucket A, snapshot record counts per table before + and after the suite (e.g. credentials count) โ€” a diff means a test is + leaking writes (this caught 48 leaked `Test` credentials in v1.12.0). diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..5de0aba --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,109 @@ +## Description + + + +## Related Issue + + + +Fixes # + +## Type of Change + + + +- [ ] Bug fix (non-breaking change that fixes an issue) +- [ ] New feature (non-breaking change that adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Code refactoring (no functional changes) +- [ ] Performance improvement +- [ ] Test coverage improvement + +## How Has This Been Tested? + + + +- [ ] Tested locally with Docker +- [ ] Tested locally with development setup +- [ ] Added new unit tests +- [ ] Existing tests pass (`uv run pytest`) +- [ ] Manual testing performed (describe below) + +**Test Details:** + + +## Design Alignment + + + +**Which design principles does this PR support?** (See [VISION.md](https://github.com/lfnovo/open-notebook/blob/main/VISION.md)) + +- [ ] Privacy First +- [ ] Simplicity Over Features +- [ ] API-First Architecture +- [ ] Multi-Provider Flexibility +- [ ] Extensibility Through Standards +- [ ] Async-First for Performance + +**Explanation:** + + +## Checklist + + + +### Code Quality +- [ ] My code follows PEP 8 style guidelines (Python) +- [ ] My code follows TypeScript best practices (Frontend) +- [ ] I have added type hints to my code (Python) +- [ ] I have added JSDoc comments where appropriate (TypeScript) +- [ ] I have performed a self-review of my code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] My changes generate no new warnings or errors + +### Testing +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes +- [ ] I ran linting: `make ruff` or `ruff check . --fix` +- [ ] I ran type checking: `make lint` or `uv run python -m mypy .` + +### Documentation +- [ ] I have updated the relevant documentation in `/docs` (if applicable) +- [ ] I have added/updated docstrings for new/modified functions +- [ ] I have updated the API documentation (if API changes were made) +- [ ] I have added comments to complex logic + +### Database Changes +- [ ] I have created migration scripts for any database schema changes (in `/migrations`) +- [ ] Migration includes both up and down scripts +- [ ] Migration has been tested locally + +### Breaking Changes +- [ ] This PR includes breaking changes +- [ ] I have documented the migration path for users +- [ ] I have updated MIGRATION.md (if applicable) + +## Screenshots (if applicable) + + + +## Additional Context + + + +## Pre-Submission Verification + +Before submitting, please verify: + +- [ ] I have read [CONTRIBUTING.md](https://github.com/lfnovo/open-notebook/blob/main/docs/7-DEVELOPMENT/contributing.md) +- [ ] I have read [VISION.md](https://github.com/lfnovo/open-notebook/blob/main/VISION.md) +- [ ] This PR addresses an approved issue assigned to me, **or** it's a small obvious fix (typo, docs, tiny bug) that doesn't need one โ€” for anything bigger without an issue, mark this PR as draft and open the issue (triage takes 1โ€“2 days) +- [ ] I have not included unrelated changes in this PR +- [ ] My PR title follows conventional commits format (e.g., "feat: add user authentication") + +--- + +**Thank you for contributing to Open Notebook!** ๐ŸŽ‰ diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 0000000..c31e9cf --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,300 @@ +name: Build and Release + +on: + workflow_dispatch: + inputs: + push_latest: + description: 'Also push v1-latest tags' + required: true + default: false + type: boolean + release: + types: [published] + +permissions: + contents: read + packages: write + +env: + GHCR_IMAGE: ghcr.io/lfnovo/open-notebook + DOCKERHUB_IMAGE: lfnovo/open_notebook + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Extract version from pyproject.toml + id: version + run: | + VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Check for Docker Hub credentials + id: check + env: + SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [[ -n ""$SECRET_DOCKER_USERNAME"" && -n ""$SECRET_DOCKER_PASSWORD"" ]]; then + echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT + echo "Docker Hub credentials available" + else + echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT + echo "Docker Hub credentials not available - will only push to GHCR" + fi + + build-regular: + needs: extract-version + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v5 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-regular-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-regular- + + - name: Prepare Docker tags for regular build + id: tags-regular + env: + ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} + GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} + ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} + run: | + TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}" + + # Determine if we should push latest tags + PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" + if [[ -z "$PUSH_LATEST" ]]; then + PUSH_LATEST="false" + fi + + # Add GHCR latest tag if requested or for non-prerelease releases + if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then + TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest" + fi + + # Add Docker Hub tags if credentials available + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}" + + if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then + TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest" + fi + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "Generated tags: ${TAGS}" + + - name: Build and push regular image + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile + target: runtime + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags-regular.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + build-single: + needs: extract-version + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v5 + with: + path: /tmp/.buildx-cache-single + key: ${{ runner.os }}-buildx-single-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-single- + + - name: Prepare Docker tags for single build + id: tags-single + env: + ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} + GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} + ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} + run: | + TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single" + + # Determine if we should push latest tags + PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" + if [[ -z "$PUSH_LATEST" ]]; then + PUSH_LATEST="false" + fi + + # Add GHCR latest tag if requested or for non-prerelease releases + if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then + TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest-single" + fi + + # Add Docker Hub tags if credentials available + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single" + + if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then + TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest-single" + fi + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "Generated tags: ${TAGS}" + + - name: Build and push single-container image + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile + target: single + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags-single.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache-single + cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-single + mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single + + summary: + needs: [extract-version, build-regular, build-single] + runs-on: ubuntu-latest + if: always() + steps: + - name: Build Summary + env: + GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_: ${{ github.event.inputs.push_latest || 'false' }} + ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} + ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} + GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} + run: | + echo "## Build Summary" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "**Push v1-Latest:** "$GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_"" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Registries:" >> $GITHUB_STEP_SUMMARY + echo "โœ… **GHCR:** \`"$ENV_GHCR_IMAGE"\`" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + echo "โœ… **Docker Hub:** \`"$ENV_DOCKERHUB_IMAGE"\`" >> $GITHUB_STEP_SUMMARY + else + echo "โญ๏ธ **Docker Hub:** Skipped (credentials not configured)" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Images Built:" >> $GITHUB_STEP_SUMMARY + + if [[ "${{ needs.build-regular.result }}" == "success" ]]; then + echo "โœ… **Regular (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY + if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then + echo "โœ… **Regular v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + echo "โœ… **Regular (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY + if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then + echo "โœ… **Regular v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY + fi + fi + elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then + echo "โญ๏ธ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [[ "${{ needs.build-single.result }}" == "success" ]]; then + echo "โœ… **Single (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY + if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then + echo "โœ… **Single v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + echo "โœ… **Single (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY + if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then + echo "โœ… **Single v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY + fi + fi + elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then + echo "โญ๏ธ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Single:** Failed" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Platforms:" >> $GITHUB_STEP_SUMMARY + echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY + echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml new file mode 100644 index 0000000..a751065 --- /dev/null +++ b/.github/workflows/build-dev.yml @@ -0,0 +1,275 @@ +name: Development Build + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + paths-ignore: + - '**.md' + - 'docs/**' + - 'notebooks/**' + - '.github/workflows/claude*.yml' + workflow_dispatch: + inputs: + platform: + description: 'Platform to build' + required: true + default: 'linux/amd64' + type: choice + options: + - linux/amd64 + - linux/arm64 + - linux/amd64,linux/arm64 + +permissions: + contents: read + packages: write + +env: + GHCR_IMAGE: ghcr.io/lfnovo/open-notebook + DOCKERHUB_IMAGE: lfnovo/open_notebook + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }} + is_push_to_main: ${{ steps.check.outputs.is_push_to_main }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Extract version from pyproject.toml + id: version + run: | + VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Check environment + id: check + env: + SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + # Check for Docker Hub credentials + if [[ -n "$SECRET_DOCKER_USERNAME" && -n "$SECRET_DOCKER_PASSWORD" ]]; then + echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT + echo "Docker Hub credentials available" + else + echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT + echo "Docker Hub credentials not available" + fi + + # Check if this is a push to main (not a PR) + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "is_push_to_main=true" >> $GITHUB_OUTPUT + echo "This is a push to main - will publish v1-dev tags" + else + echo "is_push_to_main=false" >> $GITHUB_OUTPUT + echo "This is a PR or manual run - test build only" + fi + + build-regular: + needs: extract-version + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Free up disk space + if: needs.extract-version.outputs.is_push_to_main == 'true' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + if: needs.extract-version.outputs.is_push_to_main == 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: needs.extract-version.outputs.is_push_to_main == 'true' && needs.extract-version.outputs.has_dockerhub_secrets == 'true' + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v5 + with: + path: /tmp/.buildx-cache-dev + key: ${{ runner.os }}-buildx-dev-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-dev- + + - name: Prepare Docker tags + id: tags + run: | + if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then + # Push to main: build and push v1-dev tags + TAGS="${{ env.GHCR_IMAGE }}:v1-dev" + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev" + fi + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "push=true" >> $GITHUB_OUTPUT + echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT + else + # PR or manual: test build only + echo "tags=${{ env.DOCKERHUB_IMAGE }}:${{ needs.extract-version.outputs.version }}-dev" >> $GITHUB_OUTPUT + echo "push=false" >> $GITHUB_OUTPUT + echo "platforms=${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_OUTPUT + fi + + - name: Build and push regular image + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile + target: runtime + platforms: ${{ steps.tags.outputs.platforms }} + push: ${{ steps.tags.outputs.push }} + tags: ${{ steps.tags.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache-dev + cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-dev + mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev + + build-single: + needs: extract-version + # Only build single image on push to main + if: needs.extract-version.outputs.is_push_to_main == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v5 + with: + path: /tmp/.buildx-cache-dev-single + key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-dev-single- + + - name: Prepare Docker tags + id: tags + run: | + TAGS="${{ env.GHCR_IMAGE }}:v1-dev-single" + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev-single" + fi + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + + - name: Build and push single-container image + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile + target: single + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache-dev-single + cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-dev-single + mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single + + summary: + needs: [extract-version, build-regular, build-single] + runs-on: ubuntu-latest + if: always() + steps: + - name: Development Build Summary + run: | + echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "**Event:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "**Push to Main:** ${{ needs.extract-version.outputs.is_push_to_main }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then + echo "### Published Tags:" >> $GITHUB_STEP_SUMMARY + + if [[ "${{ needs.build-regular.result }}" == "success" ]]; then + echo "โœ… **Regular:** \`${{ env.GHCR_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + echo "โœ… **Regular (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY + fi + else + echo "โŒ **Regular:** Build failed" >> $GITHUB_STEP_SUMMARY + fi + + if [[ "${{ needs.build-single.result }}" == "success" ]]; then + echo "โœ… **Single:** \`${{ env.GHCR_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then + echo "โœ… **Single (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY + fi + elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then + echo "โญ๏ธ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Single:** Build failed" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Platforms:" >> $GITHUB_STEP_SUMMARY + echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY + echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY + else + echo "### Test Build Results:" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.build-regular.result }}" == "success" ]]; then + echo "โœ… **Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Notes:" >> $GITHUB_STEP_SUMMARY + echo "- This is a test build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY + echo "- Merge to main to publish \`v1-dev\` tags" >> $GITHUB_STEP_SUMMARY + echo "- For stable releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/docs-links.yml b/.github/workflows/docs-links.yml new file mode 100644 index 0000000..3b95cf1 --- /dev/null +++ b/.github/workflows/docs-links.yml @@ -0,0 +1,16 @@ +name: Docs Link Check + +on: + pull_request: + paths: + - "**/*.md" + - "scripts/check_md_links.py" + - ".github/workflows/docs-links.yml" + +jobs: + check-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check relative markdown links + run: python3 scripts/check_md_links.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1195d8a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,159 @@ +name: Tests + +on: + pull_request: + branches: [main] + push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/workflows/claude*.yml' + +permissions: + contents: read + +jobs: + backend: + name: Backend Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync + + - name: Run tests with coverage + run: uv run pytest tests/ -v --cov=open_notebook --cov=api --cov-report=term-missing --cov-report=xml + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: backend-coverage + path: coverage.xml + + backend-lint: + name: Backend Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync + + - name: Run ruff + run: uv run ruff check . + + backend-typecheck: + name: Backend Typecheck + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync + + - name: Run mypy + run: uv run python -m mypy . + + frontend: + name: Frontend Tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run tests with coverage + run: npm run test:coverage + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: frontend-coverage + path: frontend/coverage/ + + frontend-lint: + name: Frontend Lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run ESLint + run: npm run lint + + frontend-build: + name: Frontend Build + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d10840f --- /dev/null +++ b/.gitignore @@ -0,0 +1,153 @@ +.env +prompts/patterns/user/ +/notebooks/ +data/ +.uploads/ +sqlite-db/ +surreal-data/ +docker.env +notebook_data/ +# Python-specific +*.py[cod] +__pycache__/ +*.so +todo.md +temp/ +google-credentials.json +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +/lib/ +/lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# PyCharm +.idea/ + +# VS Code +.vscode/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# macOS +.DS_Store + +# Windows +Thumbs.db +ehthumbs.db +desktop.ini + +# Linux +*~ + +# Log files +*.log + +# Database files +*.db +*.sqlite3 + +.quarentena + +claude-logs/ +.claude/sessions +**/claude-logs + + +docs/custom_gpt +doc_exports/ + +specs/ +.claude +.sisyphus + +.playwright-mcp/ + + + +*.local.yml +**/*.local.md +.harness/ + +.mcp.json + +# Local Docker Compose override (auto-merged by `docker compose up`). +# Use it for host-specific tweaks like re-exposing the SurrealDB port; +# see docker-compose.override.yml.example. +docker-compose.override.yml + +# Local QA/debug artifacts +screenshot-*.png +history.txt diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/.worktreeinclude b/.worktreeinclude new file mode 100644 index 0000000..56e4e87 --- /dev/null +++ b/.worktreeinclude @@ -0,0 +1,5 @@ +.env +.env.local +.env.* +**/.claude/settings.local.json +CLAUDE.local.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..11c8152 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,45 @@ +# Open Notebook โ€” Agent Rules + +Open Notebook is an open-source, privacy-focused alternative to Google's Notebook LM: an AI-powered research assistant with multi-provider AI support, fully self-hostable. + +This file holds the project-wide rules every coding session needs. Component rules: [open_notebook/AGENTS.md](open_notebook/AGENTS.md) (backend โ€” also covers `api/`, `commands/`, `prompts/`) and [frontend/AGENTS.md](frontend/AGENTS.md). Knowledge lives in the docs (see [Where to look](#where-to-look)) โ€” read it on demand instead of guessing. + +## Stack, ports, startup order + +Three tiers: Next.js frontend (3000) โ†’ FastAPI (5055) โ†’ SurrealDB (8000). + +Start in this order โ€” each tier depends on the one below: + +1. `make database` โ€” SurrealDB (API fails without it) +2. `make api` โ€” FastAPI; **schema migrations run automatically on startup** (check logs) +3. `make worker-start` โ€” surreal-commands worker. **Required**: podcasts, embeddings and source processing are async jobs that silently queue forever without it +4. `make frontend` โ€” UI (depends on the API for all data) + +Or all at once: `make start-all` (status: `make status`, stop: `make stop-all`). + +## Commands + +- Tests: `uv run pytest tests/` +- Python lint/typecheck: `ruff check . --fix` ยท `uv run python -m mypy .` +- Frontend (inside `frontend/`): `npm run lint` ยท `npm run test` ยท `npm run build` +- Docker release: `make docker-release` (see `.github/RELEASE_PROCESS.md`) + +## Hard rules + +- **Async-first**: every DB query, graph invocation and AI call is `await`-ed. No sync DB access. +- **Never commit secrets.** Credentials are encrypted at rest and require `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set. +- CORS is wide-open and auth is a simple password middleware โ€” **dev defaults, not production hardening**. Don't build features that assume otherwise. +- Product direction questions (does this feature fit?) โ†’ [VISION.md](VISION.md). Past decisions ("why is it like this?") โ†’ [docs/7-DEVELOPMENT/decisions/](docs/7-DEVELOPMENT/decisions/). Structural decisions made while coding should produce a new decision record there. + +## Where to look + +| Need | Location | +|---|---| +| Architecture (3 tiers, workflows, data model) | [docs/7-DEVELOPMENT/architecture.md](docs/7-DEVELOPMENT/architecture.md) | +| Step-by-step recipes (add endpoint, migration, i18nโ€ฆ) | [docs/7-DEVELOPMENT/change-playbooks.md](docs/7-DEVELOPMENT/change-playbooks.md) | +| Dev environment setup | [docs/7-DEVELOPMENT/development-setup.md](docs/7-DEVELOPMENT/development-setup.md) | +| Code standards & testing | [docs/7-DEVELOPMENT/code-standards.md](docs/7-DEVELOPMENT/code-standards.md) ยท [testing.md](docs/7-DEVELOPMENT/testing.md) | +| Product identity & current posture | [VISION.md](VISION.md) | +| Decision log (ADRs/PDRs) | [docs/7-DEVELOPMENT/decisions/](docs/7-DEVELOPMENT/decisions/) | +| Contribution process (issue-first, PRs) | [docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md) | +| User/operator docs (install, configure, troubleshoot) | [docs/](docs/index.md) | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a133a1a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,540 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- **OCR toggle** in Settings โ†’ Content Processing: a new "Enable OCR" checkbox controls whether the Docling engine runs OCR on scanned PDFs and images. It's on by default (matching content-core's behavior); turn it off to speed up processing of text-native documents. The setting is passed to content-core's `docling_ocr` config, and the label/help are translated across all 14 locales (#1104) +- **Crawl4AI** is now selectable as a URL processing engine in Settings โ†’ Content Processing, alongside Firecrawl, Jina and Simple. It renders JavaScript-heavy pages locally with no API key, or offloads to a Crawl4AI server when `CRAWL4AI_API_URL` is set. The Crawl4AI runtime (and its Chromium browser) is bundled into the Docker image, so local mode works out of the box (~300 MB larger image; no torch/CUDA). As part of this, the persisted document/URL engine choices now actually take effect: the source-processing graph reads the saved Content Settings and passes them to content-core (previously it always ran with hard-coded `auto` engines and silently ignored the selection). New engine label added across all 14 locales (#432) + +### Changed +- Upgraded the content extraction dependency from content-core 1.14.x to 2.x (2.0.4). The source-processing graph was adapted to content-core's new keyword-only `extract_content()` API: engine/model overrides now travel through a `ContentCoreConfig` object instead of the input dict, and the extraction result (`ExtractionOutput`) no longer echoes the source `url`/`file_path` back, so those are carried from the request state into the saved source asset. Because content-core 2.x no longer deletes the uploaded file after extraction, the graph now honors the `delete_source` flag itself. Transitively this replaces the AGPL-licensed PyMuPDF with MIT-licensed pdfplumber for PDF extraction and drops moviepy in favor of direct ffmpeg calls (which fixes audio extraction from MP3 files carrying chapter metadata). No user-facing configuration changes in this step โ€” document and URL engines stay on their `auto` defaults; new engine/OCR options are tracked separately under #939 (#1103) +- Podcast episode audio paths are now stored relative to the podcasts folder (`episodes//audio/.mp3`) instead of as absolute filesystem paths, validated at write time so the database can never hold an absolute or root-escaping value, and resolved + containment-checked through a single shared helper instead of per-endpoint guards. Migration 21 converts existing rows written under the known roots (plain `file:///` URIs, `/app/data/podcasts/`, `/data/podcasts/`, `./data/podcasts/`, `data/podcasts/`); previously generated episodes now survive a `DATA_FOLDER` relocation. Rows under any other root (e.g. a source checkout at a custom absolute path) or in exotic legacy forms (percent-encoded or host-qualified `file://` URIs) are left untouched and treated as legacy-invalid โ€” the same 403/audio-unavailable handling out-of-root rows already had; regenerate the episode to restore playback. Podcast jobs whose audio combination fails (podcast-creator reports an in-band `ERROR:` value) now fail with the real ffmpeg/clip error instead of reporting success for an episode with no playable audio (#1030) +- The source detail view (dialog and full page) now fetches through the shared `useSource` React Query hook instead of a hand-rolled fetch, matching the insight/note dialogs: caching and the never-retry-404 policy come from React Query, title edits and deletes go through the shared mutation hooks (so source lists refresh and a deleted source can't be served from the cache), and the `key={sourceId}` remount workaround on the parents was removed โ€” the component resets its own per-source UI state (#1106) +- The settings frontend now fetches the provider list from `GET /api/providers` (cached for the session) instead of keeping its own hardcoded provider tables, so adding a provider to the backend registry needs zero frontend edits: unknown modalities render with a fallback icon, the backend registry owns the display order, and the regex-based frontend/backend sync test was removed along with the duplicated tables in `frontend/src/lib/providers.tsx` (#1082) + +### Removed +- The legacy provider/model string fields on podcast profiles (`outline_provider`, `outline_model`, `transcript_provider`, `transcript_model` on episode profiles; `tts_provider`, `tts_model` on speaker profiles) are gone from the database, the API and the UI โ€” the app has ignored them since the Model registry references landed in v1.11. Migration 22 first best-effort maps any profile whose `outline_llm`/`transcript_llm`/`voice_model` reference is still empty to an existing model record (matching provider + name + type; no auto-creation, since a migration must not touch credentials), then drops the six columns; the startup data migration that used to retry this mapping on every boot (`open_notebook/podcasts/migration.py`) was deleted. Accepted trade-off: profiles whose mapping never converged (e.g. the provider credential was never configured) lose the legacy strings and stay unresolved โ€” they were already non-functional and the UI already flags them as needing model selection, so you just re-pick the models in the profile form once (#1107) + +### Fixed +- Uploading a file content-core can't extract now fails immediately at ingestion with a clear `415 Unsupported Media Type` error that names the detected MIME type, instead of enqueueing a background job that retried up to 15 times over ~1 hour before surfacing a generic "Failed" with no actionable detail. The pre-flight uses content-core 2.x's header-only `check_file_support()` โ€” the same routing real extraction uses, so the verdict can't disagree with what would happen downstream โ€” and the source-retry endpoint is guarded the same way; unexpected check errors (e.g. a file removed before a retry) fall through to normal extraction rather than becoming a hard rejection (#975) +- Podcast episode cards no longer show "โ€” / โ€”" for the outline, transcript and speaker model rows on new episodes: the API now resolves the snapshot's model references (`outline_llm`/`transcript_llm`/`voice_model`) to provider/name display fields at serialization time โ€” batched into a single query per request, so listing episodes never does a per-row model lookup โ€” and the card falls back to the legacy snapshot strings for old episodes and degrades to "โ€”" when a referenced model was deleted (#1114) +- Renaming a speaker profile no longer breaks the episode profiles that use it: `episode_profile.speaker_config` now stores a `record` reference instead of the profile name (migration 20 converts existing rows; references whose speaker profile no longer exists at migration time become null, and any reference that later stops resolving is treated as "needs setup" โ€” the UI asks you to pick a speaker again). The `POST /api/podcasts/generate` contract is unchanged โ€” it still accepts the speaker profile by name and resolves it at the API boundary (#630) +- Clicking a chat/Ask citation that points at a deleted source, insight, or note now shows a shared, friendly "this content no longer exists" state in all three dialogs (instead of a raw error, a blank dialog, or an empty editable note editor), 404 lookups are no longer retried, and non-404 failures show a distinct "unable to load" message (#455) +- Source insights now get `created`/`updated` timestamps stamped at creation (migration 19 mirrors the defaults used by the other tables), and the insights API returns `null` โ€” instead of the literal string `"None"` โ€” for legacy insights that predate the migration (#1045) +- `uv sync` alone now provides the full dev toolchain: the legacy `[project.optional-dependencies].dev` list was merged into `[dependency-groups].dev` (mypy included โ€” the documented `uv run python -m mypy .` previously failed on a fresh clone), Jupyter-only packages moved to a separate `notebooks` group, and the CI typecheck job no longer needs `--extra dev` (#1101) +- Optional model defaults (transformation, tools, large context, TTS, STT) can now be cleared: `PUT /api/models/defaults` honors explicit `null` (field absent still means "keep"; chat and embedding defaults reject `null`), and the default-model selects offer a "None" / "Use fallback (chat default)" option for the optional defaults (#1091) +- `docker-compose.yml` now uses the YAML list (exec) form for the SurrealDB `command`, so `SURREAL_USER` / `SURREAL_PASSWORD` values containing spaces are passed as single arguments instead of being split; the mirrored snippets in the installation docs and README (which had drifted โ€” no credential interpolation, SurrealDB port published on all interfaces) are back in sync with the shipped file (#1093) +- zh-CN and zh-TW podcast toast descriptions (speaker/episode profile created/updated/deleted/duplicated) now include the profile name via the `{{name}}` placeholder, matching the other 12 locales (#1084) +- Docker images now force the Next.js frontend to bind to `0.0.0.0` in the supervisord command itself, so container runtimes that inject `HOSTNAME` (e.g. Podman pods, where it resolves to `127.0.1.1`) can no longer make the UI unreachable. The `HOSTNAME` variable is no longer honored as a frontend bind override โ€” set the new `FRONTEND_BIND_HOST` variable instead (#994) + +## [1.12.0] - 2026-07-12 + +### Fixed +- Setup snippets no longer teach publishing SurrealDB on `0.0.0.0` โ€” the compose and `docker run` examples across the README, quick starts, installation, configuration and development docs, and the `examples/docker-compose-*.yml` files now bind port 8000 to `127.0.0.1` (matching the shipped `docker-compose.yml`), with docs pointing to `docker-compose.override.yml.example` for opt-in remote access behind a firewall or SSH tunnel; the override example itself gained the `!override` tag it needs to actually replace the base port binding instead of colliding with it (#1034) + +### Added +- Docs: cubic platform mechanics recorded as comments in `cubic.yaml` (agent limits, config precedence, memory/learning) and a "Merging PR Batches" playbook added to the maintainer guide (squash policy, CHANGELOG conflict resolution, fork rebases, competing-PR checks) (#1086) +- New `GET /api/providers` endpoint returning provider metadata from the registry (name, display name, modalities, docs URL, whether it is configured via environment variables), so clients can enumerate supported providers instead of hardcoding them (#1075) +- Release confidence process, documented and executable: `.github/RELEASE_PROCESS.md` now covers the risk-based test matrix (buckets A/B/C), the Docker image gate, the fix-loop re-test policy and the communication/credits/retro structure, backed by a new decision record (ADR-005) and versioned tooling under `scripts/release-test/` โ€” `make release-test TAG= OLD_TAG=` runs fresh-install + upgrade scenarios against real images, and `make release-stack TAG= [DUMP=]` boots a browsable, isolated release-candidate stack (optionally with a copy of dev data) for manual verification (#1052) +- CI now gates every PR on `ruff check` (backend lint), `npm run lint` (frontend ESLint) and `npm run build` (frontend production build), in addition to the existing test suites (#1068) +- CI now also gates every PR on `mypy` (backend typecheck): the repo-wide baseline went from 197 errors to 0 (enabling the pydantic mypy plugin resolved most of them; the rest got real annotations), so new type errors are blocked from here on. The `ignore_errors` burn-down also started: `open_notebook.graphs.transformation`, `open_notebook.graphs.ask` and `api.routers.models` are now type-checked (plus two stale entries for deleted modules removed); only `open_notebook.domain.notebook` remains exempt pending the surreal-basics migration (#1076) + +### Changed +- cubic AI review now skips `CHANGELOG.md`, `uv.lock` and `frontend/package-lock.json` (no reviewable logic; preserves the monthly reviewed-line quota) (#1080) +- Context building consolidated into a single implementation (`open_notebook/utils/context_builder.py`): the copy-pasted source/note assembly loops behind `POST /api/chat/context` and the removed notebook-context endpoint, plus the 495-line generalized `ContextBuilder` class (whose only caller was the source-chat graph), are now two focused functions โ€” `build_notebook_context()` (backs `POST /api/chat/context`, unchanged request/response shapes and config semantics) and `build_source_context()` (backs the source-chat graph, same context shape and 50k-token budget). Pinned by new characterization tests โ€” no behavior change for the surviving paths (#1079) +- **Removed** `POST /api/notebooks/{notebook_id}/context`: it duplicated `POST /api/chat/context` (same assembly logic, slightly different response envelope) and had zero callers โ€” frontend, docs and tests only use `/api/chat/context`. If you called it programmatically, switch to `POST /api/chat/context` (body: `{notebook_id, context_config}`; response fields: `context.sources`/`context.notes`, `token_count`, `char_count`) (#1079) +- Backend provider metadata now lives in a single registry (`open_notebook/ai/provider_registry.py`): env var config, modalities, connection-test models, OpenAI-compatible discovery URLs and docs links are defined once per provider, and `PROVIDER_ENV_CONFIG`, `PROVIDER_MODALITIES`, `TEST_MODELS` and `OPENAI_COMPAT_PROVIDERS` are derived from it. Adding a provider drops from ~6 hand-synced dicts to the registry plus two manual copies (the `SupportedProvider` Literal and the frontend provider table), both enforced by tests (#1075) +- Frontend convention cleanup (no user-facing change): hook files unified to kebab-case (`useNotebookChat.ts`/`useSourceChat.ts` โ†’ `use-notebook-chat.ts`/`use-source-chat.ts`), `src/components/source/` merged into `src/components/sources/`, the localStorage auth-token parsing ritual extracted into a single `getAuthToken()` helper (`src/lib/auth-token.ts`), and non-streaming raw `fetch` calls routed through `apiClient` (podcast audio download, auth-status check). SSE/streaming paths and the login/checkAuth credential probes deliberately keep raw `fetch` (#1077) +- Pruned unused langchain packages: removed `langchain-community` and `langchain-deepseek` from the dependencies (nothing imports them โ€” DeepSeek and xAI route through esperanto's OpenAI-compatible path, which uses `langchain-openai`). The remaining `langchain-*` provider packages are documented as runtime requirements of esperanto's dynamic `to_langchain()` and the whole langchain/langgraph family now carries explicit upper bounds; `langchain-core` and `langchain-text-splitters` (both directly imported but previously only transitive) are now declared explicitly (#1073) +- The two Docker images (regular and single-container) are now built from a single multi-stage `Dockerfile` with shared stages โ€” regular is the default (`runtime`) target, single-container is `--target single` โ€” so deploy fixes (tiktoken pre-cache, env defaults, npm retry logic) no longer have to be applied twice. `Dockerfile.single` and `supervisord.single.conf` were removed; the single image appends a small `supervisord.surrealdb.conf` to the shared `supervisord.conf` at build time. Published image names and tags are unchanged (#1066) +- Model discovery is now table-driven: the eight providers with OpenAI-compatible `/models` endpoints (OpenAI, Groq, Mistral, DeepSeek, xAI, OpenRouter, DashScope, MiniMax) share one generic discovery function configured by `OPENAI_COMPAT_PROVIDERS`, replacing eight near-identical copies (provider-specific quirks like Mistral's capability flags and OpenRouter's descriptions are preserved as hooks) (#1070) +- Internal refactor: extracted the session/source verification, record-ID normalization, LangGraph message extraction and shared response models duplicated across the chat and source-chat routers into `api/routers/_chat_shared.py`, pinned by new characterization tests โ€” no behavior change (#1072) +- Internal refactor of the sources API router: extracted a shared `SourceResponse` builder (was hand-rolled 5ร—), a single upload-cleanup helper (was pasted 6ร—), split the 293-line create-source endpoint into validation + sync/async path functions, and unified the duplicated paginated list query. No behavior change; all security checks (atomic filename claim, path-traversal containment, SSRF/LFI guards) preserved verbatim (#1069) +- Re-enabled the ruff rules for unused imports (`F401`), unused local variables (`F841`) and bare `except:` (`E722`) that were ignored to silence legacy Streamlit-era noise, and cleaned up the remaining fallout (10 unused imports, 2 unused test variables; no bare excepts remained) (#1062) +- Internal refactor with no user-facing change: split the 1,441-line API Keys settings page into focused components under `frontend/src/components/settings/` and moved the provider config tables to `frontend/src/lib/providers.tsx`, deduplicating the default-model select in the process (#1065) +- Chat, source chat, Ask and transformation prompts now steer models to write math as `$$...$$` (display) / `$...$` (inline) so formulas render via KaTeX, reserving fenced `latex` code blocks for when the user explicitly asks for the LaTeX source (#1051) +- Frontend locale files are now type-checked at compile time: every non-en-US locale declares `satisfies TranslationShape` (derived from the en-US object), so a missing or extra i18n key fails `tsc` in the editor instead of only the runtime parity test. Also removed two unused frontend dependencies (`next-themes`, `@monaco-editor/react`) and fixed `frontend/AGENTS.md` drift (14 locales, not 7; dark mode is the hand-rolled zustand theme-store, not next-themes) (#1061) + +### Fixed +- Eight podcast toast descriptions (speaker/episode profile created/updated/deleted/duplicated) showed the literal `{name}` placeholder instead of the profile name: the locale strings used single braces (which i18next ignores) and the `t()` call sites passed no values. Placeholders normalized to `{{name}}` across all locales and the actual name is now passed in from the mutation response/variables (#1077) +- Typed domain errors now return their documented HTTP status codes instead of a generic 500: the API routers used to wrap endpoint bodies in a broad `except Exception` that swallowed the `open_notebook.exceptions` hierarchy before the global handlers could map it (`NotFoundError`โ†’404, `InvalidInputError`โ†’400, `ConfigurationError`โ†’422, `RateLimitError`โ†’429, `NetworkError`/`ExternalServiceError`โ†’502). All 18 affected routers now re-raise `HTTPException` and `OpenNotebookError` and only convert genuinely unexpected exceptions into sanitized 500s. Most visible changes: a missing/unconfigured model (`ConfigurationError`) now returns 422 with an actionable message instead of 500; getting or deleting a source-chat session whose session isn't related to the source returns 404 (was a 500 wrapping the inner 404); fetching a missing credential returns 404 (was 500) (#1078) +- Frontend translations now use i18next interpolation (`t('key', { count })`) instead of manual `.replace('{count}', ...)` string surgery across ~75 call sites โ€” locale placeholders changed from `{name}` to `{{name}}` in all 14 locales. This restores proper pluralization (e.g. "used by N episodes" now goes through i18next plural forms) and lets translators reorder placeholders freely (#1074) +- Podcast generation dialog: the token/char counter no longer fires a request storm on rapid checkbox toggling (debounced, with a stale-response guard so a slow response can't overwrite a fresher count) and the dialog now closes as soon as the episode-list refetch completes instead of after a fixed 500ms timer; the 983-line component was also split (content selection panel and selection helpers extracted, duplicated context-config logic deduplicated) with no behavior changes (#1067) +- Anthropic models are now discovered live from `GET https://api.anthropic.com/v1/models` (paginated) instead of a hardcoded claude-3-era list โ€” the code comment claiming "Anthropic doesn't have a model listing API" was wrong. A refreshed static list (current Claude 4.x/5 aliases) remains as a fallback when the API call fails, and the credential-based discovery path (`discover_with_config`) uses the same live-with-fallback logic (#1070) +- Deduplicated the embedding commands (`commands/embedding_commands.py`, ~100 lines less): `embed_note`/`embed_insight`/`embed_source` now share one loadโ†’embedโ†’write core with a single error-handling epilogue, the rebuild command uses one submission-loop helper for all three kinds, and the thrice-copied `full_model_dump()` moved to `open_notebook/utils/model_utils.py`. Pure refactor โ€” same outputs, logs and retry behavior (#1071) + +### Removed +- Dead Streamlit-era service layer (~2,000 lines): `api/client.py` (a synchronous HTTP client that called the app's own API) and 13 `api/*_service.py` wrappers that consumed the app's own HTTP API โ€” none were imported by any router, command or test. Also removed the toy `process_text`/`analyze_data` demo commands (`commands/example_commands.py`) from the background worker (#1054) +- Pre-1.6 embedding job compatibility shims (the `embed_single_item`, `embed_chunk` and `vectorize_source` command handlers) โ€” they existed only so jobs queued by a pre-1.6 version could drain after an upgrade, and any worker restarted on 1.6+ has no such jobs. **Upgrade note:** if you are upgrading from a version older than 1.6 with embedding jobs still queued, drain the queue on a 1.x release before upgrading past this change. Also removed dead tooling config from `pyproject.toml`: the `[tool.mypy]` block (the real config is `mypy.ini`) and Streamlit-era ruff per-file-ignores for files that no longer exist (#1056) +- Committed QA screenshots (12 files) and a stray debug `history.txt` were removed from the repo root, with `.gitignore` rules added so they can't come back (#1053) + +### Fixed +- Podcast generation now honors the `speaker_profile` parameter of `POST /api/podcasts/generate` โ€” previously it was silently ignored and the speaker was always re-derived from the episode profile's `speaker_config`, which failed when that pointed at a renamed/deleted speaker profile even if the caller supplied a valid one (#1044) + +## [1.11.0] - 2026-07-11 + +### Added +- `VISION.md` โ€” the product's source of truth in two layers: durable identity (what Open Notebook is and is not, core principles) and current posture (the phase we're in, directional constraints, and the horizon clusters under consideration) +- Decision records at `docs/7-DEVELOPMENT/decisions/` โ€” short, immutable ADRs/PDRs answering "why is it like this?", seeded with 4 retroactive ADRs (SurrealDB, delegation to external libraries, Streamlitโ†’Next.js, background workers) and 2 PDRs (single-user first, provider-agnostic core) +- `AGENTS.md` files (root, `open_notebook/`, `frontend/`) with the normative rules for coding agents and humans โ€” commands, hard rules, and gotchas not derivable from the code; `CLAUDE.md` files are now one-line pointers to them +- Five new engineering docs pages under `docs/7-DEVELOPMENT/`: credentials, content processing, podcasts, prompts, and frontend architecture +- Contribution guidelines for AI-assisted and agent-generated PRs in the contributing guide โ€” the operator owns the PR, issue-first still applies, tests must have actually run +- CI check for broken relative links in markdown (`scripts/check_md_links.py` + `docs-links` workflow on PRs touching `*.md`) +- `cubic.yaml` โ€” AI review settings as code: PR-contract instructions, three custom review agents (vision & principles alignment backed by `VISION.md`, known mechanical caveats, security & testability) and automatic ultrareviews for auth/credential/encryption/migration changes +- Documented the flow-driven release process in `.github/RELEASE_PROCESS.md`, including the `ready` to `main` to stable release path, dev/stable image labels, and maintainer verification checklist (#938) +- List view for the Notebooks page โ€” a tile/list toggle in the header lets you switch between the visual card grid and a compact row layout (name, description, source/note counts, last updated) for easier scanning of large collections. The choice is remembered across reloads and translated across all 14 locales (#885) +- Documented the `ESPERANTO_TTS_TIMEOUT` environment variable (default `300`s) in the environment reference; raise it for slow or self-hosted TTS providers so long podcast segments don't fail with a timeout (#937) +- `SECURITY.md` with a coordinated-disclosure policy: how to privately report a vulnerability via GitHub's private vulnerability reporting, supported versions, and response expectations (#943) +- LaTeX math rendering (KaTeX) now also applies to source content, source insights, Ask answers, transformation output, and the note editor preview โ€” previously only chat had it (#269) +- Syntax highlighting for fenced code blocks in chat responses, source content, insights and Ask/Search answers โ€” light/dark aware, with 25 common languages bundled (others render as plain text) (#783) +- "Recently Viewed" section on the Notebooks page โ€” a collapsible grid of the last 12 notebooks and sources you opened, newest first, hidden when there's no view history. Backed by a new `last_viewed_at` timestamp stamped on read and a `GET /api/recently-viewed` endpoint (translated across all 14 locales) (#850) +- Per-transformation model selection โ€” each transformation can now be assigned its own language model from the transformation editor, overriding the global transformation default for that transformation only. Runs without an explicit model keep using the system default as before (#776) +- "Refresh content" action on web-link sources โ€” re-fetches the URL and re-embeds the source so its content stays current, available from the source card menu once processing has completed (translated across all 14 locales) (#259) +- Sources table can now be sorted by every column โ€” type, title, insights count, embedded status, created and updated (a new "Updated" column was added) โ€” via clickable column headers backed by new `GET /api/sources` sort fields (translated across all locales) (#895) +- EasyPanel deployment template under `examples/easypanel/` โ€” provisions the app plus a dedicated SurrealDB service with auto-generated database/encryption secrets โ€” plus an EasyPanel section in the single-container install guide (#189) +- Test coverage measurement in CI: backend via `pytest-cov` (terminal + XML reports), frontend via `@vitest/coverage-v8` and a new `test:coverage` script (#942) + +### Changed +- Developer documentation restructured: 17 knowledge-heavy `CLAUDE.md` files consolidated into the 3 `AGENTS.md` + docs pages above; `README.dev.md` became a pointer after its unique content moved into `development-setup.md` (make-workflow matrix), `.github/RELEASE_PROCESS.md` (Docker publishing) and the change playbooks (add-a-language); the maintainer guide now carries the curated label taxonomy (state funnel, `area:` labels, consolidation rules) +- Fixed stale developer docs while migrating: real migration path/format (`open_notebook/database/migrations/N.surrealql` + `AsyncMigrationManager` registration), provider count (17), locale list (7), and 9 README links that pointed at documentation pages that never existed +- The API's listen interface in the Docker images is now configurable via a new `API_HOST` environment variable instead of a hardcoded `--host 0.0.0.0`. The default is unchanged (`0.0.0.0`); set `API_HOST=::` to serve IPv6/dual-stack environments (#985) +- `docker-compose.yml` now sources the SurrealDB credentials from `SURREAL_USER` / `SURREAL_PASSWORD` (applied to both the database server and the app), defaulting to `root:root` so the zero-config quick start is unchanged. Set them in a `.env` file to use your own credentials before exposing the instance; `.env.example` and the compose file note this (#946) +- Docs no longer claim a hardcoded default API password (`open-notebook-change-me`) exists; the actual behavior is that auth is disabled entirely when `OPEN_NOTEBOOK_PASSWORD` is unset. Also removed the dead `check_api_password` helper that had been superseded by the auth middleware (#1026) + +### Fixed +- Testing a valid Google/Vertex credential no longer fails after Google retires a Gemini model. The connection test used a hard-coded model id that Google shuts down on a schedule (`gemini-2.0-flash`), so a valid key surfaced as a broken connection (#970). The Google/Vertex test now uses Google's floating `gemini-flash-latest` alias, and the provider connection test was reframed so only a rejected key, missing permissions, or an unreachable endpoint count as failures โ€” a missing/retired/rate-limited model still reports the credentials as valid. Deprecated `gemini-1.5`/`gemini-2.0` model references were also removed from the connection-test model lists and documentation +- API startup no longer crashes when SurrealDB isn't ready yet (e.g. docker-compose race on host reboot: `Temporary failure in name resolution`). The lifespan now polls a lightweight readiness probe with bounded exponential backoff (~50s budget, 5s per-probe timeout) before running migrations; migration errors themselves still fail fast (#708) +- Markdown typography styles (`prose` classes) are active again: the Tailwind v4 migration left the old `tailwind.config.ts` (which loaded `@tailwindcss/typography`) silently ignored, so rendered markdown lost its typographic styling. The plugin and class-based dark mode are now configured in `globals.css`, and markdown rendering is centralized in a shared `MarkdownRenderer` component (#783) +- Podcast generation no longer truncates on dense, long-form content (`LengthFinishReasonError` / `OUTPUT_PARSING_FAILURE`): episode profiles now support an optional `max_tokens` that is passed through to podcast_creator's outline/transcript generation, overriding its defaults โ€” settable via the episode profile API (UI follow-up in #991) (#639) +- API no longer freezes for all requests while a chat waits on the LLM. Both the notebook chat (`execute_chat`) and source chat handlers ran LangGraph's synchronous `invoke()` directly on the event loop; they now run it via `asyncio.to_thread()` (matching the existing `get_state` calls), so other requests stay responsive โ€” and the source-chat SSE can flush its early events instead of stalling until the model finishes (#704) +- Windows native install guide no longer points users at a `start-open-notebook.bat` that doesn't exist in the repo; the Quick Start now documents starting the four services manually with `uv run`, plus an optional sample launcher you can save yourself (#846) +- OpenRouter (and other providers') "Discover models" dialog no longer cuts off the submit button: the dialog now uses a fixed header/footer with a scrollable body (`grid-rows-[auto_1fr_auto]`) instead of scrolling the whole content, so the "Add" button stays visible regardless of how many models are listed (#816) +- Chat references using the short `[insight:]` form (emitted by some models) are now rendered as clickable citations like `[source_insight:]` and `[note:]` already were; `insight` is treated as an alias for `source_insight`, so clicking it opens the insight (#490) +- CRUD endpoints now return `404` (not `500`) for a non-existent resource. `ObjectModel.get()` raises `NotFoundError` rather than returning a falsy value, so the broad `except Exception` in each handler was masking it as a server error. Added an explicit `NotFoundError โ†’ 404` arm to the notebook (update / delete / delete-preview / add-source / remove-source), note (get / update / delete / list / create), model (delete), credential (update / delete) and embed handlers (#862) +- Token counting no longer raises `ValueError: disallowed special token '<|endoftext|>'` when source/context content contains special-token sequences; `token_count()` now encodes with `disallowed_special=()` so such substrings are treated as ordinary text (#667) +- Single-container image no longer hangs at "API not ready yet" on a brand-new instance. `supervisord.single.conf` ran the API and worker with `uv run` (without `--no-sync`), so at startup `uv` tried to sync dev dependencies it couldn't resolve against the `--no-dev` build. Both processes now use `uv run --no-sync`, matching the multi-container `supervisord.conf` (#609) +- Note editor now expands to fill the dialog instead of being capped at `500px`; removed the `max-h-[500px]` constraint that overrode the `flex-1` parent and cramped editing on tall windows (#932) +- Ask and source-chat responses now stream progressively instead of hanging at "Processing..." until the full answer is ready. The API's streaming endpoints now declare `text/event-stream` (with no-buffering headers), and dedicated Next.js route handlers pass the SSE body through as a stream โ€” Next.js `rewrites()` buffers SSE responses to completion (#770) +- Chat, notebook-context and podcast generation now build their context with a single batched insight query instead of one query per source (14 โ†’ 3 queries on a 12-source notebook), via the new `SourceInsight.get_for_sources()` (#1008) +- File uploads no longer block the event loop: `save_uploaded_file()` now writes via `asyncio.to_thread()`, keeping the API responsive during large uploads (#1009) +- URL validation no longer blocks the event loop on DNS resolution: `validate_url()` is now async and resolves hostnames via `asyncio.to_thread()`, so a slow DNS lookup on the model-provisioning path can't stall concurrent requests (#1011) +- Creating a credential with an unknown provider name now fails with a clear `422` at the API boundary instead of an opaque error deep in the domain layer; `provider` is validated against the 17 supported providers, and a test keeps the frontend/backend provider lists in sync (#1016) +- Podcast episode listing now batch-fetches job statuses in one query instead of one per episode, speeding up notebooks with many episodes; podcast audio-file paths are additionally verified to stay within the podcasts folder before streaming/deleting (#1018) +- Transformations no longer report success while silently losing their insight when the embedding job fails to queue: `Source.add_insight()` now raises on submission failure (handled by job-level retry), note auto-embedding degrades gracefully instead of turning a note save into a 500, and the explicit note-embed endpoint surfaces queue failures as errors (#1019) +- Clearing a credential field in the edit dialog (Ollama/OpenAI-compatible `base_url`, Vertex `project`/`location`/`credentials_path`) now actually clears it. Two mirror-image bugs made it impossible: the frontend dropped emptied fields from the PUT body (`undefined` keys are stripped by `JSON.stringify`), and the API ignored explicit `null`s (`is not None` guards) โ€” so the old value survived while the UI reported success. The frontend now sends explicit `null` and the API keys partial updates on field presence (`model_fields_set`) (#1046) + +### Security +- Resolved dependency audit findings: added npm `overrides` for vulnerable transitive frontend packages (`ws`, `brace-expansion`, `ajv`, `@eslint/plugin-kit`, `postcss`) โ€” `npm audit` now reports 0 vulnerabilities โ€” refreshed `uv.lock` (`langsmith`, `pydantic-settings`, `pip`), and hardened external `window.open(..., '_blank')` calls with `noopener,noreferrer` (#962) +- SurrealQL injection via record ids in `repo_relate()`/`repo_upsert()`/`repo_update()`: a crafted `notebook_id` on the save-insight-as-note flow could execute arbitrary SurrealQL. Record identifiers are now bound as query parameters, and the target notebook's existence is validated before relating (#1002) +- The API password is now compared with `secrets.compare_digest()` instead of `!=`, closing a timing side-channel on authentication (#1003) +- User-authored transformation prompts are no longer compiled as Jinja2 template source (a DoS vector via template loops); they are passed as plain variables into fixed developer-authored templates, so Jinja syntax inside a prompt renders as inert text. Output is unchanged for legitimate prompts (#1004) +- SSRF protection on source-URL ingestion: adding a web-link source now runs the same `validate_url()` guard already used for credential URLs, rejecting internal/private/cloud-metadata addresses (#1005) +- Provider-credential URLs are re-validated immediately before every outbound request (connection tests, model discovery and inference) instead of only at save time, closing a DNS-rebinding window; AWS's IPv6 metadata address was added to the blocklist (#1006) +- The note/transformation markdown preview now sanitizes raw HTML via `rehype-sanitize`: ` after') + expect(container.querySelector('iframe')).toBeNull() + expect(container.innerHTML).not.toContain('evil.example') + }) + + it('strips raw after') + expect(container.innerHTML).not.toContain('__pwned') + }) + + it('strips a raw after') + expect(container.querySelector('style')).toBeNull() + }) + + it('strips javascript: URLs from links', () => { + const { container } = renderPreview('[click me](javascript:alert(1))') + const link = container.querySelector('a') + expect(link?.getAttribute('href') ?? '').not.toContain('javascript:') + }) + + it('does not execute inline event-handler attributes on parsed raw elements', () => { + const { container } = renderPreview('') + expect(container.innerHTML).not.toContain('onerror') + }) + + it('still renders KaTeX math with its classes and MathML intact', () => { + const { container } = renderPreview('Inline math $x^2 + y^2 = z^2$') + expect(container.querySelector('.katex')).not.toBeNull() + expect(container.querySelector('.katex-mathml math')).not.toBeNull() + }) + + it('still syntax-highlights fenced code blocks', () => { + const { container } = renderPreview('```python\ndef hello():\n return 42\n```') + expect(container.querySelectorAll('span[class*="token"]').length).toBeGreaterThan(0) + }) + + it('still renders the code-block copy button the library injects', () => { + // The preview library's rehypeRewrite injects div.copied[data-code] with + // two octicon SVGs before user plugins run; the sanitize schema must let + // exactly that markup through or the copy feature dies silently. + const { container } = renderPreview('```js\nconst a = 1;\n```') + const button = container.querySelector('pre div.copied') + expect(button).not.toBeNull() + expect(button?.getAttribute('data-code')).toContain('const a = 1;') + expect(button?.querySelector('svg.octicon-copy')).not.toBeNull() + expect(button?.querySelector('svg.octicon-check')).not.toBeNull() + }) + + it('still renders GFM tables, task lists, and safe links', () => { + const { container } = renderPreview( + '| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n\n[a link](https://example.com)' + ) + expect(container.querySelector('table')).not.toBeNull() + expect(container.querySelectorAll('input[type="checkbox"]').length).toBe(2) + expect(container.querySelector('a')?.getAttribute('href')).toBe('https://example.com') + }) +}) diff --git a/frontend/src/components/ui/markdown-editor.tsx b/frontend/src/components/ui/markdown-editor.tsx new file mode 100644 index 0000000..e6caeff --- /dev/null +++ b/frontend/src/components/ui/markdown-editor.tsx @@ -0,0 +1,87 @@ +'use client' + +import dynamic from 'next/dynamic' +import { forwardRef } from 'react' +import remarkMath from 'remark-math' +import rehypeKatex from 'rehype-katex' +import rehypeSanitize, { defaultSchema } from 'rehype-sanitize' +import type { PluggableList } from 'unified' + +const MDEditor = dynamic( + () => import('@uiw/react-md-editor').then((mod) => mod.default), + { ssr: false } +) + +// Render `$...$` / `$$...$$` math in the live preview. @uiw/react-md-editor +// concatenates these with its defaults (gfm, prism, raw), so syntax +// highlighting and GFM are preserved. KaTeX CSS is loaded globally in +// app/layout.tsx. +// +// The library's own `raw` default lets literal HTML in the markdown source +// (e.g. pasted content, or an AI-generated note echoing an indirect prompt +// injection) render as live elements - notably a real