wehub/k-dense-ai--scientific-agent-skills
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:12:01 +08:00
commit 0fc0c0bb15
1490 changed files with 604780 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-skill-scan.yml
+111
View File
@@ -0,0 +1,111 @@
name: PR Skill Scan
on:
pull_request:
paths:
- "skills/**"
- "scan_skills.py"
- "scan_pr_skills.py"
- "pyproject.toml"
- "uv.lock"
- ".github/workflows/pr-skill-scan.yml"
permissions:
contents: read
pull-requests: write
concurrency:
group: pr-skill-scan-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
scan:
name: Scan changed skills
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout PR
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Detect changed skills
id: changed
run: |
set -euo pipefail
BASE_SHA="${{ github.event.pull_request.base.sha }}"
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
echo "Base: $BASE_SHA"
echo "Head: $HEAD_SHA"
# Files added/copied/modified/renamed under skills/<skill>/...
CHANGED_FILES=$(git diff --name-only --diff-filter=ACMR "$BASE_SHA" "$HEAD_SHA" -- 'skills/**' || true)
echo "Changed files under skills/:"
echo "$CHANGED_FILES"
# Derive unique top-level skill directories and keep only those that still exist with a SKILL.md
SKILL_DIRS=$(echo "$CHANGED_FILES" \
| awk -F/ 'NF>=2 && $1=="skills" {print $1 "/" $2}' \
| sort -u)
EXISTING=""
for d in $SKILL_DIRS; do
if [ -f "$d/SKILL.md" ]; then
EXISTING="$EXISTING $d"
fi
done
EXISTING=$(echo "$EXISTING" | xargs || true)
echo "Skill dirs to scan: '$EXISTING'"
echo "skill_dirs=$EXISTING" >> "$GITHUB_OUTPUT"
- name: Set up uv
if: steps.changed.outputs.skill_dirs != ''
uses: astral-sh/setup-uv@v8.0.0
with:
enable-cache: true
cache-dependency-glob: uv.lock
python-version: "3.13"
- name: Install dependencies
if: steps.changed.outputs.skill_dirs != ''
run: uv sync --python 3.13
- name: Run scanner on changed skills
if: steps.changed.outputs.skill_dirs != ''
id: scan
env:
SKILL_SCANNER_LLM_API_KEY: ${{ secrets.SKILL_SCANNER_LLM_API_KEY }}
SKILL_SCANNER_LLM_MODEL: ${{ vars.SKILL_SCANNER_LLM_MODEL || 'claude-sonnet-4-6' }}
run: |
uv run python scan_pr_skills.py \
--output pr_scan_comment.md \
--fail-on HIGH \
${{ steps.changed.outputs.skill_dirs }}
- name: Prepare no-op comment
if: steps.changed.outputs.skill_dirs == ''
run: |
cat > pr_scan_comment.md <<'EOF'
<!-- skill-security-scan -->
## 🛡️ Skill Security Scan
No skill directories (with a `SKILL.md`) were changed in this PR — nothing to scan.
EOF
- name: Upload scan comment as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: pr-skill-scan-comment
path: pr_scan_comment.md
if-no-files-found: ignore
- name: Post or update PR comment
if: always() && hashFiles('pr_scan_comment.md') != ''
uses: marocchino/sticky-pull-request-comment@v2
with:
header: skill-security-scan
path: pr_scan_comment.md
.github/workflows/release.yml
+101
View File
@@ -0,0 +1,101 @@
name: Create Release
on:
push:
branches:
- main
paths:
- 'pyproject.toml'
workflow_dispatch:
permissions:
contents: write
jobs:
release:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for release notes
- name: Extract version from pyproject.toml
id: get_version
run: |
VERSION=$(grep '^version' pyproject.toml | head -1 | sed 's/.*"\(.*\)".*/\1/')
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "tag=v$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION"
- name: Check if tag already exists
id: check_tag
run: |
if git rev-parse "v${{ steps.get_version.outputs.version }}" >/dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
echo "Tag v${{ steps.get_version.outputs.version }} already exists"
else
echo "exists=false" >> $GITHUB_OUTPUT
echo "Tag v${{ steps.get_version.outputs.version }} does not exist"
fi
- name: Get previous tag
id: previous_tag
if: steps.check_tag.outputs.exists == 'false'
run: |
PREVIOUS_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
if [ -z "$PREVIOUS_TAG" ]; then
echo "previous_tag=" >> $GITHUB_OUTPUT
echo "No previous tag found"
else
echo "previous_tag=$PREVIOUS_TAG" >> $GITHUB_OUTPUT
echo "Previous tag: $PREVIOUS_TAG"
fi
- name: Generate release notes
id: release_notes
if: steps.check_tag.outputs.exists == 'false'
run: |
PREVIOUS_TAG="${{ steps.previous_tag.outputs.previous_tag }}"
# Start release notes
cat > release_notes.md << 'EOF'
## What's Changed
EOF
# Generate changelog from commits
if [ -n "$PREVIOUS_TAG" ]; then
echo "Changes since $PREVIOUS_TAG:" >> release_notes.md
echo "" >> release_notes.md
# Get commits with nice formatting
git log ${PREVIOUS_TAG}..HEAD --pretty=format:"* %s (%h)" --no-merges >> release_notes.md
else
echo "Initial release of Claude Scientific Skills" >> release_notes.md
echo "" >> release_notes.md
echo "This release includes:" >> release_notes.md
git log --pretty=format:"* %s (%h)" --no-merges --max-count=20 >> release_notes.md
fi
cat release_notes.md
- name: Create Release
if: steps.check_tag.outputs.exists == 'false'
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.get_version.outputs.tag }}
name: v${{ steps.get_version.outputs.version }}
body_path: release_notes.md
draft: false
prerelease: false
generate_release_notes: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Skip release creation
if: steps.check_tag.outputs.exists == 'true'
run: |
echo "Release v${{ steps.get_version.outputs.version }} already exists. Skipping release creation."
.github/workflows/security-scan.yml
+44
View File
@@ -0,0 +1,44 @@
name: Weekly Security Scan
on:
schedule:
- cron: "0 9 * * 1" # Every Monday at 09:00 UTC
workflow_dispatch: # Allow manual trigger
permissions:
contents: write
jobs:
scan:
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@v6
- uses: astral-sh/setup-uv@v8.0.0
with:
enable-cache: true
cache-dependency-glob: uv.lock
python-version: "3.13"
- name: Install dependencies
run: uv sync --python 3.13
- name: Run security scan
env:
SKILL_SCANNER_LLM_API_KEY: ${{ secrets.SKILL_SCANNER_LLM_API_KEY }}
SKILL_SCANNER_LLM_MODEL: ${{ vars.SKILL_SCANNER_LLM_MODEL || 'claude-sonnet-4-6' }}
run: uv run python scan_skills.py
- name: Commit updated SECURITY.md
run: |
git diff --quiet SECURITY.md && exit 0
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git stash --include-untracked
git pull --rebase
git stash pop || true
git add SECURITY.md
git commit -m "chore: update security scan report [skip ci]"
git push
.gitignore
+21
View File
@@ -0,0 +1,21 @@
# OS
.DS_Store
# Python
.venv/
.python-version
__pycache__/
# Secrets
.env
# Project
temp/
research/
.agents/
AGENTS.md
skills-lock.json
uv.lock
CONTRIBUTING.md
+217
View File
@@ -0,0 +1,217 @@
# Contributing Skills
Thanks for helping improve Scientific Agent Skills. This guide explains how to add or update a skill in this repository while following the open [Agent Skills specification](https://agentskills.io/specification).
## Ways to Contribute
- Add a new scientific package, database, platform, workflow, or research method skill.
- Improve an existing skill with clearer instructions, current APIs, better examples, references, or scripts.
- Fix outdated examples, broken install steps, security issues, or documentation gaps.
- Report bugs or request new skills through GitHub Issues.
## Skill Location
All repository skills live under `skills/`:
```text
skills/
└── skill-name/
├── SKILL.md
├── references/
├── scripts/
└── assets/
```
Only `SKILL.md` is required. Use optional directories when they make the skill easier to maintain:
- `references/` for longer documentation that agents should read only when needed.
- `scripts/` for executable helpers, validators, or reusable workflow code.
- `assets/` for templates, static resources, or example data.
Keep references one level deep from `SKILL.md` where possible, and keep the main `SKILL.md` concise. The Agent Skills specification recommends keeping `SKILL.md` under 500 lines and using progressive disclosure for longer material.
## Required Skill Format
Every skill must be a directory containing a `SKILL.md` file with YAML frontmatter followed by Markdown instructions.
Use this minimum template:
```markdown
---
name: skill-name
description: Clear description of what the skill does and when an agent should use it.
metadata: {"version": "1.0", "skill-author": "Your Name"}
---
# Skill Title
## When to Use
Use this skill when...
## Workflow
1. ...
2. ...
## Examples
...
```
### Frontmatter Requirements
Follow the [Agent Skills specification](https://agentskills.io/specification) and this repository's conventions:
- `name` is required, must match the parent directory name, and must be 1-64 characters.
- `name` may contain only lowercase letters, numbers, and hyphens.
- `name` must not start or end with a hyphen and must not contain consecutive hyphens.
- `description` is required, non-empty, and must be at most 1024 characters.
- `description` should explain both what the skill does and when an agent should use it.
- `metadata.version` is required in this repository, even though `metadata` is optional in the upstream spec.
- Version values must be quoted numeric strings, such as `"1.0"` or `"1.1"`.
- **Write `metadata` as a single-line JSON object** (flow style), for example `metadata: {"version": "1.0", "skill-author": "K-Dense Inc."}`. This is valid YAML — so it parses identically in Claude Code, Cursor, Codex, Hermes, Pi, and any Agent Skills-compliant host — and it is the only form OpenClaw's line-based frontmatter reader can parse (a multi-line block `metadata:` is silently dropped there). Do not use a nested `metadata:` block.
Optional frontmatter fields from the specification may be used when relevant:
- `license`: the license for the individual skill, if different or worth stating explicitly.
- `compatibility`: environment requirements such as Python version, system packages, agent host, or network access.
- `metadata`: additional metadata. Must be a single-line JSON object (see above). Common keys: `version` (required), `skill-author`, an optional `openclaw` block, and an optional `hermes` block (see below).
- `allowed-tools`: space-separated tool permissions for hosts that support this experimental field.
- `required_environment_variables`: top-level Hermes credential declarations (see below). Other hosts ignore it.
### OpenClaw gating (`metadata.openclaw`)
OpenClaw reads an optional `openclaw` object nested inside `metadata` for dependency gating, credential injection, and display. Because it lives under `metadata`, the Agent Skills spec permits it and other hosts ignore it. It is only needed for skills with external requirements (credentials, daemons, specific binaries) — most skills omit it entirely. Supported keys:
- `requires`: hard eligibility gates — `{"bins": [...]}` (all must be on `PATH`), `{"anyBins": [...]}` (at least one), `{"env": [...]}` (vars that must be set), `{"config": [...]}`. A failed gate hides the skill from the agent, so only gate on things the skill genuinely cannot run without.
- `primaryEnv`: the main credential variable; OpenClaw injects it from its config (`skills.entries.<name>.apiKey`).
- `envVars`: descriptive (non-gating) declarations — `[{"name": "X_API_KEY", "required": true, "description": "..."}]`. Declare every env var your scripts reference so ClawHub's security analysis does not flag a metadata mismatch.
- `os`: platform filter, e.g. `["darwin", "linux"]`.
- `emoji`, `homepage`: display only.
Example (an API-key skill that stays available even without the key set, so it gates nothing and only declares the credential):
```yaml
metadata: {"version": "1.0", "skill-author": "K-Dense Inc.", "openclaw": {"primaryEnv": "EXA_API_KEY", "envVars": [{"name": "EXA_API_KEY", "required": true, "description": "Exa search API key."}]}}
```
### Hermes compatibility (`required_environment_variables` and `metadata.hermes`)
[Hermes](https://hermes-agent.nousresearch.com/docs) is Agent Skills-compatible, so every skill in this repository already loads and runs there with no changes. Two optional fields make credentialed skills first-class on Hermes:
- **`required_environment_variables`** (top level): the credentials Hermes should prompt for. Write it as a single-line JSON array — `[{"name": "X_API_KEY", "prompt": "What it is", "required_for": "full functionality"}]`. This is the one Hermes-specific field that is *not* nested under `metadata`, because Hermes reads secrets at the top level. Writing it as single-line JSON keeps it valid YAML for every host and lets OpenClaw's line-based reader skip it cleanly; Claude Code, Cursor, and Codex ignore the unknown key. Mirror the same variables you declare in `metadata.openclaw.envVars`, using `required_for: "full functionality"` for required vars and `"optional features"` for optional ones.
- **`metadata.hermes`** (nested, spec-safe like `openclaw`): optional classification and gating — `tags`, `category`, `requires_toolsets`, `fallback_for_toolsets`. A failed `requires_toolsets` gate *hides* the skill, so only gate on a tool the skill genuinely cannot run without; prefer leaving it unset so the skill stays available.
Example (an API-key skill, declaring its credential for Hermes alongside the OpenClaw block):
```yaml
required_environment_variables: [{"name": "EXA_API_KEY", "prompt": "Exa search API key.", "required_for": "full functionality"}]
metadata: {"version": "1.0", "skill-author": "Exa", "openclaw": {"primaryEnv": "EXA_API_KEY", "envVars": [{"name": "EXA_API_KEY", "required": true, "description": "Exa search API key."}]}}
```
## Versioning
Every `SKILL.md` must include a quoted `version` inside the single-line `metadata` object:
```yaml
metadata: {"version": "1.0"}
```
For a new skill, start at `"1.0"`.
When updating an existing skill, increment `metadata.version` in the same pull request:
- Use a minor bump for normal improvements, for example `"1.0"` to `"1.1"`.
- Use a major bump only for a breaking change or substantial redesign, for example `"1.9"` to `"2.0"`.
## Writing a Good Skill
Good skills are specific, practical, and easy for an agent to apply.
- Write the `description` in third person with useful trigger terms.
- Include concrete workflows, commands, and examples instead of broad background explanations.
- Prefer current official APIs, docs, and installation instructions.
- Document required Python packages, system dependencies, credentials, or network access.
- Include scientific best practices, caveats, and validation checks where they matter.
- Move long API details, tables, and extended examples into `references/`.
- Use scripts for fragile or repetitive logic instead of asking the agent to recreate it every time.
- Avoid secrets, credentials, API keys, private URLs, and unpublished data.
## Adding a New Skill
1. Fork the repository and create a branch:
```bash
git checkout -b add-skill-name
```
2. Create a new directory under `skills/` whose name matches the skill name:
```text
skills/skill-name/
```
3. Add `SKILL.md` with valid frontmatter, including `metadata.version`.
4. Add supporting `references/`, `scripts/`, or `assets/` only when they are useful.
5. Test any commands, code examples, and scripts included in the skill.
6. Update related documentation if the new skill changes repository-level lists, examples, or setup guidance.
7. Run validation and security checks before opening a pull request.
## Updating an Existing Skill
1. Read the current `SKILL.md` and any supporting files.
2. Check upstream package, API, or platform documentation for current behavior.
3. Make the smallest useful change that fixes or improves the skill.
4. Increment `metadata.version`.
5. Test changed examples, commands, and scripts.
6. Note any behavior changes in the pull request description.
## Validation
Validate Agent Skills format with the reference validator:
```bash
skills-ref validate ./skills/skill-name
```
If `skills-ref` is not installed, follow the installation instructions from the [skills-ref reference library](https://github.com/agentskills/agentskills/tree/main/skills-ref).
Security-scan new or substantially changed skills:
```bash
uv pip install cisco-ai-skill-scanner
skill-scanner scan ./skills/skill-name --use-behavioral
```
A clean scan reduces review noise but does not replace manual review.
## Pull Request Checklist
Before submitting a pull request, confirm:
- The skill directory name and `name` frontmatter match exactly.
- `SKILL.md` has valid YAML frontmatter and Markdown body content.
- `metadata` is a single-line JSON object (not a multi-line block), so it parses on OpenClaw as well as Claude Code, Cursor, Codex, Hermes, and Pi.
- If the skill needs credentials, `required_environment_variables` is present as a single-line JSON array and mirrors the variables in `metadata.openclaw.envVars`.
- `metadata.version` exists and is quoted.
- Existing skills have a version bump when changed.
- The `description` clearly says what the skill does and when to use it.
- Examples and scripts have been tested or clearly marked as illustrative.
- No secrets, credentials, private data, or unsafe instructions are included.
- Relevant official documentation is linked where useful.
- Security scanner results are clean or explained in the pull request.
## Pull Request Process
1. Push your branch to your fork.
2. Open a pull request with a clear title, such as `Add scanpy workflow examples` or `Update astropy skill for current API`.
3. Describe what changed, why it matters, and how you tested it.
4. Link related issues, package documentation, release notes, or security findings.
5. Respond to review comments and update the skill as needed.
Thank you for helping make scientific computing more accessible to AI agents and researchers.
LICENSE.md
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 K-Dense Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+804
View File
@@ -0,0 +1,804 @@
# Scientific Agent Skills
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md)
[![Version](https://img.shields.io/badge/Version-2.53.0-blue.svg)](pyproject.toml)
[![Skills](https://img.shields.io/badge/Skills-148-brightgreen.svg)](#-whats-included)
[![Databases](https://img.shields.io/badge/Databases-100%2B-orange.svg)](#-whats-included)
[![Agent Skills](https://img.shields.io/badge/Standard-Agent_Skills-blueviolet.svg)](https://agentskills.io/)
[![Security Scan](https://github.com/K-Dense-AI/scientific-agent-skills/actions/workflows/security-scan.yml/badge.svg)](https://github.com/K-Dense-AI/scientific-agent-skills/actions/workflows/security-scan.yml)
[![Works with](https://img.shields.io/badge/Works_with-Cursor_|_Claude_Code_|_Codex_|_Google_Antigravity-blue.svg)](#-getting-started)
[![X](https://img.shields.io/badge/Follow_on_X-%40k__dense__ai-000000?logo=x)](https://x.com/k_dense_ai)
[![LinkedIn](https://img.shields.io/badge/LinkedIn-K--Dense_Inc.-0A66C2?logo=linkedin)](https://www.linkedin.com/company/k-dense-inc)
[![YouTube](https://img.shields.io/badge/YouTube-K--Dense_Inc.-FF0000?logo=youtube)](https://www.youtube.com/@K-Dense-Inc)
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=K-Dense-AI/scientific-agent-skills&type=date&legend=top-left)](https://www.star-history.com/#K-Dense-AI/scientific-agent-skills&type=date&legend=top-left)
> **🔔 Claude Scientific Skills is now Scientific Agent Skills.** Same skills, broader compatibility — now works with any AI agent that supports the open [Agent Skills](https://agentskills.io/) standard, not just Claude.
> **New: [K-Dense BYOK](https://github.com/K-Dense-AI/k-dense-byok)** — A free, open-source AI co-scientist that runs on your desktop, powered by Scientific Agent Skills. Bring your own API keys, pick from 40+ models, and get a full research workspace with web search, file handling, 100+ scientific databases, and access to all 148 skills in this repo. Your data stays on your computer, and you can optionally scale to cloud compute via [Modal](https://modal.com/) for heavy workloads. [Get started here.](https://github.com/K-Dense-AI/k-dense-byok)
> **Stay up to date:** Follow K-Dense on [X](https://x.com/k_dense_ai), [LinkedIn](https://www.linkedin.com/company/k-dense-inc), and [YouTube](https://www.youtube.com/@K-Dense-Inc) for new skills, release announcements, walkthroughs, research workflow demos, and examples you can use with your own AI agent.
A comprehensive collection of **148 ready-to-use scientific and research skills** (covering cancer genomics, drug-target binding, molecular dynamics, RNA velocity, geospatial science, time series forecasting, scientific ML resource discovery via Hugging Science, 78+ scientific databases, and more) for any AI agent that supports the open [Agent Skills](https://agentskills.io/) standard, created by [K-Dense](https://k-dense.ai). Works with **Cursor, Claude Code, Codex, Google Antigravity, and more**. Transform your AI agent into a research assistant capable of executing complex multi-step scientific workflows across biology, chemistry, medicine, and beyond.
> ⭐ **Help make AI for science easier to discover:** If Scientific Agent Skills saves you time, teaches your agent a workflow, or helps your lab move faster, please [star this repository](https://github.com/K-Dense-AI/scientific-agent-skills). A star is a public signal that these open, reusable research skills are worth maintaining: it helps scientists, engineers, and open-source contributors find the project, shows which agent-skill standards are gaining real adoption, and gives us a clear reason to keep expanding the collection for the community.
---
These skills enable your AI agent to seamlessly work with specialized scientific libraries, databases, and tools across multiple scientific domains. While the agent can use any Python package or API on its own, these explicitly defined skills provide curated documentation and examples that make it significantly stronger and more reliable for the workflows below:
- 🧬 Bioinformatics & Genomics - Sequence analysis, single-cell RNA-seq, gene regulatory networks, variant annotation, phylogenetic analysis
- 🧪 Cheminformatics & Drug Discovery - Molecular property prediction, virtual screening, ADMET analysis, molecular docking, lead optimization
- 🔬 Proteomics & Mass Spectrometry - LC-MS/MS processing, peptide identification, spectral matching, protein quantification
- 🏥 Clinical Research & Precision Medicine - Clinical trials, pharmacogenomics, variant interpretation, drug safety, clinical decision support, treatment planning
- 🧠 Healthcare AI & Clinical ML - EHR analysis, physiological signal processing, medical imaging, clinical prediction models
- 🖼️ Medical Imaging & Digital Pathology - DICOM processing, whole slide image analysis, computational pathology, radiology workflows
- 🤖 Machine Learning & AI - Deep learning, reinforcement learning, time series analysis, model interpretability, Bayesian methods
- 🔮 Materials Science & Chemistry - Crystal structure analysis, phase diagrams, metabolic modeling, computational chemistry
- 🌌 Physics & Astronomy - Astronomical data analysis, coordinate transformations, cosmological calculations, symbolic mathematics, physics computations
- ⚙️ Engineering & Simulation - Discrete-event simulation, multi-objective optimization, metabolic engineering, systems modeling, process optimization
- 📊 Data Analysis & Visualization - Statistical analysis, network analysis, time series, publication-quality figures, large-scale data processing, EDA
- 🌍 Geospatial Science & Remote Sensing - Satellite imagery processing, GIS analysis, spatial statistics, terrain analysis, machine learning for Earth observation
- 🧪 Laboratory Automation - Liquid handling protocols, lab equipment control, workflow automation, LIMS integration
- 📚 Scientific Communication - Literature review, peer review, scientific writing, document processing, posters, slides, schematics, citation management
- 🔬 Multi-omics & Systems Biology - Multi-modal data integration, pathway analysis, network biology, systems-level insights
- 🧬 Protein Engineering & Design - Protein language models, structure prediction, sequence design, function annotation
- 🧰 Agent Platforms & Infrastructure - Build on Pi with SDK, RPC, extensions, custom providers/models, packages, TUI components, and session tooling
- 🎓 Research Methodology - Hypothesis generation, scientific brainstorming, critical thinking, grant writing, scholar evaluation
**Transform your AI coding agent into an 'AI Scientist' on your desktop!**
> 🎬 **New to Scientific Agent Skills?** Watch our [Getting Started with Scientific Agent Skills](https://youtu.be/ZxbnDaD_FVg) video for a quick walkthrough.
---
## 📦 What's Included
This repository provides **148 scientific and research skills** organized into the following categories:
- **100+ Scientific & Financial Databases** - A unified database-lookup skill provides deterministic, provenance-rich access to 78 public databases (PubChem, ChEMBL, UniProt, COSMIC, ClinicalTrials.gov, FRED, USPTO, and more), plus dedicated skills for DepMap, Imaging Data Commons, PrimeKG, U.S. Treasury Fiscal Data, and Hugging Science (curated catalog of scientific datasets, models, and demos across 17 scientific domains on Hugging Face). Multi-database packages like BioServices (~40 bioinformatics services), BioPython (38 NCBI sub-databases via Entrez), and gget (20+ genomics databases) add further coverage
- **70+ Optimized Python Package Skills** - Explicitly defined skills for RDKit, Scanpy, PyTorch Lightning, scikit-learn, BioPython, pyzotero, BioServices, PennyLane, Qiskit, Molecular Dynamics (OpenMM/MDAnalysis), scVelo, TimesFM, and others — with curated documentation, examples, and best practices. Note: the agent can write code using *any* Python package, not just these; these skills simply provide stronger, more reliable performance for the packages listed
- **9 Scientific Integration Skills** - Explicitly defined skills for Benchling, DNAnexus, LatchBio, OMERO, Protocols.io, Open Notebook, Ginkgo Cloud Lab, LabArchives, and Opentrons. Again, the agent is not limited to these — any API or platform reachable from Python is fair game; these skills are the optimized, pre-documented paths
- **30+ Analysis & Communication Tools** - Literature review, scientific writing, peer review, document processing, Paperzilla, Exa Search, posters, slides, schematics, infographics, Mermaid diagrams, and more
- **10+ Research & Clinical Tools** - Hypothesis generation, grant writing, clinical decision support, treatment plans, BIDS, regulatory compliance, scenario analysis, and workflow-derived skill drafting with Autoskill
Each skill includes:
- ✅ Comprehensive documentation (`SKILL.md`)
- ✅ Practical code examples
- ✅ Use cases and best practices
- ✅ Integration guides
- ✅ Reference materials
---
## 📋 Table of Contents
- [What's Included](#-whats-included)
- [Why Use This?](#-why-use-this)
- [Getting Started](#-getting-started)
- [Security Disclaimer](#%EF%B8%8F-security-disclaimer)
- [Support Open Source](#%EF%B8%8F-support-the-open-source-community)
- [Prerequisites](#%EF%B8%8F-prerequisites)
- [Quick Examples](#-quick-examples)
- [Use Cases](#-use-cases)
- [Available Skills](#-available-skills)
- [Contributing](#-contributing)
- [Troubleshooting](#-troubleshooting)
- [FAQ](#-faq)
- [Support](#-support)
- [Citation](#-citation)
- [License](#-license)
---
## 🚀 Why Use This?
### ⚡ **Accelerate Your Research**
- **Save Days of Work** - Skip API documentation research and integration setup
- **Production-Ready Code** - Tested, validated examples following scientific best practices
- **Multi-Step Workflows** - Execute complex pipelines with a single prompt
### 🎯 **Comprehensive Coverage**
- **148 Skills** - Extensive coverage across all major scientific domains
- **100+ Databases** - Unified access to 78+ databases via database-lookup, plus dedicated data access skills and multi-database packages like BioServices, BioPython, and gget
- **70+ Optimized Python Package Skills** - RDKit, Scanpy, PyTorch Lightning, scikit-learn, BioServices, PennyLane, Qiskit, Molecular Dynamics (OpenMM/MDAnalysis), scVelo, TimesFM, and others (the agent can use any Python package; these are the pre-documented, higher-performing paths)
### 🔧 **Easy Integration**
- **Simple Setup** - Copy skills to your skills directory and start working
- **Automatic Discovery** - Your agent automatically finds and uses relevant skills
- **Well Documented** - Each skill includes examples, use cases, and best practices
### 🌟 **Maintained & Supported**
- **Regular Updates** - Continuously maintained and expanded by K-Dense team
- **Community Driven** - Open source with active community contributions
- **Enterprise Ready** - Commercial support available for advanced needs
---
## 🎯 Getting Started
### Option 1: npx (all platforms)
Install Scientific Agent Skills with a single command:
```bash
npx skills add K-Dense-AI/scientific-agent-skills
```
This is the official standard approach for installing Agent Skills across **all platforms**, including **Claude Code**, **Claude Cowork**, **Codex**, **Gemini CLI**, **Google Antigravity**, **Cursor**, **OpenClaw**, **NVIDIA NemoClaw**, **Hermes**, **Pi**, and any other agent that supports the open [Agent Skills](https://agentskills.io/) standard.
### Option 2: GitHub CLI (`gh skill`)
If you use the [GitHub CLI](https://cli.github.com/) (v2.90.0+), you can install skills with [`gh skill`](https://github.blog/changelog/2026-04-16-manage-agent-skills-with-github-cli/):
```bash
# Browse and install interactively
gh skill install K-Dense-AI/scientific-agent-skills
# Install a specific skill directly
gh skill install K-Dense-AI/scientific-agent-skills scanpy
# Target a specific agent host
gh skill install K-Dense-AI/scientific-agent-skills --agent cursor
gh skill install K-Dense-AI/scientific-agent-skills --agent claude-code
gh skill install K-Dense-AI/scientific-agent-skills --agent codex
gh skill install K-Dense-AI/scientific-agent-skills --agent gemini
```
`gh skill` automatically installs to the correct directory for your agent host and records provenance metadata for supply chain integrity.
#### Version pinning
Pin to a specific release tag or commit SHA for reproducible installs:
```bash
# Pin to a release tag
gh skill install K-Dense-AI/scientific-agent-skills --pin v1.0.0
# Pin to a commit SHA
gh skill install K-Dense-AI/scientific-agent-skills --pin abc123def
```
#### Keeping skills up to date
```bash
# Check for updates interactively
gh skill update
# Update all installed skills
gh skill update --all
```
### Other Agent Skills hosts (OpenClaw, NemoClaw, Pi, Hermes, …)
You usually don't need anything host-specific. `npx skills add` (Option 1) installs into the shared `~/.agents/skills/` convention, and any compliant client that scans that directory — including **OpenClaw**, **NVIDIA NemoClaw** (an OpenClaw-based secure runtime), and **Pi** — discovers the skills automatically. Project-scoped installs land in `.agents/skills/` and work the same way. To install without the CLI, clone straight into either location:
```bash
git clone https://github.com/K-Dense-AI/scientific-agent-skills.git ~/.agents/skills/scientific-agent-skills # user-level
git clone https://github.com/K-Dense-AI/scientific-agent-skills.git .agents/skills/scientific-agent-skills # project-level
```
**Hermes** is the one host that uses its own registry instead of the shared directory, so add the repo as a tap:
```bash
hermes skills tap add K-Dense-AI/scientific-agent-skills
```
These skills stay portable across all of them: `metadata` is single-line JSON (so OpenClaw's line-based reader parses it), credentialed skills declare a top-level `required_environment_variables` field (so Hermes prompts for keys), and unknown fields are ignored everywhere else. Because 148 skills add up to a lot of standing context, consider installing a topical subset rather than the whole collection.
> **NemoClaw note:** NemoClaw runs agents inside NVIDIA OpenShell with default-deny outbound networking. Skills are discovered and loaded normally, but any skill that needs the network — package installs via `uv`, or API calls (Exa, Parallel, Benchling, NCBI, Materials Project, …) — only works once the operator pre-approves the relevant domains in the OpenShell TUI.
**That's it!** Your AI agent will automatically discover the skills and use them when relevant to your scientific tasks. You can also invoke any skill manually by mentioning the skill name in your prompt.
---
## ⚠️ Security Disclaimer
> **Skills can execute code and influence your coding agent's behavior. Review what you install.**
Agent Skills are powerful — they can instruct your AI agent to run arbitrary code, install packages, make network requests, and modify files on your system. A malicious or poorly written skill has the potential to steer your coding agent into harmful behavior.
We take security seriously. All contributions go through a review process, and we run LLM-based security scans (via [Cisco AI Defense Skill Scanner](https://github.com/cisco-ai-defense/skill-scanner)) on every skill in this repository. However, as a small team with a growing number of community contributions, we cannot guarantee that every skill has been exhaustively reviewed for all possible risks.
**It is ultimately your responsibility to review the skills you install and decide which ones to trust.**
We recommend the following:
- **Do not install everything at once.** Only install the skills you actually need for your work. While installing the full collection was reasonable when K-Dense created and maintained every skill, the repository now includes many community contributions that we may not have reviewed as thoroughly.
- **Read the `SKILL.md` before installing.** Each skill's documentation describes what it does, what packages it uses, and what external services it connects to. If something looks suspicious, don't install it.
- **Check the contribution history.** Skills authored by K-Dense (`K-Dense-AI`) have been through our internal review process. Community-contributed skills have been reviewed to the best of our ability, but with limited resources.
- **Run the security scanner yourself.** Before installing third-party skills, scan them locally:
```bash
uv pip install cisco-ai-skill-scanner
skill-scanner scan /path/to/skill --use-behavioral
```
- **Report anything suspicious.** If you find a skill that looks malicious or behaves unexpectedly, please [open an issue](https://github.com/K-Dense-AI/scientific-agent-skills/issues) immediately so we can investigate.
All skills are scanned on an approximately weekly basis, and [SECURITY.md](SECURITY.md) is updated with the latest results. We try to address security gaps as they arise.
---
## ❤️ Support the Open Source Community
Scientific Agent Skills is powered by **50+ incredible open source projects** maintained by dedicated developers and research communities worldwide. Projects like Biopython, Scanpy, RDKit, scikit-learn, PyTorch Lightning, and many others form the foundation of these skills.
**If you find value in this repository, please consider supporting the projects that make it possible:**
- ⭐ **Star their repositories** on GitHub
- 💰 **Sponsor maintainers** via GitHub Sponsors or NumFOCUS
- 📝 **Cite projects** in your publications
- 💻 **Contribute** code, docs, or bug reports
👉 **[View the full list of projects to support](docs/open-source-sponsors.md)**
---
## ⚙️ Prerequisites
- **Python**: 3.13+ for repository tooling; individual skill dependencies may support broader Python ranges
- **uv**: Python package manager (required for installing skill dependencies)
- **Client**: Any agent that supports the [Agent Skills](https://agentskills.io/) standard (Cursor, Claude Code, Gemini CLI, Codex, Google Antigravity, etc.)
- **System**: macOS, Linux, or Windows with WSL2
- **Dependencies**: Automatically handled by individual skills (check `SKILL.md` files for specific requirements)
### Installing uv
The skills use `uv` as the package manager for installing Python dependencies. Install it using the instructions for your operating system:
**macOS and Linux:**
```bash
curl -LsSf https://astral.sh/uv/install.sh | sh
```
**Windows:**
```powershell
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
```
**Alternative (via pip):**
```bash
pip install uv
```
After installation, verify it works by running:
```bash
uv --version
```
For more installation options and details, visit the [official uv documentation](https://docs.astral.sh/uv/).
---
## 💡 Quick Examples
Once you've installed the skills, you can ask your AI agent to execute complex multi-step scientific workflows. Here are some example prompts:
### 🧪 Drug Discovery Pipeline
**Goal**: Find novel EGFR inhibitors for lung cancer treatment
**Prompt**:
```
Use available skills you have access to whenever possible. Query ChEMBL for EGFR inhibitors (IC50 < 50nM), analyze structure-activity relationships
with RDKit, generate improved analogs with datamol, perform virtual screening with DiffDock
against AlphaFold EGFR structure, search PubMed for resistance mechanisms, check COSMIC for
mutations, and create visualizations and a comprehensive report.
```
**Skills Used**: database-lookup, rdkit, datamol, diffdock, paper-lookup, scientific-visualization
---
### 🔬 Single-Cell RNA-seq Analysis
**Goal**: Comprehensive analysis of 10X Genomics data with public data integration
**Prompt**:
```
Use available skills you have access to whenever possible. Load 10X dataset with Scanpy, perform QC and doublet removal, integrate with Cellxgene
Census data, identify cell types using NCBI Gene markers, run differential expression with
PyDESeq2, infer gene regulatory networks with Arboreto, enrich pathways via Reactome/KEGG,
and identify therapeutic targets with Open Targets.
```
**Skills Used**: scanpy, cellxgene-census, database-lookup, pydeseq2, arboreto
---
### 🧬 Multi-Omics Biomarker Discovery
**Goal**: Integrate RNA-seq, proteomics, and metabolomics to predict patient outcomes
**Prompt**:
```
Use available skills you have access to whenever possible. Analyze RNA-seq with PyDESeq2, process mass spec with pyOpenMS, integrate metabolites from
HMDB/Metabolomics Workbench, map proteins to pathways (UniProt/KEGG), find interactions via
STRING, correlate omics layers with statsmodels, build predictive model with scikit-learn,
and search ClinicalTrials.gov for relevant trials.
```
**Skills Used**: pydeseq2, pyopenms, database-lookup, statsmodels, scikit-learn
---
### 🎯 Virtual Screening Campaign
**Goal**: Discover allosteric modulators for protein-protein interactions
**Prompt**:
```
Use available skills you have access to whenever possible. Retrieve AlphaFold structures, identify interaction interface with BioPython, search ZINC
for allosteric candidates (MW 300-500, logP 2-4), filter with RDKit, dock with DiffDock,
rank with DeepChem, check PubChem suppliers, search USPTO patents, and optimize leads with
MedChem/molfeat.
```
**Skills Used**: database-lookup, biopython, rdkit, diffdock, deepchem, medchem, molfeat
---
### 🏥 Clinical Variant Interpretation
**Goal**: Analyze VCF file for hereditary cancer risk assessment
**Prompt**:
```
Use available skills you have access to whenever possible. Parse VCF with pysam, annotate variants with Ensembl VEP, query ClinVar for pathogenicity,
check COSMIC for cancer mutations, retrieve gene info from NCBI Gene, analyze protein impact
with UniProt, search PubMed for case reports, check ClinPGx for pharmacogenomics, generate
clinical report with document processing tools, and find matching trials on ClinicalTrials.gov.
```
**Skills Used**: pysam, database-lookup, paper-lookup, clinical-reports, docx, pdf
---
### 🌐 Systems Biology Network Analysis
**Goal**: Analyze gene regulatory networks from RNA-seq data
**Prompt**:
```
Use available skills you have access to whenever possible. Query NCBI Gene for annotations, retrieve sequences from UniProt, identify interactions via
STRING, map to Reactome/KEGG pathways, analyze topology with Torch Geometric, reconstruct
GRNs with Arboreto, assess druggability with Open Targets, model with PyMC, visualize
networks, and search GEO for similar patterns.
```
**Skills Used**: database-lookup, torch-geometric, arboreto, pymc, networkx, scientific-visualization
> 📖 **Want more examples?** Check out [docs/examples.md](docs/examples.md) for comprehensive workflow examples and detailed use cases across all scientific domains.
---
## 🔬 Use Cases
### 🧪 Drug Discovery & Medicinal Chemistry
- **Virtual Screening**: Screen millions of compounds from PubChem/ZINC against protein targets
- **Lead Optimization**: Analyze structure-activity relationships with RDKit, generate analogs with datamol
- **ADMET Prediction**: Predict absorption, distribution, metabolism, excretion, and toxicity with DeepChem
- **Molecular Docking**: Predict binding poses with DiffDock and rescore poses with affinity-oriented tools
- **Bioactivity Mining**: Query ChEMBL for known inhibitors and analyze SAR patterns
### 🧬 Bioinformatics & Genomics
- **Sequence Analysis**: Process DNA/RNA/protein sequences with BioPython and pysam
- **Single-Cell Analysis**: Analyze 10X Genomics data with Scanpy, identify cell types, infer GRNs with Arboreto
- **Variant Annotation**: Annotate VCF files with Ensembl VEP, query ClinVar for pathogenicity
- **Variant Database Management**: Build scalable VCF databases with TileDB-VCF for incremental sample addition, efficient population-scale queries, and compressed storage of genomic variant data
- **Gene Discovery**: Query NCBI Gene, UniProt, and Ensembl for comprehensive gene information
- **Network Analysis**: Identify protein-protein interactions via STRING, map to pathways (KEGG, Reactome)
### 🏥 Clinical Research & Precision Medicine
- **Clinical Trials**: Search ClinicalTrials.gov for relevant studies, analyze eligibility criteria
- **Variant Interpretation**: Annotate variants with ClinVar, COSMIC, and ClinPGx for pharmacogenomics
- **Drug Safety**: Query FDA databases for adverse events, drug interactions, and recalls
- **Precision Therapeutics**: Match patient variants to targeted therapies and clinical trials
### 🔬 Multi-Omics & Systems Biology
- **Multi-Omics Integration**: Combine RNA-seq, proteomics, and metabolomics data
- **Pathway Analysis**: Enrich differentially expressed genes in KEGG/Reactome pathways
- **Network Biology**: Reconstruct gene regulatory networks, identify hub genes
- **Biomarker Discovery**: Integrate multi-omics layers to predict patient outcomes
### 📊 Data Analysis & Visualization
- **Statistical Analysis**: Perform hypothesis testing, power analysis, and experimental design
- **Publication Figures**: Create publication-quality visualizations with matplotlib and seaborn
- **Network Visualization**: Visualize biological networks with NetworkX
- **Report Generation**: Generate comprehensive reports with the PDF, DOCX, PPTX, XLSX, MarkItDown, LiteParse, and clinical-reporting skills
### 🧪 Laboratory Automation
- **Protocol Design**: Create Opentrons protocols for automated liquid handling
- **LIMS Integration**: Integrate with Benchling and LabArchives for data management
- **Workflow Automation**: Automate multi-step laboratory workflows
---
## 📚 Available Skills
This repository contains **148 scientific and research skills** organized across multiple domains. Each skill provides comprehensive documentation, code examples, and best practices for working with scientific libraries, databases, and tools.
### Skill Categories
> **Note:** The Python package and integration skills listed below are *explicitly defined* skills — curated with documentation, examples, and best practices for stronger, more reliable performance. They are not a ceiling: the agent can install and use *any* Python package or call *any* API, even without a dedicated skill. The skills listed simply make common workflows faster and more dependable.
#### 🧬 **Bioinformatics & Genomics** (23 skills)
- RNA-seq pipelines: Bulk RNA-seq (end-to-end FASTQ -> counts -> DE -> enrichment orchestrator)
- Sequence analysis: BioPython, pysam, scikit-bio, BioServices
- Single-cell analysis: Scanpy, AnnData, scvi-tools, scVelo (RNA velocity), Arboreto, Cellxgene Census
- Genomic tools: gget, geniml, gtars, deepTools, FlowIO, Polars-Bio, Zarr, TileDB-VCF
- Differential expression: PyDESeq2
- Functional enrichment: Pathway Enrichment (ORA, GSEA/preranked, ssGSEA via gseapy + g:Profiler; GO, KEGG, Reactome, WikiPathways, MSigDB)
- Phylogenetics: ETE Toolkit, Phylogenetics (MAFFT, IQ-TREE 2, FastTree)
#### 🧪 **Cheminformatics & Drug Discovery** (10 skills)
- Molecular manipulation: RDKit, Datamol, Molfeat
- Deep learning: DeepChem, TorchDrug
- Docking & screening: DiffDock
- Molecular dynamics: OpenMM + MDAnalysis (MD simulation & trajectory analysis)
- Cloud quantum chemistry: Rowan (pKa, docking, cofolding)
- Drug-likeness: MedChem
- Benchmarks: PyTDC
#### 🔬 **Proteomics & Mass Spectrometry** (2 skills)
- Spectral processing: matchms, pyOpenMS
#### 🏥 **Clinical Research & Precision Medicine** (8 skills)
- Clinical databases: via Database Lookup (ClinicalTrials.gov, ClinVar, ClinPGx, COSMIC, FDA, cBioPortal, Monarch, and more)
- Cancer genomics: DepMap (cancer dependency scores, drug sensitivity)
- Cancer imaging: Imaging Data Commons (NCI radiology & pathology datasets via idc-index)
- Healthcare AI: PyHealth, NeuroKit2, Clinical Decision Support
- Clinical documentation: Clinical Reports, Treatment Plans
#### 🖼️ **Medical Imaging & Digital Pathology** (3 skills)
- DICOM processing: pydicom
- Whole slide imaging: histolab, PathML
#### 🧠 **Neuroscience & Electrophysiology** (2 skills)
- Data standards: BIDS (Brain Imaging Data Structure for neuroscience and biomedical datasets)
- Neural recordings: Neuropixels-Analysis (extracellular spikes, silicon probes, spike sorting)
#### 🤖 **Machine Learning & AI** (14 core skills)
- Deep learning: PyTorch Lightning, Transformers, Stable Baselines3, PufferLib
- Classical ML: scikit-learn, scikit-survival, SHAP
- Time series: aeon, TimesFM (Google's zero-shot foundation model for univariate forecasting)
- Bayesian methods: PyMC
- Optimization: PyMOO
- Graph ML: Torch Geometric
- Dimensionality reduction: UMAP-learn
- Statistical modeling: statsmodels
#### 🔮 **Materials Science, Chemistry & Physics** (7 skills)
- Materials: Pymatgen
- Metabolic modeling: COBRApy
- Astronomy: Astropy
- Quantum computing: Cirq, PennyLane, Qiskit, QuTiP
#### ⚙️ **Engineering & Simulation** (4 skills)
- Numerical computing: MATLAB/Octave
- Computational fluid dynamics: FluidSim
- Discrete-event simulation: SimPy
- Symbolic math: SymPy
#### 📊 **Data Analysis & Visualization** (21 skills)
- Visualization: Matplotlib, Seaborn, Scientific Visualization
- Geospatial analysis: GeoPandas, GeoMaster (remote sensing, GIS, satellite imagery, spatial ML, 500+ examples)
- Data processing: Dask, Polars, Vaex
- Network analysis: NetworkX
- Document processing: LiteParse (local PDF/document parsing with bounding boxes and OCR), MarkItDown, PDF, DOCX, PPTX, and XLSX
- Infographics: Infographics (AI-powered professional infographic creation)
- Diagrams: Markdown & Mermaid Writing (text-based diagrams as default documentation standard)
- Exploratory data analysis: EDA workflows
- Statistical analysis: Statistical Analysis workflows
- Experimental design: Experimental Design (randomization, blocking, factorial/fractional-factorial DOE, crossover, cluster, sequential designs; pyDOE3)
- Statistical power: Statistical Power (sample-size & power for t-tests, ANOVA, proportions, correlation, regression — closed-form plus simulation-based for GLMs, mixed models, and cluster designs)
#### 🧪 **Laboratory Automation** (6 skills)
- Liquid handling: PyLabRobot and Opentrons
- Cloud lab: Ginkgo Cloud Lab (protein expression & purification across cell-free/E. coli/Pichia, IVT RNA synthesis, thermal shift and Echo-MS assays, SPR onboarding, fluorescent pixel art via autonomous RAC infrastructure)
- Protocol management: Protocols.io
- LIMS integration: Benchling, LabArchives
#### 🔬 **Multi-omics & Systems Biology** (4 skills)
- Pathway analysis: via Database Lookup (KEGG, Reactome, STRING) and PrimeKG
- Multi-omics: HypoGeniC
- Data management: LaminDB
#### 🧬 **Protein Engineering & Design** (4 skills)
- Protein language models: ESM
- Glycoengineering: Glycoengineering (N/O-glycosylation prediction, therapeutic antibody optimization)
- Cloud laboratory platform: Adaptyv (automated protein testing and validation)
- Cloud structure & design platform: Tamarind (managed-GPU access to AlphaFold, Boltz, Chai, ESMFold, RFdiffusion, ProteinMPNN, BoltzGen, antibody/nanobody design, DiffDock/Vina docking, binding affinity, and MSA generation via REST API or MCP)
#### 📚 **Scientific Communication** (26 skills)
- Literature: Paper Lookup (PubMed, PMC, bioRxiv, medRxiv, arXiv, OpenAlex, Crossref, Semantic Scholar, CORE, Unpaywall), Literature Review, Paperzilla
- Advanced paper search: BGPT Paper Search (25+ structured fields per paper — methods, results, sample sizes, quality scores — from full text, not just abstracts)
- Web search: Parallel Web, Exa Search, and Research Lookup
- Research notebooks: Open Notebook (self-hosted NotebookLM alternative — PDFs, videos, audio, web pages; 16+ AI providers; multi-speaker podcast generation)
- Writing: Scientific Writing, Peer Review
- Document processing: LiteParse, PDF, DOCX, PPTX, XLSX, and MarkItDown
- Publishing and paper workflows: Venue Templates
- Presentations: Scientific Slides, LaTeX Posters, PPTX Posters
- Diagrams: Scientific Schematics, Markdown & Mermaid Writing
- Infographics: Infographics (10 types, 8 styles, colorblind-safe palettes)
- Citations: Citation Management, pyzotero
- Illustration: Generate Image (AI image generation with FLUX.2 Pro and Gemini 3 Pro (Nano Banana Pro))
#### 🔬 **Scientific Databases & Data Access** (6 skills → 100+ databases total)
> A unified database-lookup skill provides deterministic REST API access to 78 public databases across all domains, with retrieval contracts, pagination/count reconciliation, and endpoint provenance. Dedicated skills cover specialized data platforms. Multi-database packages like BioServices (~40 bioinformatics services), BioPython (38 NCBI sub-databases via Entrez), and gget (20+ genomics databases) add further coverage.
- Unified access: Database Lookup (78 databases spanning chemistry, genomics, clinical, pathways, patents, economics, and more — PubChem, ChEMBL, UniProt, PDB, AlphaFold, KEGG, Reactome, STRING, ClinVar, COSMIC, ClinicalTrials.gov, FDA, FRED, USPTO, SEC EDGAR, and dozens more — with auditable filters and provenance)
- Cancer genomics: DepMap (cancer cell line dependencies, drug sensitivity, gene effect profiles)
- Cancer imaging: Imaging Data Commons (NCI radiology & pathology datasets via idc-index)
- Knowledge graph: PrimeKG (precision medicine knowledge graph — genes, drugs, diseases, phenotypes)
- Fiscal data: U.S. Treasury Fiscal Data (national debt, Treasury statements, auctions, exchange rates)
- Scientific ML resource catalog: Hugging Science (curated index of datasets, models, blog posts, and interactive Spaces across 17 scientific domains — astronomy, biology, chemistry, climate, genomics, materials science, medicine, physics, scientific reasoning, and more — with usage patterns for `datasets`, `transformers`, and `gradio_client`)
#### 🔧 **Infrastructure & Platforms** (11 skills)
- Cloud compute: Modal
- GPU acceleration: Optimize for GPU (CuPy, Numba CUDA, Warp, cuDF, cuML, cuGraph, KvikIO, cuCIM, cuxfilter, cuVS, cuSpatial, RAFT)
- Genomics platforms: DNAnexus, LatchBio
- Workflow engines: Nextflow (build/run/debug Nextflow & nf-core pipelines — DSL2 modules, executors/containers, HPC/cloud scaling) and pacsomatic (operator toolkit for the nf-core/pacsomatic tumor-normal somatic variant-calling workflow)
- Microscopy: OMERO
- Automation: Opentrons
- Resource detection: Get Available Resources
- Workflow mining: Autoskill (local screenpipe-based repeated workflow detection and skill drafting)
- Agent platform development: Pi Agent (using Pi as a terminal coding harness and building on it with SDK, RPC/JSONL, extensions, custom providers/models, packages, TUI components, and session tooling)
#### 🎓 **Research Methodology & Planning** (12 skills)
- Ideation: Scientific Brainstorming, Hypothesis Generation
- Autonomous optimization: Arbor (Hypothesis Tree Refinement — iteratively improve a code/model/agent-harness/data artifact against a dev evaluator while a held-out test gate guards against overfitting)
- Critical analysis: Scientific Critical Thinking, Scholar Evaluation
- Scenario analysis: What-If Oracle (4–6 branch possibility exploration, contingency planning, decision stress-testing)
- Multi-perspective deliberation: Consciousness Council (diverse expert viewpoints, devil's advocate analysis)
- Cognitive profiling: DHDNA Profiler (extract thinking patterns and cognitive signatures from any text)
- Funding: Research Grants
- Discovery: Research Lookup, Paper Lookup (10 academic databases)
- Market analysis: Market Research Reports
#### ⚖️ **Regulatory & Standards** (1 skill)
- Medical device standards: ISO 13485 Certification
> 📖 **For complete details on all skills**, see [docs/skills.md](docs/skills.md)
> 💡 **Looking for practical examples?** Check out [docs/examples.md](docs/examples.md) for comprehensive workflow examples across all scientific domains.
---
## 🤝 Contributing
We welcome contributions to expand and improve this scientific skills repository!
For detailed instructions on adding or updating a skill, see [CONTRIBUTING.md](CONTRIBUTING.md). The guide covers repository structure, required `SKILL.md` frontmatter, Agent Skills specification requirements, versioning, validation, security scanning, and pull request expectations.
### Ways to Contribute
✨ **Add New Skills**
- Create skills for additional scientific packages or databases
- Add integrations for scientific platforms and tools
📚 **Improve Existing Skills**
- Enhance documentation with more examples and use cases
- Add new workflows and reference materials
- Improve code examples and scripts
- Fix bugs or update outdated information
🐛 **Report Issues**
- Submit bug reports with detailed reproduction steps
- Suggest improvements or new features
### How to Contribute
1. **Fork** the repository
2. **Create** a feature branch (`git checkout -b feature/amazing-skill`)
3. **Follow** [CONTRIBUTING.md](CONTRIBUTING.md) and the existing directory structure
4. **Ensure** all new skills include valid `SKILL.md` files with required frontmatter and `metadata.version`
5. **Test** your examples and workflows thoroughly
6. **Commit** your changes (`git commit -m 'Add amazing skill'`)
7. **Push** to your branch (`git push origin feature/amazing-skill`)
8. **Submit** a pull request with a clear description of your changes
### Contribution Guidelines
✅ **Adhere to the [Agent Skills Specification](https://agentskills.io/specification)** — Every skill must follow the official spec (valid `SKILL.md` frontmatter, naming conventions, directory structure)
✅ Include a quoted `metadata.version` value in every `SKILL.md`
✅ Increment `metadata.version` when updating an existing skill
✅ Maintain consistency with existing skill documentation format
✅ Ensure all code examples are tested and functional
✅ Follow scientific best practices in examples and workflows
✅ Update relevant documentation when adding new capabilities
✅ Provide clear comments and docstrings in code
✅ Include references to official documentation
### Security Scanning
All skills in this repository are security-scanned using [Cisco AI Defense Skill Scanner](https://github.com/cisco-ai-defense/skill-scanner), an open-source tool that detects prompt injection, data exfiltration, and malicious code patterns in Agent Skills.
If you are contributing a new skill, we recommend running the scanner locally before submitting a pull request:
```bash
uv pip install cisco-ai-skill-scanner
skill-scanner scan /path/to/your/skill --use-behavioral
```
> **Note:** A clean scan result reduces noise in review, but does not guarantee a skill is free of all risk. Contributed skills are also reviewed manually before merging.
### Recognition
Contributors are recognized in our community and may be featured in:
- Repository contributors list
- Special mentions in release notes
- K-Dense community highlights
Your contributions help make scientific computing more accessible and enable researchers to leverage AI tools more effectively!
### Support Open Source
This project builds on 50+ amazing open source projects. If you find value in these skills, please consider [supporting the projects we depend on](docs/open-source-sponsors.md).
---
## 🔧 Troubleshooting
### Common Issues
**Problem: Skills not loading**
- Verify skill folders are in the correct directory (see [Getting Started](#-getting-started))
- Each skill folder must contain a `SKILL.md` file
- Restart your agent/IDE after copying skills
- In Cursor, check Settings → Rules to confirm skills are discovered
**Problem: Missing Python dependencies**
- Solution: Check the specific `SKILL.md` file for required packages
- Install dependencies: `uv pip install package-name`
**Problem: API rate limits**
- Solution: Many databases have rate limits. Review the specific database documentation
- Consider implementing caching or batch requests
**Problem: Authentication errors**
- Solution: Some services require API keys. Check the `SKILL.md` for authentication setup
- Verify your credentials and permissions
**Problem: Outdated examples**
- Solution: Report the issue via GitHub Issues
- Check the official package documentation for updated syntax
**Problem: `gh skill install` or docs link to `scientific-skills/` fails (v2.43.0+)**
- As of v2.43.0, skills live under `skills/` (not `scientific-skills/`) to match the Agent Skills layout expected by GitHub CLI
- Update manual copy paths, bookmarks, and citations from `scientific-skills/<name>` to `skills/<name>`
- Re-run `gh skill install K-Dense-AI/scientific-agent-skills` after pulling the latest release
---
## ❓ FAQ
### General Questions
**Q: Is this free to use?**
A: Yes! This repository is MIT licensed. However, each individual skill has its own license specified in the `license` metadata field within its `SKILL.md` file—be sure to review and comply with those terms.
**Q: Why are all skills grouped together instead of separate packages?**
A: We believe good science in the age of AI is inherently interdisciplinary. Bundling all skills together makes it trivial for you (and your agent) to bridge across fields—e.g., combining genomics, cheminformatics, clinical data, and machine learning in one workflow—without worrying about which individual skills to install or wire together.
**Q: Can I use this for commercial projects?**
A: The repository itself is MIT licensed, which allows commercial use. However, individual skills may have different licenses—check the `license` field in each skill's `SKILL.md` file to ensure compliance with your intended use.
**Q: Do all skills have the same license?**
A: No. Each skill has its own license specified in the `license` metadata field within its `SKILL.md` file. These licenses may differ from the repository's MIT License. Users are responsible for reviewing and adhering to the license terms of each individual skill they use.
**Q: How often is this updated?**
A: We regularly update skills to reflect the latest versions of packages and APIs. Major updates are announced in release notes.
**Q: Can I use this with other AI models?**
A: The skills follow the open [Agent Skills](https://agentskills.io/) standard and work with any compatible agent, including Cursor, Claude Code, Codex, Google Antigravity, OpenClaw, NVIDIA NemoClaw, Hermes, and Pi.
### Installation & Setup
**Q: Do I need all the Python packages installed?**
A: No! Only install the packages you need. Each skill specifies its requirements in its `SKILL.md` file.
**Q: What if a skill doesn't work?**
A: First check the [Troubleshooting](#-troubleshooting) section. If the issue persists, file an issue on GitHub with detailed reproduction steps.
**Q: Do the skills work offline?**
A: Database skills require internet access to query APIs. Package skills work offline once Python dependencies are installed.
### Contributing
**Q: Can I contribute my own skills?**
A: Absolutely! We welcome contributions. See the [Contributing](#-contributing) section for guidelines and best practices.
**Q: How do I report bugs or suggest features?**
A: Open an issue on GitHub with a clear description. For bugs, include reproduction steps and expected vs actual behavior.
---
## 💬 Support
Need help? Here's how to get support:
- 📖 **Documentation**: Check the relevant `SKILL.md` and `references/` folders
- 🐛 **Bug Reports**: [Open an issue](https://github.com/K-Dense-AI/scientific-agent-skills/issues)
- 💡 **Feature Requests**: [Submit a feature request](https://github.com/K-Dense-AI/scientific-agent-skills/issues/new)
- 📣 **Updates and demos**: Follow [X](https://x.com/k_dense_ai), [LinkedIn](https://www.linkedin.com/company/k-dense-inc), and [YouTube](https://www.youtube.com/@K-Dense-Inc) to keep up with new skills, tutorials, and Scientific Agent Skills releases
- 💼 **Enterprise Support**: Contact [K-Dense](https://k-dense.ai/) for commercial support
---
## 📖 Citation
If you use Scientific Agent Skills in your research or project, please cite the overall collection and, when relevant, the individual skill or skills that materially supported your work.
The collection citation helps others find the repository, understand the broader skill ecosystem used in your workflow, and credit the maintenance effort behind Scientific Agent Skills. Individual skill citations give more precise credit for the specific package, database, or workflow guidance your agent used.
Recommended practice:
- Always cite **Scientific Agent Skills** using one of the formats below.
- Also cite each individual skill that directly contributed to your analysis, code, figures, reports, or research workflow.
- If a skill wraps or documents an external package, database, or platform, cite that upstream project too when your field's norms require it.
### Collection Citation
#### BibTeX
```bibtex
@software{scientific_agent_skills_2026,
author = {{K-Dense Inc.}},
title = {Scientific Agent Skills: A Comprehensive Collection of Scientific Tools for AI Agents},
year = {2026},
url = {https://github.com/K-Dense-AI/scientific-agent-skills},
note = {148 skills covering databases, packages, integrations, and analysis tools}
}
```
#### APA
```
K-Dense Inc. (2026). Scientific Agent Skills: A comprehensive collection of scientific tools for AI agents [Computer software]. https://github.com/K-Dense-AI/scientific-agent-skills
```
#### MLA
```
K-Dense Inc. Scientific Agent Skills: A Comprehensive Collection of Scientific Tools for AI Agents. 2026, github.com/K-Dense-AI/scientific-agent-skills.
```
#### Plain Text
```
Scientific Agent Skills by K-Dense Inc. (2026)
Available at: https://github.com/K-Dense-AI/scientific-agent-skills
```
### Individual Skill Citation
When citing a specific skill, include the skill name, version from `metadata.version` in that skill's `SKILL.md`, and the direct skill URL. For example:
```bibtex
@software{scientific_agent_skills_astropy_2026,
author = {{K-Dense Inc.}},
title = {Astropy Skill for Scientific Agent Skills},
year = {2026},
url = {https://github.com/K-Dense-AI/scientific-agent-skills/tree/main/skills/astropy},
note = {Version 1.0, part of Scientific Agent Skills}
}
```
Plain text format:
```text
Astropy skill for Scientific Agent Skills, version 1.0.
K-Dense Inc. (2026).
https://github.com/K-Dense-AI/scientific-agent-skills/tree/main/skills/astropy
```
We appreciate acknowledgment in publications, presentations, or projects that benefit from these skills.
---
## 📄 License
This project is licensed under the **MIT License**.
**Copyright © 2026 K-Dense Inc.** ([k-dense.ai](https://k-dense.ai/))
### Key Points:
- ✅ **Free for any use** (commercial and noncommercial)
- ✅ **Open source** - modify, distribute, and use freely
- ✅ **Permissive** - minimal restrictions on reuse
- ⚠️ **No warranty** - provided "as is" without warranty of any kind
See [LICENSE.md](LICENSE.md) for full terms.
### Individual Skill Licenses
> ⚠️ **Important**: Each skill has its own license specified in the `license` metadata field within its `SKILL.md` file. These licenses may differ from the repository's MIT License and may include additional terms or restrictions. **Users are responsible for reviewing and adhering to the license terms of each individual skill they use.**
README.wehub.md
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目：`K-Dense-AI/scientific-agent-skills`
- 原始仓库：https://github.com/K-Dense-AI/scientific-agent-skills
- 导入方式：上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
SECURITY.md
+3767
View File
File diff suppressed because it is too large Load Diff
docs/examples.md
+3699
View File
File diff suppressed because it is too large Load Diff
docs/k-dense-web.gif
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 22 MiB

docs/open-source-sponsors.md
+186
View File
@@ -0,0 +1,186 @@
# Support the Open Source Projects We Depend On
Scientific Agent Skills is built on the shoulders of giants. The 148 skills in this repository leverage dozens of incredible open source projects created and maintained by dedicated developers and research communities around the world.
**If you find value in these skills, please consider supporting the underlying open source projects that make them possible.**
---
## How to Support Open Source
1. **Star repositories** on GitHub - It's free and helps projects gain visibility
2. **Sponsor maintainers** directly through GitHub Sponsors, Open Collective, or project-specific donation pages
3. **Contribute** code, documentation, or bug reports
4. **Cite** projects in your publications
5. **Share** projects with colleagues
---
## Featured Projects by Domain
### Bioinformatics & Genomics
| Project | Description | Links |
|---------|-------------|-------|
| **Biopython** | Computational molecular biology toolkit | [GitHub](https://github.com/biopython/biopython) - [Donate](https://numfocus.org/donate-to-biopython) |
| **Scanpy** | Single-cell analysis in Python | [GitHub](https://github.com/scverse/scanpy) - [scverse](https://scverse.org/) |
| **AnnData** | Annotated data matrices for single-cell | [GitHub](https://github.com/scverse/anndata) |
| **scvi-tools** | Deep learning for single-cell omics | [GitHub](https://github.com/scverse/scvi-tools) |
| **Arboreto** | Gene regulatory network inference | [GitHub](https://github.com/aertslab/arboreto) |
| **pysam** | SAM/BAM/VCF file interface | [GitHub](https://github.com/pysam-developers/pysam) |
| **scikit-bio** | Bioinformatics library | [GitHub](https://github.com/scikit-bio/scikit-bio) |
| **gget** | Gene and transcript info retrieval | [GitHub](https://github.com/pachterlab/gget) |
| **deepTools** | Tools for deep-sequencing data | [GitHub](https://github.com/deeptools/deepTools) |
| **ETE Toolkit** | Phylogenetic tree analysis | [GitHub](https://github.com/etetoolkit/ete) |
### Cheminformatics & Drug Discovery
| Project | Description | Links |
|---------|-------------|-------|
| **RDKit** | Cheminformatics toolkit | [GitHub](https://github.com/rdkit/rdkit) - [Donate](https://github.com/sponsors/rdkit) |
| **Datamol** | Molecular manipulation made easy | [GitHub](https://github.com/datamol-io/datamol) |
| **DeepChem** | Deep learning for chemistry | [GitHub](https://github.com/deepchem/deepchem) |
| **TorchDrug** | Drug discovery with PyTorch | [GitHub](https://github.com/DeepGraphLearning/torchdrug) |
| **molfeat** | Molecular featurization | [GitHub](https://github.com/datamol-io/molfeat) |
| **MedChem** | Medicinal chemistry filters | [GitHub](https://github.com/datamol-io/medchem) |
| **PyTDC** | Therapeutics Data Commons | [GitHub](https://github.com/mims-harvard/TDC) |
### Proteomics & Mass Spectrometry
| Project | Description | Links |
|---------|-------------|-------|
| **matchms** | Mass spectrometry data processing | [GitHub](https://github.com/matchms/matchms) |
| **pyOpenMS** | Mass spectrometry toolkit | [GitHub](https://github.com/OpenMS/OpenMS) |
### Machine Learning & AI
| Project | Description | Links |
|---------|-------------|-------|
| **PyTorch Lightning** | Deep learning framework | [GitHub](https://github.com/Lightning-AI/pytorch-lightning) - [Sponsor](https://github.com/sponsors/Lightning-AI) |
| **Transformers** | State-of-the-art NLP | [GitHub](https://github.com/huggingface/transformers) |
| **scikit-learn** | Machine learning in Python | [GitHub](https://github.com/scikit-learn/scikit-learn) - [Donate](https://numfocus.org/donate-to-scikit-learn) |
| **PyTorch Geometric** | Geometric deep learning | [GitHub](https://github.com/pyg-team/pytorch_geometric) |
| **PyMC** | Probabilistic programming | [GitHub](https://github.com/pymc-devs/pymc) - [Donate](https://numfocus.org/donate-to-pymc) |
| **SHAP** | Model interpretability | [GitHub](https://github.com/shap/shap) |
| **Stable Baselines3** | Reinforcement learning | [GitHub](https://github.com/DLR-RM/stable-baselines3) |
| **scikit-survival** | Survival analysis | [GitHub](https://github.com/sebp/scikit-survival) |
| **aeon** | Time series ML toolkit | [GitHub](https://github.com/aeon-toolkit/aeon) |
| **PyMOO** | Multi-objective optimization | [GitHub](https://github.com/anyoptimization/pymoo) |
| **UMAP** | Dimensionality reduction | [GitHub](https://github.com/lmcinnes/umap) |
### Data Science & Visualization
| Project | Description | Links |
|---------|-------------|-------|
| **Matplotlib** | Plotting library | [GitHub](https://github.com/matplotlib/matplotlib) - [Donate](https://numfocus.org/donate-to-matplotlib) |
| **Seaborn** | Statistical visualization | [GitHub](https://github.com/mwaskom/seaborn) |
| **Plotly** | Interactive visualizations | [GitHub](https://github.com/plotly/plotly.py) |
| **NetworkX** | Network analysis | [GitHub](https://github.com/networkx/networkx) - [Donate](https://numfocus.org/donate-to-networkx) |
| **SymPy** | Symbolic mathematics | [GitHub](https://github.com/sympy/sympy) - [Donate](https://numfocus.org/donate-to-sympy) |
| **statsmodels** | Statistical modeling | [GitHub](https://github.com/statsmodels/statsmodels) |
| **GeoPandas** | Geospatial data in Python | [GitHub](https://github.com/geopandas/geopandas) |
| **Polars** | Fast DataFrame library | [GitHub](https://github.com/pola-rs/polars) |
| **Dask** | Parallel computing | [GitHub](https://github.com/dask/dask) - [Donate](https://numfocus.org/donate-to-dask) |
| **Vaex** | Out-of-core DataFrames | [GitHub](https://github.com/vaexio/vaex) |
### Medical Imaging & Digital Pathology
| Project | Description | Links |
|---------|-------------|-------|
| **pydicom** | DICOM file handling | [GitHub](https://github.com/pydicom/pydicom) |
| **histolab** | Digital pathology preprocessing | [GitHub](https://github.com/histolab/histolab) |
| **PathML** | Pathology ML toolkit | [GitHub](https://github.com/Dana-Farber-AIOS/pathml) |
### Healthcare & Clinical
| Project | Description | Links |
|---------|-------------|-------|
| **PyHealth** | Healthcare AI toolkit | [GitHub](https://github.com/sunlabuiuc/PyHealth) |
| **NeuroKit2** | Neurophysiological signal processing | [GitHub](https://github.com/neuropsychology/NeuroKit) |
### Materials Science & Physics
| Project | Description | Links |
|---------|-------------|-------|
| **Pymatgen** | Materials analysis | [GitHub](https://github.com/materialsproject/pymatgen) |
| **COBRApy** | Metabolic modeling | [GitHub](https://github.com/opencobra/cobrapy) |
| **Astropy** | Astronomy library | [GitHub](https://github.com/astropy/astropy) - [Donate](https://numfocus.org/donate-to-astropy) |
### Quantum Computing
| Project | Description | Links |
|---------|-------------|-------|
| **Qiskit** | IBM quantum computing SDK | [GitHub](https://github.com/Qiskit/qiskit) |
| **Cirq** | Google quantum computing | [GitHub](https://github.com/quantumlib/Cirq) |
| **PennyLane** | Quantum ML library | [GitHub](https://github.com/PennyLaneAI/pennylane) |
| **QuTiP** | Quantum toolbox in Python | [GitHub](https://github.com/qutip/qutip) |
### Simulation & Engineering
| Project | Description | Links |
|---------|-------------|-------|
| **SimPy** | Discrete-event simulation | [GitHub](https://github.com/TeamSim/SimPy) |
| **FluidSim** | CFD framework | [GitHub](https://github.com/fluiddyn/fluidsim) |
### Laboratory & Automation
| Project | Description | Links |
|---------|-------------|-------|
| **PyLabRobot** | Lab automation control | [GitHub](https://github.com/PyLabRobot/pylabrobot) |
### Protein Engineering
| Project | Description | Links |
|---------|-------------|-------|
| **ESM** | Evolutionary scale modeling | [GitHub](https://github.com/Biohub/esm) |
### Data Formats & I/O
| Project | Description | Links |
|---------|-------------|-------|
| **Zarr** | Chunked array storage | [GitHub](https://github.com/zarr-developers/zarr-python) |
| **FlowIO** | Flow cytometry I/O | [GitHub](https://github.com/whitews/FlowIO) |
---
## NumFOCUS-Sponsored Projects
Many of the projects above are sponsored by [NumFOCUS](https://numfocus.org/), a nonprofit supporting open source scientific computing. Consider [donating to NumFOCUS](https://numfocus.org/donate) to support the broader ecosystem.
**NumFOCUS-sponsored projects in this collection:**
- Biopython
- scikit-learn
- Matplotlib
- NetworkX
- SymPy
- Dask
- Astropy
- PyMC
---
## scverse Ecosystem
The [scverse](https://scverse.org/) consortium maintains foundational tools for single-cell omics:
- Scanpy
- AnnData
- scvi-tools
- And more
Consider supporting their mission to advance single-cell research.
---
## A Note from K-Dense
At K-Dense, we believe in giving back to the communities that make our work possible. We encourage all users of Scientific Agent Skills to:
1. **Acknowledge** these projects when you use them in research
2. **Contribute** back improvements when you can
3. **Support** maintainers financially if you derive commercial value
The open source scientific Python ecosystem is a shared resource. Let's keep it thriving together.
---
*This list is not exhaustive. Many other excellent open source projects power the skills in this repository. If you notice a project that should be listed here, please open a PR!*
docs/skills.md
+222
View File
@@ -0,0 +1,222 @@
# Scientific Skills
## Scientific Databases & Data Access
- **Database Lookup** - Deterministically query 78 public scientific, biomedical, materials science, regulatory, finance, and demographics databases through documented REST APIs. Adds retrieval contracts, endpoint provenance, pagination/count reconciliation, rate-limit guidance, and untrusted-response handling for reproducible lookups of compounds, drugs, proteins, genes, pathways, variants, clinical trials, patents, SEC filings, economic indicators, structures, astronomical objects, earthquakes, weather, and related database-backed scientific facts
- **DepMap** - Query the Cancer Dependency Map (DepMap) for cancer cell line gene dependency scores (CRISPR Chronos), drug sensitivity data, and gene effect profiles. Use for identifying cancer-specific vulnerabilities, synthetic lethal interactions, and validating oncology drug targets
- **Imaging Data Commons** - Query and download public cancer imaging data from NCI Imaging Data Commons using idc-index. Use for accessing large-scale radiology (CT, MR, PET) and pathology datasets for AI training or research. No authentication required. Query by metadata, visualize in browser, check licenses
- **PrimeKG** - Query the Precision Medicine Knowledge Graph (PrimeKG) for multiscale biological data including genes, drugs, diseases, phenotypes, and more. Integrates 20+ biomedical resources into a single knowledge graph for drug repurposing, disease mechanism exploration, and target identification
- **U.S. Treasury Fiscal Data** - Free, open REST API from the U.S. Department of the Treasury providing 54 datasets and 179 data tables covering federal fiscal data. No API key required. Access national debt (Debt to the Penny back to 1993, Historical Debt back to 1790), Daily Treasury Statements (TGA balances, deposits/withdrawals), Monthly Treasury Statements (federal budget receipts and outlays), Treasury securities auctions data (bills, notes, bonds, TIPS, FRNs since 1979), average interest rates on Treasury securities, Treasury reporting exchange rates (quarterly for 170+ currencies), I Bond and savings bond rates, TIPS/CPI data, and more. Supports filtering, sorting, pagination, and CSV/XML/JSON output formats
- **Hugging Science** - Curated, LLM-friendly catalog of scientific datasets, models, blog posts, and interactive Spaces hosted on Hugging Face, spanning 17 scientific domains (astronomy, benchmark, biology, biotechnology, chemistry, climate, conservation, earth-science, ecology, energy, engineering, genomics, materials-science, mathematics, medicine, physics, scientific-reasoning). Discovery happens via huggingscience.co (with `llms.txt`, `llms-full.txt`, and per-topic markdown files designed for agent consumption); usage goes through standard Hugging Face APIs. Includes a bundled `fetch_catalog.py` script for filtered access by topic, type (datasets/models/blogs/spaces), or free-text search, plus reference guides for loading datasets via the `datasets` library (with streaming for billion-token corpora), running models via `transformers` or the Inference API/Providers (handling `trust_remote_code` requirements for custom architectures like Evo-2), and calling Spaces via `gradio_client` (with a worked BoltzGen example for protein binder design). Authenticates gated resources via `HF_TOKEN` from `.env`. Use cases: discovering the right dataset/model for a scientific ML task without trawling the broader Hub, fine-tuning on curated scientific data, citing methodology blogs from dataset/model authors, running interactive scientific demos (binder design, theorem proving, weather modeling) without local GPU setup, and bridging from "I need a model for protein/genome/molecule/climate/materials/astronomy" to working code
## Scientific Integrations
### Laboratory Information Management Systems (LIMS) & R&D Platforms
- **Benchling Integration** - Toolkit for integrating with Benchling's R&D platform, providing programmatic access to laboratory data management including registry entities (DNA sequences, proteins), inventory systems (samples, containers, locations), electronic lab notebooks (entries, protocols), workflows (tasks, automation), and data exports using Python SDK and REST API
### Cloud Platforms for Genomics & Biomedical Data
- **DNAnexus Integration** - Comprehensive toolkit for working with the DNAnexus cloud platform for genomics and biomedical data analysis. Covers building and deploying apps/applets (Python/Bash), managing data objects (files, records, databases), running analyses and workflows, using the dxpy Python SDK, and configuring app metadata and dependencies (dxapp.json setup, system packages, Docker, assets). Enables processing of FASTQ/BAM/VCF files, bioinformatics pipelines, job execution, workflow orchestration, and platform operations including project management and permissions
### Laboratory Automation
- **Opentrons Integration** - Toolkit for creating, editing, and debugging Opentrons Python Protocol API v2 protocols for laboratory automation using Flex and OT-2 robots. Enables automated liquid handling, pipetting workflows, hardware module control (thermocycler, temperature, magnetic, heater-shaker, absorbance plate reader), labware management, and complex protocol development for biological and chemical experiments
- **Ginkgo Cloud Lab** - Submit and manage protocols on Ginkgo Bioworks Cloud Lab (cloud.ginkgo.bio), a web-based interface for autonomous lab execution on Reconfigurable Automation Carts (RACs). Covers the full catalog: protein expression and purification across cell-free, E. coli, and Pichia systems (HiBiT/luminescence, A280 yield, and LabChip purity/size tiers, plus minibinder screening), cell-free validation ($39) and 24-condition DoE optimization ($199), IVT mRNA/circRNA synthesis with qPCR, thermal shift / developability assays, Echo-MS enzyme detection and analyte method onboarding, SPR target onboarding for kinetics, and fluorescent pixel art ($25/plate, 7-color E. coli palette). Includes the EstiMate AI agent for custom protocol feasibility and pricing
### Electronic Lab Notebooks (ELN)
- **LabArchives Integration** - Toolkit for interacting with LabArchives Electronic Lab Notebook (ELN) REST API. Provides programmatic access to notebooks (backup, retrieval, management), entries (creation, comments, attachments), user authentication, site reports and analytics, and third-party integrations (Protocols.io, GraphPad Prism, SnapGene, Geneious, Jupyter, REDCap). Includes Python scripts for configuration setup, notebook operations, and entry management. Supports multi-regional API endpoints (US, UK, Australia) and OAuth authentication
- **Open Notebook** - Self-hosted, open-source alternative to Google NotebookLM for AI-powered research and document analysis. Organizes research materials into notebooks, ingests diverse content sources (PDFs, videos, audio, web pages, Office documents), generates AI-powered notes and summaries, creates multi-speaker podcasts from research, enables document chat with context-aware AI, and searches across materials with full-text and vector search. Supports 16+ AI providers including OpenAI, Anthropic, Google, Ollama, Groq, and Mistral with complete data privacy through self-hosting
### Workflow Platforms & Cloud Execution
- **LatchBio Integration** - Integration with the Latch platform for building, deploying, and executing bioinformatics workflows. Provides comprehensive support for creating serverless bioinformatics pipelines using Python decorators, deploying Nextflow/Snakemake pipelines, managing cloud data (LatchFile, LatchDir) and structured Registry (Projects, Tables, Records), configuring computational resources (CPU, GPU, memory, storage), and using pre-built Latch Verified workflows (RNA-seq, AlphaFold, DESeq2, single-cell analysis, CRISPR editing). Enables automatic containerization, UI generation, workflow versioning, and execution on scalable cloud infrastructure with comprehensive data management
- **Nextflow** - Build, run, and debug Nextflow data pipelines and nf-core workflows end to end. Covers writing and testing DSL2 modules/subworkflows (processes, channels, operators, nf-test), running community pipelines (nf-core/rnaseq, nf-core/sarek), authoring samplesheets and `nextflow.config`, configuring executors and containers (Docker, Singularity/Apptainer, Conda, Wave), scaling to HPC/SLURM or cloud (AWS Batch, Google Batch, Azure, Kubernetes), and debugging failed or `-resume` runs. Use for any reproducible scientific/bioinformatics workflow work and for authoring nf-core-compliant pipelines, modules, configs, and linting
- **pacsomatic** - Operator toolkit for running the nf-core/pacsomatic matched tumor-normal (somatic variant calling) workflow from BAM inputs. Validates run inputs, generates pacsomatic-compliant samplesheets, prepares reproducible Nextflow launch artifacts, runs locally or submits to schedulers (LSF/Slurm/PBS/SGE), and triages execution and startup failures. Use to prepare launch commands/scripts, perform dry-run checks, or troubleshoot pipeline and scheduler submission errors
### Microscopy & Bio-image Data
- **OMERO Integration** - Toolkit for interacting with OMERO microscopy data management systems using Python. Provides comprehensive access to microscopy images stored in OMERO servers, including dataset and screening data retrieval, pixel data analysis, annotation and metadata management, regions of interest (ROIs) creation and analysis, batch processing, OMERO.scripts development, and OMERO.tables for structured data storage. Essential for researchers working with high-content screening data, multi-dimensional microscopy datasets, or collaborative image repositories
### Protocol Management & Sharing
- **Protocols.io Integration** - Integration with protocols.io API for managing scientific protocols. Enables programmatic access to protocol discovery (search by keywords, DOI, category), protocol lifecycle management (create, update, publish with DOI), step-by-step procedure documentation, collaborative development with workspaces and discussions, file management (upload data, images, documents), experiment tracking and documentation, and data export. Supports OAuth authentication, protocol PDF generation, materials management, threaded comments, workspace permissions, and institutional protocol repositories. Essential for protocol standardization, reproducibility, lab knowledge management, and scientific collaboration
## Scientific Packages
### Bioinformatics & Genomics
- **AnnData** - Python package for handling annotated data matrices, specifically designed for single-cell genomics data. Provides efficient storage and manipulation of high-dimensional data with associated annotations (observations/cells and variables/genes). Key features include: HDF5-based h5ad file format for efficient I/O and compression, integration with pandas DataFrames for metadata, support for sparse matrices (scipy.sparse) for memory efficiency, layered data organization (X for main data matrix, obs for observation annotations, var for variable annotations, obsm/varm for multi-dimensional annotations, obsp/varp for pairwise matrices), and seamless integration with Scanpy, scvi-tools, and other single-cell analysis packages. Supports lazy loading, chunked operations, and conversion to/from other formats (CSV, HDF5, Zarr). Use cases: single-cell RNA-seq data management, multi-modal single-cell data (RNA+ATAC, CITE-seq), spatial transcriptomics, and any high-dimensional annotated data requiring efficient storage and manipulation
- **Arboreto** - Scalable Python GRN inference (Aerts Lab) using GRNBoost2 (gradient boosting) and GENIE3 (random forest) with Dask distributed execution. Accepts pandas DataFrames, dense NumPy arrays, or sparse CSC matrices (cells × genes). Features: local multi-core and cluster scheduling, TF-restricted search, `limit` for top-N links, and `diy()` for custom regressors. Core engine for pySCENIC (cisTarget pruning and regulon activity are downstream). Use for bulk or single-cell expression matrices when inferring TF–target regulatory links from co-expression patterns
- **BioPython** - Comprehensive Python library for computational biology and bioinformatics providing tools for sequence manipulation, database access, and biological data analysis. Key features include: sequence objects (Seq, SeqRecord, SeqIO) for DNA/RNA/protein sequences with biological alphabet validation, file format parsers (FASTA, FASTQ, GenBank, EMBL, Swiss-Prot, PDB, SAM/BAM, VCF, GFF), NCBI database access (Entrez Programming Utilities for PubMed, GenBank, BLAST, taxonomy), BLAST integration (running searches, parsing results), sequence alignment (pairwise and multiple sequence alignment with Bio.Align), phylogenetics (tree construction and manipulation with Bio.Phylo), population genetics (Hardy-Weinberg, F-statistics), protein structure analysis (PDB parsing, structure calculations), and statistical analysis tools. Supports integration with NumPy, pandas, and other scientific Python libraries. Use cases: sequence analysis, database queries, phylogenetic analysis, sequence alignment, file format conversion, and general bioinformatics workflows
- **BioServices** - Python library providing unified programmatic access to 40+ biological web services and databases. Supports major bioinformatics resources including KEGG (pathway and compound data), UniProt (protein sequences and annotations), ChEBI (chemical entities), ChEMBL (bioactive molecules), Reactome (pathways), IntAct (protein interactions), BioModels (biological models), and many others. Features consistent API across different services, automatic result caching, error handling and retry logic, support for both REST and SOAP web services, and conversion of results to Python objects (dictionaries, lists, BioPython objects). Handles authentication, rate limiting, and API versioning. Use cases: automated data retrieval from multiple biological databases, building bioinformatics pipelines, database integration workflows, and programmatic access to biological web resources without manual web browsing
- **Bulk RNA-seq** - End-to-end orchestrator for bulk RNA-seq differential-expression studies, chaining the repo's skills into one defensible pipeline: raw FASTQ -> QC/trimming (FastQC, fastp/Trim Galore) -> alignment and quantification (STAR, Salmon, featureCounts) -> a gene-level counts matrix -> differential expression (PyDESeq2) -> pathway/GSEA enrichment (Pathway Enrichment) -> publication figures (scientific-visualization). Offers two interchangeable upstream paths with a decision guide — nf-core/rnaseq driven through the Nextflow skill, or standalone STAR/Salmon recipes — plus the glue no single tool provides: experimental-design and QC guidance (replication, batch/confounding, strandedness, mapping-rate and PCA gates), a quant->counts bridge script (build_counts_matrix.py: aggregates Salmon quant.sf via pytximport, selects the correct STAR ReadsPerGene strand column, or parses featureCounts into a PyDESeq2-ready counts.csv plus metadata template), and a samplesheet/design validator (validate_samplesheet.py). Emphasizes reproducibility (pinned pipeline/tool/reference versions, containers) and correct statistics (integer counts, gene-ID mapping to symbols, FDR control). Use cases: going from sequencing reads to differentially expressed genes and enriched pathways, running or configuring nf-core/rnaseq, STAR/Salmon quantification, building a counts matrix for DESeq2, and scoping a complete reproducible bulk RNA-seq analysis. For single-cell RNA-seq use Scanpy instead
- **Cellxgene Census** - Python package for querying and analyzing versioned public single-cell and spatial transcriptomics data from CZ CELLxGENE Discover. Provides access to the current stable LTS Census release with 217M+ total cells, 125M+ unique cells, and 1,845 datasets, with standardized annotations and metadata across human, mouse, marmoset, rhesus macaque, and chimpanzee. Key features include: efficient TileDB-SOMA access for scalable metadata and expression queries, AnnData and Scanpy integration, source H5AD URI/download helpers, precomputed summary counts, embeddings, spatial Census access, and TileDB-SOMA-ML guidance for PyTorch loaders. Enables subsetting datasets by organism, cell type, tissue, disease, assay, or other metadata before downloading, reducing transfer and memory requirements. Use cases: large-scale single-cell analysis, spatial transcriptomics access, cell-type discovery, cross-dataset comparisons, reference atlas construction, and exploratory analysis of public Census data
- **deepTools** - Comprehensive suite of Python tools for exploring and visualizing next-generation sequencing (NGS) data, particularly ChIP-seq, RNA-seq, and ATAC-seq experiments. Provides command-line tools and Python API for processing BAM and bigWig files. Key features include: quality control metrics (plotFingerprint, plotCorrelation), coverage track generation (bamCoverage for creating bigWig files), matrix generation for heatmaps (computeMatrix, plotHeatmap, plotProfile), comparative analysis (multiBigwigSummary, plotPCA), and efficient handling of large files. Supports normalization methods, binning options, and various visualization outputs. Designed for high-throughput analysis workflows and publication-quality figure generation. Use cases: ChIP-seq peak visualization, RNA-seq coverage analysis, ATAC-seq signal tracks, comparative genomics, and NGS data exploration
- **FlowIO** - Python library for reading and manipulating Flow Cytometry Standard (FCS) files, the standard format for flow cytometry data. Provides efficient parsing of FCS files (versions 2.0, 3.0, 3.1), access to event data (fluorescence intensities, scatter parameters), metadata extraction (keywords, parameters, acquisition settings), and conversion to pandas DataFrames or NumPy arrays. Features include: support for large FCS files, handling of multiple data segments, access to text segments and analysis segments, and integration with flow cytometry analysis workflows. Enables programmatic access to flow cytometry data for downstream analysis, visualization, and machine learning applications. Use cases: flow cytometry data analysis, high-throughput screening, immune cell profiling, and automated processing of FCS files
- **gget** - Command-line tool and Python package for efficient querying of genomic databases with a simple, unified interface. Provides fast access to Ensembl (gene information and sequences), UniProt, NCBI BLAST/BLAT and viral sequence data, PDB/AlphaFold structures, CELLxGENE Census, OpenTargets, cBioPortal, COSMIC, Enrichr, Bgee, and 8cubeDB mouse specificity/expression metrics. Features include: single-command queries without complex API setup, automatic result formatting, batch query support, integration with pandas DataFrames, and support for both command-line and Python API usage. Optimized for speed and ease of use, making database queries accessible to users without extensive bioinformatics experience. Use cases: quick gene lookups, sequence retrieval, viral dataset downloads, enrichment analysis, disease/drug associations, protein structure access, and rapid database queries in bioinformatics workflows
- **geniml** - Genomic interval machine learning toolkit providing unsupervised methods for building ML models on BED files. Key capabilities include Region2Vec (word2vec-style embeddings of genomic regions and region sets using tokenization and neural language modeling), BEDspace (joint embeddings of regions and metadata labels using StarSpace for cross-modal queries), scEmbed (Region2Vec applied to single-cell ATAC-seq data generating cell-level embeddings for clustering and annotation with scanpy integration), consensus peak building (four statistical methods CC/CCF/ML/HMM for creating reference universes from BED collections), and comprehensive utilities (BBClient for BED caching, BEDshift for genomic randomization preserving context, evaluation metrics for embedding quality, Text2BedNN for neural search backends). Part of BEDbase ecosystem. Supports Python API and CLI workflows, pre-trained models on Hugging Face, and integration with gtars for tokenization. Use cases: region similarity searches, dimension reduction of chromatin accessibility data, scATAC-seq clustering and cell-type annotation, metadata-aware genomic queries, universe construction for standardized references, and any ML task requiring genomic region feature vectors
- **gtars** - High-performance Rust toolkit for genomic interval analysis providing specialized tools for overlap detection using IGD (Integrated Genome Database) indexing, coverage track generation (uniwig module for WIG/BigWig formats), genomic tokenization for machine learning applications (TreeTokenizer for deep learning models), reference sequence management (refget protocol compliance), fragment processing for single-cell genomics (barcode-based splitting and cluster analysis), and fragment scoring against reference datasets. Offers Python bindings with NumPy integration, command-line tools (gtars-cli), and Rust library. Key modules include: tokenizers (convert genomic regions to ML tokens), overlaprs (efficient overlap computation), uniwig (ATAC-seq/ChIP-seq/RNA-seq coverage profiles), refget (GA4GH-compliant sequence digests), bbcache (BEDbase.org integration), scoring (fragment enrichment metrics), and fragsplit (single-cell fragment manipulation). Supports parallel processing, memory-mapped files, streaming for large datasets, and serves as foundation for geniml genomic ML package. Ideal for genomic ML preprocessing, regulatory element analysis, variant annotation, chromatin accessibility profiling, and computational genomics workflows
- **Polars-Bio** - High-performance genomic interval operations and bioinformatics file I/O on Polars DataFrames. Provides overlap, nearest, merge, coverage, complement, and subtract operations for BED/VCF/BAM/GFF intervals. Streaming and cloud-native architecture for large-scale genomic analyses, serving as a faster alternative to bioframe
- **pysam** - Read, write, and manipulate genomic data files (SAM/BAM/CRAM alignments, VCF/BCF variants, FASTA/FASTQ sequences) with pileup analysis, coverage calculations, and bioinformatics workflows
- **PyDESeq2** - Python implementation of the DESeq2 differential gene expression analysis method for bulk RNA-seq data. Provides statistical methods for determining differential expression between experimental conditions using negative binomial generalized linear models. Key features include: size factor estimation for library size normalization, dispersion estimation and shrinkage, hypothesis testing with Wald test or likelihood ratio test, multiple testing correction (Benjamini-Hochberg FDR), results filtering and ranking, and integration with pandas DataFrames. Handles complex experimental designs, batch effects, and replicates. Produces fold-change estimates, p-values, and adjusted p-values for each gene. Use cases: identifying differentially expressed genes between conditions, RNA-seq experiment analysis, biomarker discovery, and gene expression studies requiring rigorous statistical analysis
- **Pathway Enrichment** - Pathway and gene-set enrichment analysis on gene lists or ranked gene data, with result interpretation. Supports over-representation analysis (ORA via Enrichr/Fisher's exact/hypergeometric), preranked and standard Gene Set Enrichment Analysis (GSEA), and single-sample scoring (ssGSEA/GSVA) using gseapy and the official g:Profiler client. Covers gene-set libraries (GO Biological Process/Molecular Function/Cellular Component, KEGG, Reactome, WikiPathways, and MSigDB collections including Hallmark, C2 canonical pathways, C5 ontology, and C7 immune signatures), gene-ID mapping (Ensembl/Entrez to symbols via Biomart, g:Convert, or mygene) and organism handling, choice of the statistical background universe, multiple-testing correction (Benjamini-Hochberg FDR vs g:Profiler g:SCS vs Bonferroni), redundancy reduction (enrichment maps, leading-edge genes, term clustering), and publication-ready tables plus dotplots/bar plots/GSEA running-score plots. Includes a CLI helper (run_enrichment.py) that runs ORA or preranked GSEA end-to-end — automatically building the ranking metric from a DESeq2 results table — and writes a results table and dotplot. Cross-references PyDESeq2 and Scanpy upstream (sources of differentially expressed genes and cluster markers) and database-lookup/gget for gene-ID mapping and Reactome/KEGG/STRING APIs. Use cases: functional interpretation of differentially expressed genes, CRISPR-screen hits, and single-cell cluster markers; GO/KEGG/Reactome/WikiPathways enrichment; preranked GSEA from DESeq2 statistics; pathway activity scoring per sample or cell; and building defensible, reproducible enrichment analyses that avoid common pitfalls (gene-ID/organism mismatch, wrong background, thresholding before GSEA)
- **Scanpy** - Comprehensive Python toolkit for single-cell RNA-seq data analysis built on AnnData (scanpy 1.12.x; Python 3.12+). Provides end-to-end workflows for preprocessing (quality control, scrublet doublet detection, normalization, log transformation), dimensionality reduction (PCA, UMAP, t-SNE), Leiden clustering, marker gene identification, pseudobulk aggregation via `get.aggregate()`, trajectory inference (PAGA, diffusion maps), and visualization. Key features include: efficient handling of large datasets using sparse matrices and experimental Dask out-of-core support, integration with scvi-tools for advanced analysis, batch correction methods (ComBat), and publication-quality plotting. Optional GPU acceleration via rapids-singlecell. Use cases: single-cell RNA-seq analysis, cell-type identification, exploratory cluster markers, pseudobulk DE workflows (with pydeseq2), trajectory analysis, and comprehensive single-cell genomics workflows
- **scVelo** - RNA velocity analysis for estimating cell state transitions from unspliced/spliced mRNA dynamics. Infers trajectory directions, computes latent time, and identifies driver genes in single-cell RNA-seq data. Complements Scanpy/scVI-tools for trajectory inference, enabling the study of cellular differentiation dynamics and lineage decisions at single-cell resolution
- **scvi-tools** - Probabilistic deep learning models for single-cell omics analysis. PyTorch-based framework providing variational autoencoders (VAEs) for dimensionality reduction, batch correction, differential expression, and data integration across modalities. Includes 30+ models: scVI/scANVI (RNA-seq integration and cell type annotation), totalVI/totalANVI (CITE-seq protein+RNA), MultiVI (multiome RNA+ATAC integration), PeakVI (ATAC-seq analysis), DestVI/Stereoscope/Tangram (spatial transcriptomics deconvolution), MethylVI (methylation), CytoVI (flow/mass cytometry), VeloVI (RNA velocity), contrastiveVI (perturbation studies), and Solo (doublet detection). Supports seamless integration with Scanpy/AnnData ecosystem, GPU acceleration, reference mapping (scArches), and probabilistic differential expression with uncertainty quantification
- **scikit-bio** - Python library for bioinformatics providing data structures, algorithms, and parsers for biological sequence analysis. Built on NumPy, SciPy, and pandas. Key features include: sequence objects (DNA, RNA, protein sequences) with biological alphabet validation, sequence alignment algorithms (local, global, semiglobal), phylogenetic tree manipulation, diversity metrics (alpha diversity, beta diversity, phylogenetic diversity), distance metrics for sequences and communities, file format parsers (FASTA, FASTQ, QIIME formats, Newick), and statistical analysis tools. Provides scikit-learn compatible transformers for machine learning workflows. Supports efficient processing of large sequence datasets. Use cases: sequence analysis, microbial ecology (16S rRNA analysis), metagenomics, phylogenetic analysis, and bioinformatics research requiring sequence manipulation and diversity calculations
- **TileDB-VCF** - High-performance C++ library with Python and CLI interfaces for efficient storage and retrieval of genomic variant-call data using TileDB multidimensional sparse array technology. Enables scalable VCF/BCF ingestion with incremental sample addition, compressed storage, parallel queries across genomic regions and samples, and export capabilities for population genomics workflows. Key features include: memory-efficient queries, cloud storage integration (S3, Azure, GCS), and CLI tools for dataset creation, sample ingestion, data export, and statistics. Supports building variant databases for large cohorts, population-scale genomics studies, and association analysis. Use cases: population genomics databases, cohort studies, variant discovery workflows, genomic data warehousing, and scaling to enterprise-level analysis with TileDB-Cloud platform
- **Zarr** - Python library (Zarr-Python 3.x) implementing chunked, compressed N-dimensional arrays for local disk and cloud object storage (S3, GCS via fsspec). Supports Zarr format 2 and 3, `zarr.codecs` compression (Blosc, gzip, zstd), partial chunk reads, consolidated metadata, sharding, and integration with NumPy, Dask, and Xarray. Use for out-of-core arrays, cloud-native pipelines, and large scientific datasets (genomics, imaging, climate). Skill: `zarr-python`
### Data Management & Infrastructure
- **LaminDB** - Open-source lineage-native lakehouse for biology that makes datasets and models queryable, traceable, validated, reproducible, and FAIR. Covers LaminDB 2.x setup, artifact and collection registration, feature-based query/search, lineage tracking with `ln.track()`/`ln.finish()` and `@ln.flow()`/`@ln.step()`, schema validation for DataFrame/AnnData/SpatialData/TileDB-SOMA, and ontology-backed annotation with Bionty 2.x. Includes current guidance for projects, branches, spaces, storage backends (local, S3, GCS, S3-compatible, Hugging Face), SQLite/Postgres deployment, safe credential handling, external-data validation, and integrations with Nextflow/nf-lamin, Snakemake, Redun, W&B, MLflow, Lightning, scVI-tools, DuckDB, and Vitessce.
- **Modal** - Serverless cloud platform for running Python code with minimal configuration, specialized for AI/ML workloads and scientific computing. Execute functions on powerful GPUs (T4, L4, A10, A100, L40S, H100, H200, B200, B200+), scale automatically from zero to thousands of containers, and pay only for compute used. Key features include: declarative container image building with uv (recommended)/pip/apt package management, automatic autoscaling with configurable limits and buffer containers, GPU acceleration with multi-GPU support (up to 8 GPUs per container, up to 1,536 GB VRAM), persistent storage via Volumes (v1 and v2) for model weights and datasets, secret management for API keys and credentials, scheduled jobs with cron expressions, web endpoints for deploying serverless APIs (FastAPI, ASGI, WSGI, WebSockets), parallel execution with `.map()` for batch processing, input concurrency and dynamic batching for I/O-bound workloads, Sandboxes for isolated execution of untrusted or agent-generated code with network (CIDR) restrictions, and resource configuration (CPU cores, memory, ephemeral disk up to 3 TiB). Supports custom Docker images, Micromamba/Conda environments, integration with Hugging Face/Weights & Biases, and distributed multi-GPU training. Free tier includes $30/month credits. Use cases: ML model deployment and inference (LLMs, image generation, speech, embeddings), GPU-accelerated training and fine-tuning, batch processing large datasets in parallel, scheduled compute-intensive jobs, serverless API deployment with autoscaling, protein folding and computational biology, scientific computing requiring distributed compute or specialized hardware, and data pipeline automation
- **Optimize for GPU** - GPU-accelerate Python code using the NVIDIA RAPIDS ecosystem and related libraries for 10x–1000x speedups on suitable workloads. Covers 12 GPU libraries with decision framework for choosing the right tool: CuPy (drop-in NumPy/SciPy replacement for array operations, FFT, linear algebra), Numba CUDA (custom GPU kernels with fine-grained thread/block/shared-memory control), Warp (JIT-compiled simulation kernels with built-in spatial types for physics, mesh ray casting, differentiable programming, and robotics), cuDF (drop-in pandas replacement for dataframe ETL, groupby, joins), cuML (drop-in scikit-learn replacement for classification, regression, clustering, dimensionality reduction, preprocessing), cuGraph (drop-in NetworkX replacement for PageRank, centrality, community detection, shortest paths), KvikIO (GPUDirect Storage for high-performance file IO bypassing CPU memory, S3/HTTP direct-to-GPU reads, Zarr GPU backend), cuxfilter (GPU-accelerated interactive cross-filtering dashboards with Bokeh, Datashader, and Deck.gl), cuCIM (drop-in scikit-image replacement for image filtering, morphology, segmentation, plus fast whole-slide image reading for digital pathology), cuVS (GPU-accelerated vector/similarity search with CAGRA, IVF-Flat, IVF-PQ for RAG and recommender systems), cuSpatial (GPU-accelerated GeoPandas replacement for spatial joins, point-in-polygon, trajectory analysis), and RAFT/pylibraft (sparse eigensolvers, device memory management, multi-GPU communication). All libraries interoperate via CUDA Array Interface for zero-copy data sharing. Includes optimization workflow (profile first, assess GPU suitability, start with drop-in replacements, minimize host-device transfers), code transformation patterns for each library, memory management principles, and common pitfalls. Use cases: accelerating NumPy/pandas/scikit-learn/NetworkX/scikit-image/GeoPandas/Faiss workloads, physics simulation, differentiable rendering, particle systems, vector search for RAG pipelines, GPUDirect Storage file IO, interactive data exploration dashboards, geospatial analysis, medical imaging, and sparse eigenvalue problems
### Cheminformatics & Drug Discovery
- **Datamol** - Python library for molecular manipulation and featurization built on RDKit with enhanced workflows and performance optimizations. Provides utilities for molecular I/O (reading/writing SMILES, SDF, MOL files), molecular standardization and sanitization, molecular transformations (tautomer enumeration, stereoisomer generation), molecular featurization (descriptors, fingerprints, graph representations), parallel processing for large datasets, and integration with machine learning pipelines. Features include: optimized RDKit operations, caching for repeated computations, molecular filtering and preprocessing, and seamless integration with pandas DataFrames. Designed for drug discovery and cheminformatics workflows requiring efficient processing of large compound libraries. Use cases: molecular preprocessing for ML models, compound library management, molecular similarity searches, and cheminformatics data pipelines
- **DeepChem** - Molecular machine learning library (PyPI 2.8.0; Python 3.7–3.11) with TensorFlow, PyTorch, and JAX backends. Graph neural networks (GCN, GAT, MPNN, AttentiveFP), 30+ MoleculeNet benchmarks, diverse featurizers (fingerprints, descriptors, graphs), and pretrained models (ChemBERTa, GROVER, MolFormer). Install backend-specific extras (`deepchem[torch]`, `[tensorflow]`, `[jax]`). Use for ADMET/toxicity prediction, materials properties, and transfer learning on small datasets
- **DiffDock** - Diffusion-based molecular docking method for predicting protein-ligand binding poses with DiffDock-L confidence scores. Uses diffusion models to generate diverse, high-quality poses without exhaustive search. Key features include: fast GPU inference compared to traditional docking methods, generation of multiple diverse poses, confidence scoring for pose quality, batch CSV input for screening campaigns, and support for protein PDB files or ESMFold sequence inputs with SMILES/SDF/MOL2 ligands. DiffDock does not directly predict binding affinity; combine generated poses with GNINA, MM/GBSA, or free-energy workflows when affinity ranking is required. Use cases: virtual screening, lead optimization, binding pose prediction, structure-based drug design, and initial pose generation for downstream refinement
- **MedChem** - Python library for molecular filtering and prioritization in drug discovery (datamol-io). Apply literature-derived drug-likeness rules (Lipinski, Veber, CNS, lead-like), named structural alert catalogs (PAINS, Brenk, NIBR), complexity thresholds against ZINC-derived percentiles, chemical group detection, and a custom query language for multi-criteria filtering. Returns pandas DataFrames from rule/structural filters and boolean masks from the functional API. Integrates with datamol/RDKit and supports parallel processing. Use cases: compound library triage, hit-to-lead filtering, PAINS deprioritization, screening-deck curation, and lead optimization workflows
- **Molfeat** - Comprehensive Python library providing 100+ molecular featurizers for converting molecules into numerical representations suitable for machine learning. Includes molecular fingerprints (ECFP, MACCS, RDKit, Pharmacophore), molecular descriptors (2D/3D descriptors, constitutional, topological, electronic), graph-based representations (molecular graphs, line graphs), and pre-trained models (MolBERT, ChemBERTa, Uni-Mol embeddings). Features unified API across different featurizer types, caching for performance, parallel processing, and integration with popular ML frameworks (scikit-learn, PyTorch, TensorFlow). Supports both traditional cheminformatics descriptors and modern learned representations. Use cases: molecular property prediction, virtual screening, molecular similarity searches, and preparing molecular data for machine learning models
- **PyTDC** - Python library providing access to Therapeutics Data Commons (TDC), a collection of curated datasets and benchmarks for drug discovery and development. Includes datasets for ADMET prediction (absorption, distribution, metabolism, excretion, toxicity), drug-target interactions, drug-drug interactions, drug response prediction, molecular generation, and retrosynthesis. Features standardized data formats, data loaders with automatic preprocessing, benchmark tasks with evaluation metrics, leaderboards for model comparison, and integration with popular ML frameworks. Provides both single-molecule and drug-pair datasets, covering various stages of drug discovery from target identification to clinical outcomes. Use cases: benchmarking ML models for drug discovery, ADMET prediction model development, drug-target interaction prediction, and drug discovery research
- **RDKit** - Open-source cheminformatics toolkit for molecular informatics and drug discovery. Provides comprehensive functionality for molecular I/O (reading/writing SMILES, SDF, MOL, PDB files), molecular descriptors (200+ 2D and 3D descriptors), molecular fingerprints (Morgan, RDKit, MACCS, topological torsions), SMARTS pattern matching for substructure searches, molecular alignment and 3D coordinate generation, pharmacophore perception, reaction handling, and molecular drawing. Features high-performance C++ core with Python bindings, support for large molecule sets, and extensive documentation. Widely used in pharmaceutical industry and academic research. Use cases: molecular property calculation, virtual screening, molecular similarity searches, substructure matching, molecular visualization, and general cheminformatics workflows
- **Rowan** - Cloud-based quantum chemistry platform with Python API for computational chemistry workflows. Provides access to 45+ chemistry calculations including pKa prediction, redox potentials, solubility, conformer searching, geometry optimization, protein-ligand docking (AutoDock Vina), and AI-powered protein cofolding (Chai-1, Boltz-1/2). Supports DFT, semiempirical (GFN-xTB), and neural network potential methods (AIMNet2, Egret). Key features include: automatic cloud resource allocation, unified API for diverse computational methods, RDKit-native interface for seamless cheminformatics integration, workflow organization with folders and projects, batch processing, and web interface for visualization. Requires API key from labs.rowansci.com. Use cases: molecular property prediction, structure-based drug design, virtual screening campaigns, protein-ligand binding prediction, conformational analysis, and automated computational chemistry pipelines
- **TorchDrug** - PyTorch-based machine learning platform for drug discovery with 40+ datasets, 20+ GNN models for molecular property prediction, protein modeling, knowledge graph reasoning, molecular generation, and retrosynthesis planning
### Proteomics & Mass Spectrometry
- **matchms** - Processing and similarity matching of mass spectrometry data with 40+ filters, spectral library matching (Cosine, Modified Cosine, Neutral Losses), metadata harmonization, molecular fingerprint comparison, and support for multiple file formats (MGF, MSP, mzML, JSON)
- **pyOpenMS** - Comprehensive mass spectrometry data analysis for proteomics and metabolomics (LC-MS/MS processing, peptide identification, feature detection, quantification, chemical calculations, and integration with search engines like Comet, Mascot, MSGF+)
### Medical Imaging & Digital Pathology
- **histolab** - Digital pathology toolkit for whole slide image (WSI) processing and analysis. Provides automated tissue detection, tile extraction for deep learning pipelines, and preprocessing for gigapixel histopathology images. Key features include: multi-format WSI support (SVS, TIFF, NDPI), three tile extraction strategies (RandomTiler for sampling, GridTiler for complete coverage, ScoreTiler for quality-driven selection), automated tissue masks with customizable filters, built-in scorers (NucleiScorer, CellularityScorer), Macenko and Reinhard stain normalization (0.6.0+), pyramidal image handling, visualization tools (thumbnails, mask overlays, tile previews), and H&E stain decomposition. Supports multiple tissue sections, artifact removal, pen annotation exclusion, and reproducible extraction with seeding. Requires Python 3.8–3.11, OpenSlide, and Linux or macOS. Use cases: creating training datasets for computational pathology, extracting informative tiles for tumor classification, whole-slide tissue characterization, quality assessment of histology samples, automated nuclei density analysis, and preprocessing for digital pathology deep learning workflows
- **PathML** - Comprehensive computational pathology toolkit for whole slide image analysis, tissue segmentation, and machine learning on pathology data. Provides end-to-end workflows for digital pathology research including data loading, preprocessing, feature extraction, and model deployment
- **pydicom** - Pure Python package for working with DICOM (Digital Imaging and Communications in Medicine) files. Provides comprehensive support for reading, writing, and manipulating medical imaging data from CT, MRI, X-ray, ultrasound, PET scans and other modalities. Key features include: pixel data extraction and manipulation with automatic decompression (JPEG/JPEG 2000/RLE), metadata access and modification with 1000+ standardized DICOM tags, image format conversion (PNG/JPEG/TIFF), anonymization tools for removing Protected Health Information (PHI), windowing and display transformations (VOI LUT application), multi-frame and 3D volume processing, DICOM sequence handling, and support for multiple transfer syntaxes. Use cases: medical image analysis, PACS system integration, radiology workflows, research data processing, DICOM anonymization, format conversion, image preprocessing for machine learning, multi-slice volume reconstruction, and clinical imaging pipelines
### Healthcare AI & Clinical Machine Learning
- **NeuroKit2** - Comprehensive biosignal processing toolkit for analyzing physiological data including ECG, EEG, EDA, RSP, PPG, EMG, and EOG signals. Use this skill when processing cardiovascular signals, brain activity, electrodermal responses, respiratory patterns, muscle activity, or eye movements. Key features include: automated signal processing pipelines (cleaning, peak detection, delineation, quality assessment), heart rate variability analysis across time/frequency/nonlinear domains (SDNN, RMSSD, LF/HF, DFA, entropy measures), EEG analysis (frequency band power, microstates, source localization), autonomic nervous system assessment (sympathetic indices, respiratory sinus arrhythmia), comprehensive complexity measures (25+ entropy types, 15+ fractal dimensions, Lyapunov exponents), event-related and interval-related analysis modes, epoch creation and averaging for stimulus-locked responses, multi-signal integration with unified workflows, and extensive signal processing utilities (filtering, decomposition, peak correction, spectral analysis). Includes modular reference documentation across 12 specialized domains. Use cases: heart rate variability for cardiovascular health assessment, EEG microstates for consciousness studies, electrodermal activity for emotion research, respiratory variability analysis, psychophysiology experiments, affective computing, stress monitoring, sleep staging, autonomic dysfunction assessment, biofeedback applications, and multi-modal physiological signal integration for comprehensive human state monitoring
- **PyHealth** - Comprehensive healthcare AI toolkit for developing, testing, and deploying machine learning models with clinical data. Provides specialized tools for electronic health records (EHR), physiological signals, medical imaging, and clinical text analysis. Key features include: 10+ healthcare datasets (MIMIC-III/IV, eICU, OMOP, sleep EEG, COVID-19 CXR), 20+ predefined clinical prediction tasks (mortality, hospital readmission, length of stay, drug recommendation, sleep staging, EEG analysis), 33+ models (Logistic Regression, MLP, CNN, RNN, Transformer, GNN, plus healthcare-specific models like RETAIN, SafeDrug, GAMENet, StageNet), comprehensive data processing (sequence processors, signal processors, medical code translation between ICD-9/10, NDC, RxNorm, ATC systems), training/evaluation utilities (Trainer class, fairness metrics, calibration, uncertainty quantification), and interpretability tools (attention visualization, SHAP, ChEFER). 3x faster than pandas for healthcare data processing. Use cases: ICU mortality prediction, hospital readmission risk assessment, safe medication recommendation with drug-drug interaction constraints, sleep disorder diagnosis from EEG signals, medical code standardization and translation, clinical text to ICD coding, length of stay estimation, and any clinical ML application requiring interpretability, fairness assessment, and calibrated predictions for healthcare deployment
### Clinical Documentation & Decision Support
- **Clinical Decision Support** - Generate professional clinical decision support (CDS) documents for pharmaceutical and clinical research settings. Includes patient cohort analyses (biomarker-stratified with outcomes) and treatment recommendation reports (evidence-based guidelines with decision algorithms). Features GRADE evidence grading, statistical analysis (hazard ratios, survival curves, waterfall plots), biomarker integration (genomic alterations, gene expression signatures, IHC markers), and regulatory compliance. Use cases: pharmaceutical cohort reporting, clinical guideline development, comparative effectiveness analyses, treatment algorithm creation, and evidence synthesis for drug development
- **Clinical Reports** - Write comprehensive clinical reports following established guidelines and standards. Covers case reports (CARE guidelines), diagnostic reports (radiology, pathology, laboratory), clinical trial reports (ICH-E3, SAE, CSR), and patient documentation (SOAP notes, H&P, discharge summaries). Includes templates, regulatory compliance (HIPAA, FDA, ICH-GCP), and validation tools. Use cases: journal case reports, diagnostic findings documentation, clinical trial reporting, patient progress notes, and regulatory submissions
- **Treatment Plans** - Generate concise (3-4 page), focused medical treatment plans in LaTeX/PDF format for all clinical specialties. Supports general medical treatment, rehabilitation therapy, mental health care, chronic disease management, perioperative care, and pain management. Features SMART goal frameworks, evidence-based interventions, HIPAA compliance, and professional formatting. Use cases: individualized patient care plans, rehabilitation programs, psychiatric treatment plans, surgical care pathways, and pain management protocols
### Neuroscience & Electrophysiology
- **BIDS** - Brain Imaging Data Structure (BIDS) standard for organizing and describing neuroscience and biomedical research datasets. While originating for MRI, BIDS now covers 11 modalities: imaging (MRI structural/functional/diffusion/perfusion, PET, microscopy), electrophysiology (EEG, MEG, iEEG, EMG), and other data (NIRS, motion capture, behavioral, MR spectroscopy), with active BEPs extending to microelectrode electrophysiology (Neuropixels), stimuli, and more. Covers the BIDS directory hierarchy, file naming conventions with entities (subject, session, task, acquisition, run, etc.), and JSON sidecar metadata. Key features include: dataset creation and validation workflows, querying BIDS datasets with PyBIDS (BIDSLayout), DICOM-to-BIDS conversion using HeuDiConv (ReproIn turnkey, map-into-reproin, and custom heuristic modes), dcm2bids (config-file-based), and BIDScoin (GUI-based), metadata inheritance and sidecar management, events files for task fMRI, participants and scans TSV files, BIDS derivatives conventions for preprocessed data and analysis outputs, BIDS-Apps interface (fMRIPrep, MRIQC, QSIPrep), machine-readable BIDS schema (bids_schema.json) and BEP listing (beps.yml) with update script, and .bidsignore configuration. Includes detailed reference documentation for the complete BIDS specification entity table (35 entities in schema ordering), required and recommended metadata fields for every modality, standard template spaces, and conversion tool workflows with examples. Use cases: organizing neuroscience data for sharing and analysis, validating BIDS compliance before repository submission (OpenNeuro, DANDI), converting DICOM scanner data to BIDS format, creating BIDS-compliant derivatives, querying datasets programmatically, and preparing data for BIDS-Apps processing pipelines
- **Neuropixels-Analysis** - Comprehensive toolkit for analyzing Neuropixels high-density neural recordings using SpikeInterface, Allen Institute, and International Brain Laboratory (IBL) best practices. Supports the full workflow from raw data to publication-ready curated units. Key features include: data loading from SpikeGLX, Open Ephys, and NWB formats, preprocessing pipelines (highpass filtering, phase shift correction for Neuropixels 1.0, bad channel detection, common average referencing), motion/drift estimation and correction (nonrigid_fast_and_accurate, nonrigid_accurate, and DREDge presets), spike sorting integration (Kilosort4 GPU, SpykingCircus2, Tridesclous2, Mountainsort5 CPU), comprehensive postprocessing (waveform extraction, template computation, spike amplitudes, correlograms, unit locations), quality metrics computation (SNR, ISI violations, presence ratio, amplitude cutoff, drift metrics), automated curation using Allen Institute and IBL criteria with configurable thresholds, model-based curation with pretrained UnitRefine classifiers (noise/neural and SUA/MUA) loaded from Hugging Face via spikeinterface.curation, AI-assisted visual curation for uncertain units using vision-language models, and export to Phy for manual review or NWB for sharing. Supports Neuropixels 1.0 (960 electrodes, 384 channels) and Neuropixels 2.0 (single and 4-shank configurations). Use cases: extracellular electrophysiology analysis, spike sorting from silicon probes, neural population recordings, systems neuroscience research, unit quality assessment, publication-ready neural data processing, and integration of AI-assisted curation for borderline units
### Protein Engineering & Design
- **Adaptyv** - Cloud laboratory platform for automated protein testing and validation. Submit protein sequences via API or web interface and receive experimental results in approximately 21 days. Supports multiple assay types including binding assays (biolayer interferometry for protein-target interactions, KD/kon/koff measurements), expression testing (quantify protein expression levels in E. coli, mammalian, yeast, or insect cells), thermostability measurements (DSF and CD for Tm determination and thermal stability profiling), and enzyme activity assays (kinetic parameters, substrate specificity, inhibitor testing). Includes computational optimization tools for pre-screening sequences: NetSolP/SoluProt for solubility prediction, SolubleMPNN for sequence redesign to improve expression, ESM for sequence likelihood scoring, ipTM (AlphaFold-Multimer) for interface stability assessment, and pSAE for aggregation risk quantification. Platform features automated workflows from expression through purification to assay execution with quality control, webhook notifications for experiment completion, batch submission support for high-throughput screening, and comprehensive results with kinetic parameters, confidence metrics, and raw data access. Use cases: antibody affinity maturation, therapeutic protein developability assessment, enzyme engineering and optimization, protein stability improvement, AI-driven protein design validation, library screening for expression and function, lead optimization with experimental feedback, and integration of computational design with wet-lab validation in iterative design-build-test-learn cycles
- **ESM (Evolutionary Scale Modeling)** - Protein language models from EvolutionaryScale/Biohub for protein design, structure prediction, and representation learning. Covers current `esm` SDK workflows for local ESM3/ESMC open models, Forge-hosted ESM3 and ESMC inference with `ESM_API_KEY`, Biohub-hosted ESMC embeddings, and ESMFold2 all-atom structure prediction. Use cases: novel protein design, sequence/structure co-design, protein embeddings, function annotation, variant generation, and directed evolution workflows
- **Glycoengineering** - Analyze and engineer protein glycosylation. Scan sequences for N-glycosylation sequons (N-X-S/T), predict O-glycosylation hotspots, and access curated glycoengineering tools (NetOGlyc, GlycoShield, GlycoWorkbench). Use for glycoprotein engineering, therapeutic antibody optimization, and vaccine design
- **Molecular Dynamics** - Run and analyze molecular dynamics simulations with OpenMM and MDAnalysis. Set up protein/small molecule systems, define force fields, run energy minimization and production MD, analyze trajectories (RMSD, RMSF, contact maps, free energy surfaces). Use for structural biology, drug binding studies, and biophysics research
- **Tamarind** - Run a large catalog of open-source molecular design and structural biology tools on the Tamarind Bio managed-GPU cloud via its REST API (`x-api-key` header) or MCP server — no local GPUs required. Covers structure prediction (AlphaFold, Boltz-2, Chai-1, ESMFold2), protein/binder/de novo design (RFdiffusion, ProteinMPNN/LigandMPNN, BoltzGen, BindCraft), antibody and nanobody design, humanization and developability, protein-ligand docking (DiffDock, AutoDock Vina) and binding-affinity prediction, MSA generation, and molecular dynamics — all through one uniform job API with single and batch submission and tool chaining (design → fold → score). Discovers tools and schemas live (`GET /tools`, MCP `getAvailableTools`/`getJobSchema`) and reads the key from `TAMARIND_API_KEY`. There is no official Python SDK — use plain `requests` or the MCP server. Use cases: cloud structure prediction and protein design without provisioning hardware, high-throughput batch characterization of sequences/designs, and chaining design-fold-score pipelines
### Machine Learning & Deep Learning
- **aeon** - Comprehensive scikit-learn compatible Python toolkit for time series machine learning providing state-of-the-art algorithms across 7 domains: classification (13 algorithm categories including ROCKET variants, deep learning with InceptionTime/ResNet/FCN, distance-based with DTW/ERP/LCSS, shapelet-based, dictionary methods like BOSS/WEASEL, and hybrid ensembles HIVECOTE), regression (9 categories mirroring classification approaches), clustering (k-means/k-medoids with temporal distances, deep learning autoencoders, spectral methods), forecasting (ARIMA, ETS, Theta, Threshold Autoregressive, TCN, DeepAR), anomaly detection (STOMP/MERLIN matrix profile, clustering-based CBLOF/KMeans, isolation methods, copula-based COPOD), segmentation (ClaSP, FLUSS, HMM, binary segmentation), and similarity search (MASS algorithm, STOMP motif discovery, approximate nearest neighbors). Includes 40+ distance metrics (elastic: DTW/DDTW/WDTW/Shape-DTW, edit-based: ERP/EDR/LCSS/TWE/MSM, lock-step: Euclidean/Manhattan), extensive transformations (ROCKET/MiniRocket/MultiRocket for features, Catch22/TSFresh for statistics, SAX/PAA for symbolic representation, shapelet transforms, wavelets, matrix profile), 20+ deep learning architectures (FCN, ResNet, InceptionTime, TCN, autoencoders with attention mechanisms), comprehensive benchmarking tools (UCR/UEA archives with 100+ datasets, published results repository, statistical testing), and performance-optimized implementations using numba. Features progressive model complexity from fast baselines (MiniRocket: <1 second training, 0.95+ accuracy on many benchmarks) to state-of-the-art ensembles (HIVECOTE V2), GPU acceleration support, and extensive visualization utilities. Use cases: physiological signal classification (ECG, EEG), industrial sensor monitoring, financial forecasting, change point detection, pattern discovery, activity recognition from wearables, predictive maintenance, climate time series analysis, and any sequential data requiring specialized temporal modeling beyond standard ML
- **Cirq** - Google quantum computing framework for designing, simulating, and running quantum circuits. Best suited for targeting Google Quantum AI hardware, designing noise-aware circuits, and running quantum characterization experiments. Provides low-level circuit design, noise modeling, and hardware-specific optimizations. For IBM hardware use Qiskit; for quantum ML with autodiff use PennyLane; for physics simulations use QuTiP
- **PufferLib** - High-performance reinforcement learning library achieving 1M-4M steps/second through optimized vectorization, native multi-agent support, and efficient PPO training (PuffeRL). Use this skill for RL training on any environment (Gymnasium, PettingZoo, Atari, Procgen), creating custom PufferEnv environments, developing policies (CNN, LSTM, multi-input architectures), optimizing parallel simulation performance, or scaling multi-agent systems. Includes Ocean suite (20+ environments), seamless framework integration with automatic space flattening, zero-copy vectorization with shared memory buffers, distributed training support, and comprehensive reference guides for training workflows, environment development, vectorization optimization, policy architectures, and third-party integrations
- **PyMC** - Comprehensive Python library for Bayesian statistical modeling and probabilistic programming. Provides intuitive syntax for building probabilistic models, advanced MCMC sampling algorithms (NUTS, nutpie, Metropolis-Hastings, Slice sampling), variational inference methods (ADVI, SVGD), Gaussian processes, time series distributions, and model comparison tools (WAIC, LOO). Features include automatic differentiation via PyTensor, compiled backends such as Numba/JAX, parallel sampling, model diagnostics and convergence checking, PyMC 6 data-container prediction workflows, and integration with ArviZ for visualization and analysis. Supports hierarchical models, mixture models, survival analysis, and custom distributions. Use cases: Bayesian data analysis, uncertainty quantification, A/B testing, time series forecasting, hierarchical modeling, and probabilistic machine learning
- **PyMOO** - Python framework for multi-objective optimization using evolutionary algorithms. Provides implementations of state-of-the-art algorithms including NSGA-II, NSGA-III, MOEA/D, SPEA2, and reference-point based methods. Features include: support for constrained and unconstrained optimization, multiple problem types (continuous, discrete, mixed-variable), performance indicators (hypervolume, IGD, GD), visualization tools (Pareto front plots, convergence plots), and parallel evaluation support. Supports custom problem definitions, algorithm configuration, and result analysis. Designed for engineering design, parameter optimization, and any problem requiring optimization of multiple conflicting objectives simultaneously. Use cases: multi-objective optimization problems, Pareto-optimal solution finding, engineering design optimization, and research in evolutionary computation
- **PyTorch Lightning** - Deep learning framework (lightning 2.6+) that organizes PyTorch code to eliminate boilerplate while maintaining full flexibility. Automates training workflows (epoch/batch iteration, optimizer steps, gradient management, checkpointing), supports multi-GPU/TPU training with DDP/FSDP/DeepSpeed strategies, includes LightningModule for model organization, Trainer for automation, LightningDataModule for data pipelines, callbacks for extensibility, and integrations with TensorBoard, Wandb, and MLflow for experiment tracking
- **PennyLane** - Cross-platform Python library for quantum computing, quantum machine learning, and quantum chemistry. Enables building and training quantum circuits with automatic differentiation, seamless integration with PyTorch/JAX/NumPy, and device-independent execution across simulators and quantum hardware (IBM, Amazon Braket, Google, Rigetti, IonQ). Key features include: quantum circuit construction with QNodes (quantum functions with automatic differentiation), 100+ quantum gates and operations (Pauli, Hadamard, rotation, controlled gates), circuit templates and layers for common ansatze (StronglyEntanglingLayers, BasicEntanglerLayers, UCCSD for chemistry), gradient computation methods (parameter-shift rule for hardware, backpropagation for simulators, adjoint differentiation), quantum chemistry module (molecular Hamiltonian construction, VQE for ground state energy, differentiable Hartree-Fock solver), ML framework integration (TorchLayer for PyTorch models, JAX transformations, TensorFlow deprecated), built-in optimizers (Adam, GradientDescent, QNG, Rotosolve), measurement types (expectation values, probabilities, samples, state vectors), device ecosystem (default.qubit simulator, lightning.qubit for performance, hardware plugins for IBM/Braket/Cirq/Rigetti/IonQ), and Catalyst for just-in-time compilation with adaptive circuits. Supports variational quantum algorithms (VQE, QAOA), quantum neural networks, hybrid quantum-classical models, data encoding strategies (angle, amplitude, IQP embeddings), and pulse-level programming. Use cases: variational quantum eigensolver for molecular simulations, quantum circuit machine learning with gradient-based optimization, hybrid quantum-classical neural networks, quantum chemistry calculations with differentiable workflows, quantum algorithm prototyping with hardware-agnostic code, quantum machine learning research with automatic differentiation, and deploying quantum circuits across multiple quantum computing platforms
- **Qiskit** - World's most popular open-source quantum computing framework for building, optimizing, and executing quantum circuits with 13M+ downloads and 74% developer preference. Provides comprehensive tools for quantum algorithm development including circuit construction with 100+ quantum gates (Pauli, Hadamard, CNOT, rotation gates, controlled gates), circuit transpilation with 83x faster optimization than competitors producing circuits with 29% fewer two-qubit gates, primitives for execution (Sampler for bitstring measurements and probability distributions, Estimator for expectation values and observables), visualization tools (circuit diagrams in matplotlib/LaTeX, result histograms, Bloch sphere, state visualizations), backend-agnostic execution (local simulators including StatevectorSampler and Aer, IBM Quantum hardware with 100+ qubit systems, IonQ trapped ion, Amazon Braket multi-provider), session and batch modes for iterative and parallel workloads, error mitigation with configurable resilience levels (readout error correction, ZNE, PEC reducing sampling overhead by 100x), four-step patterns workflow (Map classical problems to quantum circuits, Optimize through transpilation, Execute with primitives, Post-process results), algorithm libraries including Qiskit Nature for quantum chemistry (molecular Hamiltonians, VQE for ground states, UCCSD ansatz, multiple fermion-to-qubit mappings), Qiskit Optimization for combinatorial problems (QAOA, portfolio optimization, MaxCut), and Qiskit Machine Learning (quantum kernels, VQC, QNN), support for Python/C/Rust with modular architecture, parameterized circuits for variational algorithms, quantum Fourier transform, Grover search, Shor's algorithm, pulse-level control, IBM Quantum Runtime for cloud execution with job management and queuing, and comprehensive documentation with textbook and tutorials. Use cases: variational quantum eigensolver for molecular ground state energy, QAOA for combinatorial optimization problems, quantum chemistry simulations with multiple ansatze and mappings, quantum machine learning with kernel methods and neural networks, hybrid quantum-classical algorithms, quantum algorithm research and prototyping across multiple hardware platforms, quantum circuit optimization and benchmarking, quantum error mitigation and characterization, quantum information science experiments, and production quantum computing workflows on real quantum hardware
- **QuTiP** - Quantum Toolbox in Python for simulating and analyzing quantum mechanical systems. Provides comprehensive tools for both closed (unitary) and open (dissipative) quantum systems including quantum states (kets, bras, density matrices, Fock states, coherent states), quantum operators (creation/annihilation operators, Pauli matrices, angular momentum operators, quantum gates), time evolution solvers (Schrödinger equation with sesolve, Lindblad master equation with mesolve, quantum trajectories with Monte Carlo mcsolve, Bloch-Redfield brmesolve, Floquet methods for periodic Hamiltonians), analysis tools (expectation values, entropy measures, fidelity, concurrence, correlation functions, steady state calculations), visualization (Bloch sphere with animations, Wigner functions, Q-functions, Fock distributions, matrix histograms), and advanced methods (Hierarchical Equations of Motion for non-Markovian dynamics, permutational invariance for identical particles, stochastic solvers, superoperators). Supports tensor products for composite systems, partial traces, time-dependent Hamiltonians, multiple dissipation channels, and parallel processing. Includes extensive documentation, tutorials, and examples. Use cases: quantum optics simulations (cavity QED, photon statistics), quantum computing (gate operations, circuit dynamics), open quantum systems (decoherence, dissipation), quantum information theory (entanglement dynamics, quantum channels), condensed matter physics (spin chains, many-body systems), and general quantum mechanics research and education
- **scikit-learn** - Industry-standard Python library for classical machine learning (current stable **1.8.0**, Python 3.11–3.14, free-threaded CPython wheels). Provides comprehensive supervised learning (classification: Logistic Regression, SVM, Decision Trees, Random Forests with 17+ variants, Gradient Boosting with XGBoost-compatible HistGradientBoosting, Naive Bayes, KNN, Neural Networks/MLP; regression: Linear, Ridge, Lasso, ElasticNet, SVR, ensemble methods), unsupervised learning (clustering: K-Means, DBSCAN, native HDBSCAN since 1.3, OPTICS, Agglomerative/Hierarchical, Spectral, Gaussian Mixture Models, BIRCH, MeanShift; dimensionality reduction: PCA, Kernel PCA, t-SNE, Isomap, LLE, MDS, ClassicalMDS, NMF, TruncatedSVD, FastICA, LDA; outlier detection: IsolationForest, LocalOutlierFactor, OneClassSVM), data preprocessing (scaling: StandardScaler, MinMaxScaler, RobustScaler; encoding: OneHotEncoder, OrdinalEncoder, LabelEncoder; imputation: SimpleImputer, KNNImputer, IterativeImputer; feature engineering: PolynomialFeatures, KBinsDiscretizer, text vectorization with CountVectorizer/TfidfVectorizer), model evaluation (cross-validation: KFold, StratifiedKFold, TimeSeriesSplit, GroupKFold; hyperparameter tuning: GridSearchCV, RandomizedSearchCV, HalvingGridSearchCV; metrics: 30+ evaluation metrics for classification/regression/clustering including accuracy, precision, recall, F1, ROC-AUC, MSE, RMSE via root_mean_squared_error, R², silhouette score), and Pipeline/ColumnTransformer for production-ready workflows. Features consistent API (fit/predict/transform), extensive documentation, integration with NumPy/pandas/SciPy, joblib persistence, and scikit-learn-compatible ecosystem (XGBoost, LightGBM, CatBoost, imbalanced-learn, UMAP via umap-learn). Optimized implementations using Cython/OpenMP for performance. Use cases: predictive modeling, customer segmentation, anomaly detection, feature engineering, model selection/validation, text classification, image classification (with feature extraction), time series forecasting (with preprocessing), medical diagnosis, fraud detection, recommendation systems, and any tabular data ML task requiring interpretable models or established algorithms
- **scikit-survival** - Survival analysis and time-to-event modeling with censored data. Built on scikit-learn, provides Cox proportional hazards models (CoxPHSurvivalAnalysis, CoxnetSurvivalAnalysis with elastic net regularization), ensemble methods (Random Survival Forests, Gradient Boosting), Survival Support Vector Machines (linear and kernel), non-parametric estimators (Kaplan-Meier, Nelson-Aalen), competing risks analysis, and specialized evaluation metrics (concordance index, time-dependent AUC, Brier score). Handles right-censored data, integrates with scikit-learn pipelines, and supports feature selection and hyperparameter tuning via cross-validation
- **SHAP** - Model interpretability and explainability using Shapley values from game theory. Provides unified approach to explain any ML model with TreeExplainer (fast exact explanations for XGBoost/LightGBM/Random Forest), DeepExplainer (TensorFlow/PyTorch neural networks), KernelExplainer (model-agnostic), and LinearExplainer. Includes comprehensive visualizations (waterfall plots for individual predictions, beeswarm plots for global importance, scatter plots for feature relationships, bar/force/heatmap plots), supports model debugging, fairness analysis, feature engineering guidance, and production deployment
- **Stable Baselines3** - PyTorch-based reinforcement learning library (current stable **2.8.0**, Python 3.10+, PyTorch >= 2.3). Provides reliable implementations of RL algorithms (PPO, SAC, DQN, TD3, DDPG, A2C, HER, RecurrentPPO via SB3-Contrib). Use this skill for training RL agents on standard or custom Gymnasium environments, implementing callbacks for monitoring and control, using vectorized environments for parallel training, creating custom environments with proper Gymnasium API implementation, and integrating with deep RL workflows. Includes comprehensive training templates, evaluation utilities, algorithm selection guidance (on-policy vs off-policy, continuous vs discrete actions), support for multi-input policies (dict observations), goal-conditioned learning with HER, and integration with TensorBoard for experiment tracking
- **statsmodels** - Statistical modeling and econometrics (OLS, GLM, logit/probit, ARIMA, time series forecasting, hypothesis testing, diagnostics)
- **TimesFM Forecasting** - Zero-shot time series forecasting with Google's TimesFM foundation model. Use for any univariate time series (sales, sensors, energy, vitals, weather) without training a custom model. Supports CSV/DataFrame/array inputs with point forecasts and prediction intervals. Includes a preflight system checker script to verify RAM/GPU before first use
- **Torch Geometric** - PyTorch Geometric (PyG 2.7.x) for graph neural networks on PyTorch 2.6+. Covers `Data`/`HeteroData`, 60+ conv layers (GCN, GAT, GraphSAGE, GIN), node/link/graph classification, heterogeneous graphs, neighbor sampling (`NeighborLoader`, `LinkNeighborLoader`), OGB and built-in datasets, custom dataset loading, GNN explainability, and scaling via DDP, Lightning, and `torch.compile`
- **Transformers** - Hugging Face Transformers 5.x (PyTorch) for NLP, computer vision, audio, and multimodal tasks. Provides 1M+ pre-trained Hub models via pipelines (text-classification, NER, QA, summarization, translation, text-generation, image-classification, object-detection, ASR, VQA), Trainer API fine-tuning with mixed precision and distributed training, flexible text generation (greedy, beam search, sampling), and Auto classes for architecture selection (BERT, GPT, T5, ViT, BART, etc.)
- **UMAP-learn** - Python implementation of Uniform Manifold Approximation and Projection (UMAP) for dimensionality reduction and manifold learning. Provides fast, scalable nonlinear dimensionality reduction that preserves both local and global structure of high-dimensional data. Key features include: support for both supervised and unsupervised dimensionality reduction, ability to handle mixed data types, integration with scikit-learn API, and efficient implementation using numba for performance. Produces low-dimensional embeddings (typically 2D or 3D) suitable for visualization and downstream analysis. Often outperforms t-SNE in preserving global structure while maintaining local neighborhoods. Use cases: data visualization, feature extraction, preprocessing for machine learning, single-cell data analysis, and exploratory data analysis of high-dimensional datasets
### Materials Science & Chemistry
- **Astropy** - Comprehensive Python library for astronomy and astrophysics providing core functionality for astronomical research and data analysis. Includes coordinate system transformations (ICRS, Galactic, FK5, AltAz), physical units and quantities with automatic dimensional consistency, FITS file operations (reading, writing, manipulating headers and data), cosmological calculations (luminosity distance, lookback time, Hubble parameter, Planck/WMAP models), precise time handling across multiple time scales (UTC, TAI, TT, TDB) and formats (JD, MJD, ISO), table operations with unit support (FITS, CSV, HDF5, VOTable), WCS transformations between pixel and world coordinates, astronomical constants, modeling framework, visualization tools, and statistical functions. Use for celestial coordinate transformations, unit conversions, FITS image/table processing, cosmological distance calculations, barycentric time corrections, catalog cross-matching, and astronomical data analysis
- **COBRApy** - Python package for constraint-based reconstruction and analysis (COBRA) of metabolic networks. Provides tools for building, manipulating, and analyzing genome-scale metabolic models (GEMs). Key features include: flux balance analysis (FBA) for predicting optimal metabolic fluxes, flux variability analysis (FVA), gene knockout simulations, pathway analysis, model validation, and integration with other COBRA Toolbox formats (SBML, JSON). Supports various optimization objectives (biomass production, ATP production, metabolite production), constraint handling (reaction bounds, gene-protein-reaction associations), and model comparison. Includes utilities for model construction, gap filling, and model refinement. Use cases: metabolic engineering, systems biology, biotechnology applications, understanding cellular metabolism, and predicting metabolic phenotypes
- **Pymatgen** - Python Materials Genomics (pymatgen) library for materials science computation and analysis. Provides comprehensive tools for crystal structure manipulation, phase diagram construction, electronic structure analysis, and materials property calculations. Key features include: structure objects with symmetry analysis, space group determination, structure matching and comparison, phase diagram generation from formation energies, band structure and density of states analysis, defect calculations, surface and interface analysis, and integration with DFT codes (VASP, Quantum ESPRESSO, ABINIT). Supports Materials Project database integration, structure file I/O (CIF, POSCAR, VASP), and high-throughput materials screening workflows. Use cases: materials discovery, crystal structure analysis, phase stability prediction, electronic structure calculations, and computational materials science research
### Engineering & Simulation
- **MATLAB/Octave** - Numerical computing environment for matrix operations, data analysis, visualization, and scientific computing. MATLAB is commercial software optimized for matrix operations, while GNU Octave is a free open-source alternative with high compatibility. Key features include: matrix operations (creation, manipulation, linear algebra), comprehensive mathematics (eigenvalues, SVD, FFT, ODEs, optimization, statistics), 2D/3D visualization (plot, surf, contour, with extensive customization), data import/export (CSV, Excel, MAT files, images), programming constructs (functions, scripts, control flow, OOP), signal processing (FFT, filtering, convolution), and Python integration (calling Python from MATLAB and vice versa). Supports vectorized operations for performance, anonymous functions, tables for mixed data types, and cell arrays for heterogeneous data. GNU Octave provides compatibility with most MATLAB scripts with minor differences (comments with #, block terminators like endif, compound operators like +=). Scripts can be executed via `matlab -nodisplay -r "run('script.m'); exit;"` or `octave script.m`. Use cases: numerical simulations, signal processing, image processing, control systems, statistical analysis, algorithm prototyping, data visualization, and any scientific computing task requiring matrix operations or numerical methods
- **FluidSim** - Object-oriented Python framework for high-performance computational fluid dynamics (CFD) simulations using pseudospectral methods with FFT. Provides solvers for periodic-domain equations including 2D/3D incompressible Navier-Stokes equations (with/without stratification), shallow water equations, and Föppl-von Kármán elastic plate equations. Key features include: Pythran/Transonic compilation for performance comparable to Fortran/C++, MPI parallelization for large-scale simulations, hierarchical parameter configuration with type safety, comprehensive output management (physical fields in HDF5, spatial means, energy/enstrophy spectra, spectral energy budgets), custom forcing mechanisms (time-correlated random forcing, proportional forcing, script-defined forcing), flexible initial conditions (noise, vortex, dipole, Taylor-Green, from file, in-script), online and offline visualization, and integration with ParaView/VisIt for 3D visualization. Supports workflow features including simulation restart/continuation, parametric studies with batch execution, cluster submission integration, and adaptive CFL-based time stepping. Use cases: 2D/3D turbulence studies with energy cascade analysis, stratified oceanic and atmospheric flows with buoyancy effects, geophysical flows with rotation (Coriolis effects), vortex dynamics and fundamental fluid mechanics research, high-resolution direct numerical simulation (DNS), parametric studies exploring parameter spaces, validation studies (Taylor-Green vortex), and any periodic-domain fluid dynamics research requiring HPC-grade performance with Python flexibility
- **SimPy** - Process-based discrete-event simulation framework for modeling systems with processes, queues, and resource contention (manufacturing, service operations, network traffic, logistics). Supports generator-based process definition, multiple resource types (Resource, PriorityResource, PreemptiveResource, Container, Store), event-driven scheduling, process interaction mechanisms (signaling, interruption, parallel/sequential execution), real-time simulation synchronized with wall-clock time, and comprehensive monitoring capabilities for utilization, wait times, and queue statistics
- **SymPy** - Symbolic mathematics in Python for exact computation using mathematical symbols rather than numerical approximations. Provides comprehensive support for symbolic algebra (simplification, expansion, factorization), calculus (derivatives, integrals, limits, series), equation solving (algebraic, differential, systems of equations), matrices and linear algebra (eigenvalues, decompositions, solving linear systems), physics (classical mechanics with Lagrangian/Hamiltonian formulations, quantum mechanics, vector analysis, units), number theory (primes, factorization, modular arithmetic, Diophantine equations), geometry (2D/3D analytic geometry), combinatorics (permutations, combinations, partitions, group theory), logic and sets, statistics (probability distributions, random variables), special functions (gamma, Bessel, orthogonal polynomials), and code generation (lambdify to NumPy/SciPy functions, C/Fortran code generation, LaTeX output for documentation). Emphasizes exact arithmetic using rational numbers and symbolic representations, supports assumptions for improved simplification (positive, real, integer), integrates seamlessly with NumPy/SciPy through lambdify for fast numerical evaluation, and enables symbolic-to-numeric pipelines for scientific computing workflows
### Data Analysis & Visualization
- **Dask** - Parallel and distributed computing for larger-than-RAM pandas/NumPy workflows (dask 2026.x; Python 3.10+). Provides lazy DataFrames (expression-based query planning since 2025.1), Arrays, Bags, delayed tasks, and Futures via dask.distributed. Scales from multi-core laptops to clusters with a diagnostic dashboard. PyArrow 16+ required for DataFrame I/O; cloud paths need s3fs/gcsfs. Use for parallel file processing, out-of-core ETL, and scaling existing pandas/NumPy code. For single-machine out-of-core analytics without a cluster, consider vaex; for in-memory speed use polars
- **GeoMaster** - Comprehensive geospatial science skill covering remote sensing, GIS, spatial analysis, machine learning for earth observation, and 30+ scientific domains. Supports satellite imagery processing (Sentinel, Landsat, MODIS, SAR, hyperspectral), vector and raster data operations, spatial statistics, point cloud processing, network analysis, cloud-native workflows (STAC, COG, Planetary Computer), and 8 programming languages (Python, R, Julia, JavaScript, C++, Java, Go, Rust) with 500+ code examples. Use for remote sensing workflows, GIS analysis, spatial ML, Earth observation data processing, terrain analysis, hydrological modeling, marine spatial analysis, atmospheric science, and any geospatial computation task
- **GeoPandas** - Python library extending pandas for working with geospatial vector data including shapefiles, GeoJSON, and GeoPackage files. Provides GeoDataFrame and GeoSeries data structures combining geometric data with tabular attributes for spatial analysis. Key features include: reading/writing spatial file formats (Shapefile, GeoJSON, GeoPackage, PostGIS, Parquet) with Arrow acceleration for 2-4x faster I/O, geometric operations (buffer, simplify, centroid, convex hull, affine transformations) through Shapely integration, spatial analysis (spatial joins with predicates like intersects/contains/within, nearest neighbor joins, overlay operations for union/intersection/difference, dissolve for aggregation, clipping), coordinate reference system (CRS) management (setting CRS, reprojecting between coordinate systems, UTM estimation), and visualization (static choropleth maps with matplotlib, interactive maps with folium, multi-layer mapping, classification schemes with mapclassify). Supports spatial indexing for performance, filtering during read operations (bbox, mask, SQL WHERE), and integration with cartopy for cartographic projections. Use cases: spatial data manipulation, buffer analysis, spatial joins between datasets, dissolving boundaries, calculating areas/distances in projected CRS, reprojecting coordinate systems, creating choropleth maps, converting between spatial file formats, PostGIS database integration, and geospatial data analysis workflows
- **Matplotlib** - Comprehensive Python plotting library for creating publication-quality static, animated, and interactive visualizations. Provides extensive customization options for creating figures, subplots, axes, and annotations. Key features include: support for multiple plot types (line, scatter, bar, histogram, contour, 3D, and many more), extensive customization (colors, fonts, styles, layouts), multiple backends (PNG, PDF, SVG, interactive backends), LaTeX integration for mathematical notation, and integration with NumPy and pandas. Includes specialized modules (pyplot for MATLAB-like interface, artist layer for fine-grained control, backend layer for rendering). Supports complex multi-panel figures, color maps, legends, and annotations. Use cases: scientific figure creation, data visualization, exploratory data analysis, publication graphics, and any application requiring high-quality plots
- **NetworkX** - Comprehensive toolkit for creating, analyzing, and visualizing complex networks and graphs. Supports four graph types (Graph, DiGraph, MultiGraph, MultiDiGraph) with nodes as any hashable objects and rich edge attributes. Provides 100+ algorithms including shortest paths (Dijkstra, Bellman-Ford, A*), centrality measures (degree, betweenness, closeness, eigenvector, PageRank), clustering (coefficients, triangles, transitivity), community detection (modularity-based, label propagation, Girvan-Newman), connectivity analysis (components, cuts, flows), tree algorithms (MST, spanning trees), matching, graph coloring, isomorphism, and traversal (DFS, BFS). Includes 50+ graph generators for classic (complete, cycle, wheel), random (Erdős-Rényi, Barabási-Albert, Watts-Strogatz, stochastic block model), lattice (grid, hexagonal, hypercube), and specialized networks. Supports I/O across formats (edge lists, GraphML, GML, JSON, Pajek, GEXF, DOT) with Pandas/NumPy/SciPy integration. Visualization capabilities include 8+ layout algorithms (spring/force-directed, circular, spectral, Kamada-Kawai), customizable node/edge appearance, interactive visualizations with Plotly/PyVis, and publication-quality figure generation. Use cases: social network analysis, biological networks (protein-protein interactions, gene regulatory networks, metabolic pathways), transportation systems, citation networks, knowledge graphs, web structure analysis, infrastructure networks, and any domain involving pairwise relationships requiring structural analysis or graph-based modeling
- **Polars** - High-performance DataFrame library written in Rust with Python bindings for fast data manipulation, ETL, analytics, and pandas migration. Provides expression-based transformations, lazy query optimization, automatic parallel execution, streaming out-of-core processing, Arrow interoperability, and optional GPU execution. Supports common data sources and formats including CSV, Parquet, JSON/NDJSON, Excel, Arrow IPC, cloud object storage, databases, and BigQuery. Use cases: large-scale data processing, memory-conscious analytical pipelines, feature engineering, and high-performance DataFrame workflows
- **Seaborn** - Statistical data visualization with dataset-oriented interface, automatic confidence intervals, publication-quality themes, colorblind-safe palettes, and comprehensive support for exploratory analysis, distribution comparisons, correlation matrices, regression plots, and multi-panel figures
- **Vaex** - High-performance Python library for lazy, out-of-core DataFrames to process and visualize tabular datasets larger than available RAM. Processes over a billion rows per second through memory-mapped files (HDF5, Apache Arrow), lazy evaluation, and virtual columns (zero memory overhead). Provides instant file opening, efficient aggregations across billions of rows, interactive visualizations without sampling, machine learning pipelines with transformers (scalers, encoders, PCA), and seamless integration with pandas/NumPy/Arrow. Includes comprehensive ML framework (vaex.ml) with feature scaling, categorical encoding, dimensionality reduction, and integration with scikit-learn/XGBoost/LightGBM/CatBoost. Supports distributed computing via Dask, asynchronous operations, and state management for production deployment. Use cases: processing gigabyte to terabyte datasets, fast statistical aggregations on massive data, visualizing billion-row datasets, ML pipelines on big data, converting between data formats, and working with astronomical, financial, or scientific large-scale datasets
### Phylogenetics & Evolutionary Biology
- **ETE Toolkit** - Python library for phylogenetic tree manipulation, visualization, and analysis. Provides comprehensive tools for working with phylogenetic trees including tree construction, manipulation (pruning, collapsing, rooting), tree comparison (Robinson-Foulds distance, tree reconciliation), annotation (node colors, labels, branch styles), and publication-quality visualization. Key features include: support for multiple tree formats (Newick, Nexus, PhyloXML), integration with phylogenetic software (PhyML, RAxML, FastTree), tree annotation with metadata, interactive tree visualization, and export to various image formats (PNG, PDF, SVG). Supports species trees, gene trees, and reconciliation analysis. Use cases: phylogenetic analysis, tree visualization, evolutionary biology research, comparative genomics, and teaching phylogenetics
- **Phylogenetics** - Build and analyze phylogenetic trees using MAFFT (multiple alignment), IQ-TREE 2 (maximum likelihood), and FastTree (fast NJ/ML). Visualize with ETE3 or FigTree. Use for evolutionary analysis, microbial genomics, viral phylodynamics, protein family analysis, and molecular clock studies
### Multi-omics & AI Agent Frameworks
- **Pi Agent** - Build with and use Pi, the minimal terminal coding harness. Covers installing and configuring Pi, authenticating providers, adding custom models and providers, creating Pi skills/extensions/packages/themes/prompt templates, embedding Pi through the Node/TypeScript SDK, integrating over RPC or JSON event streams, parsing session JSONL files, and building custom TUI components. Use cases: building custom agent UIs, exposing Pi from another process or language, wiring private model gateways, packaging reusable Pi workflows, and helping agents operate Pi as a platform rather than only a CLI
- **HypoGeniC** - Automated hypothesis generation and testing using large language models to accelerate scientific discovery. Provides three frameworks: HypoGeniC (data-driven hypothesis generation from observational data), HypoRefine (synergistic approach combining literature insights with empirical patterns through an agentic system), and Union methods (mechanistic combination of literature and data-driven hypotheses). Features iterative refinement that improves hypotheses by learning from challenging examples, Redis caching for API cost reduction, and customizable YAML-based prompt templates. Includes command-line tools for generation (hypogenic_generation) and testing (hypogenic_inference). Research applications have demonstrated 14.19% accuracy improvement in AI-content detection and 7.44% in deception detection. Use cases: deception detection in reviews, AI-generated content identification, mental stress detection, exploratory research without existing literature, hypothesis-driven analysis in novel domains, and systematic exploration of competing explanations
### Autonomous Research & Optimization Frameworks
- **Arbor** - Autonomous Optimization (AO) via Hypothesis Tree Refinement (HTR), adapted from the Arbor framework (Jin et al., 2026; RUC-NLPIR/Arbor). Iteratively improves a real artifact — a codebase, model-training recipe, agent/search harness, data-synthesis pipeline, or prompt — against a measurable objective over many experiment cycles, without overfitting to the feedback signal. The agent acts as a long-lived **coordinator** that maintains a persistent **hypothesis tree** (each node binds a falsifiable hypothesis, a distilled reusable insight, and a pointer to the artifact version that realizes it) and runs the six-step loop Observe → Ideate → Select → Dispatch → Backpropagate → Decide; short-lived **executor** subagents each test one hypothesis in an isolated git worktree and return a dev score, factual result, distilled insight, and branch reference. A **held-out merge gate** promotes a candidate to the best artifact only when its improvement transfers to a test evaluator the search never optimized against, separating exploratory dev gains from verified progress. Insights are propagated up the tree so local findings become direction-level constraints and global priors, and falsified branches are pruned with their reasons recorded as negative constraints. Ships a `tree.py` state manager (init/observe/add-node/set-evidence/propagate/prune/merge/status/validate) that keeps the tree consistent and auditable, plus reference docs on the HTR methodology, the executor brief, the final-report template, and how to run the upstream `arbor` CLI instead. Use cases: "improve my model's eval score in fewer steps", optimizing an agent or search harness for higher pass rate/accuracy, tuning a data-generation pipeline judged by downstream model behavior, MLE-bench / Kaggle-style "improve the submission" tasks, and any long-horizon "make this artifact better and don't just memorize the dev set" workflow that needs structured, branching exploration with an audit trail
### Scientific Communication & Publishing
- **BGPT Paper Search** - Search scientific papers and retrieve structured experimental data extracted from full-text studies via the BGPT MCP server. Returns 25+ fields per paper including methods, results, sample sizes, quality scores, and conclusions. Use for literature reviews, evidence synthesis, and finding experimental details not available in abstracts alone
- **pyzotero** - Python client for the Zotero Web API v3. Programmatically manage Zotero reference libraries: retrieve, create, update, and delete items, collections, tags, and attachments. Export citations as BibTeX, CSL-JSON, and formatted bibliography HTML. Supports user and group libraries, local mode for offline access, paginated retrieval with `everything()`, full-text content indexing, saved search management, and file upload/download. Optional CLI and built-in MCP server (pyzotero 1.12+) for searching local Zotero 7 libraries including full-text PDF search and Semantic Scholar integration. Use cases: building research automation pipelines that integrate with Zotero, bulk importing references, exporting bibliographies programmatically, managing large reference collections, syncing library metadata, enriching bibliographic data, and connecting LLM agents to a local Zotero library.
- **Citation Management** - Comprehensive citation management for academic research. Search Google Scholar and PubMed for papers, extract accurate metadata from multiple sources (CrossRef, PubMed, arXiv), validate citations, and generate properly formatted BibTeX entries. Features include converting DOIs, PMIDs, or arXiv IDs to BibTeX, cleaning and formatting bibliography files, finding highly cited papers, checking for duplicates, and ensuring consistent citation formatting. Use cases: building bibliographies for manuscripts, verifying citation accuracy, citation deduplication, and maintaining reference databases
- **Generate Image** - AI-powered image generation and editing for scientific illustrations, schematics, and visualizations using OpenRouter's image generation models. Supports multiple models including google/gemini-3-pro-image-preview (high quality, recommended default) and black-forest-labs/flux.2-pro (fast, high quality). Key features include: text-to-image generation from detailed prompts, image editing capabilities (modify existing images with natural language instructions), automatic base64 encoding/decoding, PNG output with configurable paths, and comprehensive error handling. Requires OpenRouter API key (via .env file or environment variable). Use cases: generating scientific diagrams and illustrations, creating publication-quality figures, editing existing images (changing colors, adding elements, removing backgrounds), producing schematics for papers and presentations, visualizing experimental setups, creating graphical abstracts, and generating conceptual illustrations for scientific communication
- **Infographics** - Create professional infographics using Nano Banana Pro AI with smart iterative refinement. Uses Gemini 3 Pro for quality review. Integrates research-lookup and web search for accurate data. Supports 10 infographic types, 8 industry styles, and colorblind-safe palettes
- **LaTeX Posters** - Create professional research posters in LaTeX using beamerposter, tikzposter, or baposter. Support for conference presentations, academic posters, and scientific communication with layout design, color schemes, multi-column formats, figure integration, and poster-specific best practices. Features compliance with conference size requirements (A0, A1, 36×48"), complex multi-column layouts, and integration of figures, tables, equations, and citations. Use cases: conference poster sessions, thesis defenses, symposia presentations, and research group templates
- **Market Research Reports** - Generate comprehensive market research reports (50+ pages) in the style of top consulting firms (McKinsey, BCG, Gartner). Features professional LaTeX formatting, extensive visual generation, deep integration with research-lookup for data gathering, and multi-framework strategic analysis including Porter's Five Forces, PESTLE, SWOT, TAM/SAM/SOM, and BCG Matrix. Use cases: investment decisions, strategic planning, competitive landscape analysis, market sizing, and market entry evaluation
- **PPTX Posters** - Create professional research posters using PowerPoint/HTML formats for researchers who prefer WYSIWYG tools over LaTeX. Features design principles, layout templates, quality checklists, and export guidance for poster sessions. Use cases: conference posters when LaTeX is not preferred, quick poster creation, and collaborative poster design
- **Scientific Schematics** - Create publication-quality scientific diagrams using Nano Banana Pro AI with smart iterative refinement. Uses Gemini 3 Pro for quality review with document-type-specific thresholds (journal: 8.5/10, conference: 8.0/10, poster: 7.0/10). Specializes in neural network architectures, system diagrams, flowcharts, biological pathways, and complex scientific visualizations. Features natural language input, automatic quality assessment, and publication-ready output. Use cases: creating figures for papers, generating workflow diagrams, visualizing experimental designs, and producing graphical abstracts
- **Scientific Slides** - Build slide decks and presentations for research talks using PowerPoint and LaTeX Beamer. Features slide structure, design templates, timing guidance, and visual validation. Emphasizes visual engagement with minimal text, research-backed content with proper citations, and story-driven narrative. Use cases: conference presentations, academic seminars, thesis defenses, grant pitches, and professional talks
- **Venue Templates** - Access comprehensive LaTeX templates, formatting requirements, and submission guidelines for major scientific publication venues (Nature, Science, PLOS, IEEE, ACM), academic conferences (NeurIPS, ICML, CVPR, CHI), research posters, and grant proposals (NSF, NIH, DOE, DARPA). Provides ready-to-use templates and detailed specifications for successful academic submissions. Use cases: manuscript preparation, conference papers, research posters, and grant proposals with venue-specific formatting
### Document Processing & Conversion
- **DOCX** - Create, read, edit, and manipulate Word documents (.docx files). Supports professional documents with formatting like tables of contents, headings, page numbers, letterheads, find-and-replace, tracked changes, comments, and image insertion. Use for reports, memos, letters, templates, and any Word document deliverable
- **MarkItDown** - Python utility for converting 20+ file formats to Markdown optimized for LLM processing. Converts Office documents (PDF, DOCX, PPTX, XLSX), images with OCR, audio with transcription, web content (HTML, YouTube transcripts, EPUB), and structured data (CSV, JSON, XML) while preserving document structure (headings, lists, tables, hyperlinks). Key features include: Azure Document Intelligence integration for enhanced PDF table extraction, LLM-powered image descriptions using GPT-4o, batch processing with ZIP archive support, modular installation for specific formats, streaming approach without temporary files, and plugin system for custom converters. Supports Python 3.10+. Use cases: preparing documents for RAG systems, extracting text from PDFs and Office files, transcribing audio to text, performing OCR on images and scanned documents, converting YouTube videos to searchable text, processing HTML and EPUB books, converting structured data to readable format, document analysis pipelines, and LLM training data preparation
- **LiteParse** - Fast, local-first document parser (Rust core, Python/Node bindings) for layout-aware text extraction with bounding boxes—no cloud API required. Parses PDFs natively; converts Office and image formats via LibreOffice/ImageMagick when installed. Outputs layout-preserved plain text or structured JSON with per-page `text_items` (coordinates, font metadata, OCR confidence). Built-in Tesseract OCR with optional HTTP OCR servers (EasyOCR/PaddleOCR-compatible API), page subsets, encrypted PDFs, stdin/bytes parsing, and PNG page screenshots for multimodal agents. CLI (`lit parse`, `lit batch-parse`, `lit screenshot`) and Python API (`LiteParse`, `search_items`). Targets liteparse 2.0.0, Python 3.10+. Use when you need spatial grounding for RAG, batch literature ingestion, or agent vision—not for Markdown (MarkItDown) or PDF merge/split/forms (pdf skill)
- **Markdown & Mermaid Writing** - Comprehensive markdown and Mermaid diagram writing skill for creating scientific documents, reports, analyses, and visualizations. Establishes text-based diagrams as the default documentation standard with full style guides, 24 diagram type references, and 9 document templates
- **PDF** - Read, create, combine, split, rotate, watermark, encrypt/decrypt, and manipulate PDF files. Supports extracting text/tables, filling forms, OCR on scanned PDFs, image extraction, and format conversion
- **PPTX** - Create, read, edit, and manipulate PowerPoint presentations (.pptx files). Use for slide decks, pitch decks, extracting text from presentations, editing existing slides, combining or splitting presentations, and working with templates, layouts, speaker notes, and comments
- **XLSX** - Spreadsheet creation, editing, and analysis with support for formulas, formatting, data analysis, and visualization
### Laboratory Automation & Equipment Control
- **PyLabRobot** - Hardware-agnostic, pure Python SDK for automated and autonomous laboratories. Provides unified interface for controlling liquid handling robots (Hamilton STAR/STARlet, Opentrons OT-2, Tecan EVO), plate readers (BMG CLARIOstar), heater shakers, incubators, centrifuges, pumps, and scales. Key features include: modular resource management system for plates, tips, and containers with hierarchical deck layouts and JSON serialization; comprehensive liquid handling operations (aspirate, dispense, transfer, serial dilutions, plate replication) with automatic tip and volume tracking; backend abstraction enabling hardware-agnostic protocols that work across different robots; ChatterboxBackend for protocol simulation and testing without hardware; browser-based visualizer for real-time 3D deck state visualization; cross-platform support (Windows, macOS, Linux, Raspberry Pi); and integration capabilities for multi-device workflows combining liquid handlers, analytical equipment, and material handling devices. Use cases: automated sample preparation, high-throughput screening, serial dilution protocols, plate reading workflows, laboratory protocol development and validation, robotic liquid handling automation, and reproducible laboratory automation with state tracking and persistence
### Tool Discovery & Computational Resources
- **Autoskill** - Observe the user's screen via the local screenpipe daemon, detect repeated research workflows, match them against existing scientific-agent-skills, and draft new skills (or composition recipes that chain existing ones) for patterns not yet covered. Use when the user asks to analyze their recent work and propose skills based on what they actually do. Requires the screenpipe daemon running locally on port 3030; all detection runs locally and only redacted cluster summaries reach the LLM
- **Get Available Resources** - Detect available computational resources and generate strategic recommendations for scientific computing tasks at the start of any computationally intensive scientific task. Automatically identifies CPU capabilities, GPU availability (NVIDIA CUDA, AMD ROCm, Apple Silicon Metal), memory constraints, and disk space. Creates JSON file with resource information and recommendations for parallel processing (joblib, multiprocessing), out-of-core computing (Dask, Zarr), GPU acceleration (PyTorch, JAX), or memory-efficient strategies. Use cases: determining optimal computational approaches before data analysis, model training, or large file operations
### Research Methodology & Proposal Writing
- **Paperzilla** - Chat with your agent about Paperzilla projects, recommendations, and canonical papers. Use for recent project recommendations, recommendation triage, canonical paper details, markdown-based summaries, relevance-to-my-research discussions, recommendation feedback, JSON export, and Atom feed URLs
- **Paper Lookup** - Search 10 academic paper databases via their REST APIs to find research papers, preprints, and scholarly articles. Covers biomedical literature (PubMed, PMC full text), preprint servers (bioRxiv, medRxiv, arXiv), multidisciplinary indexes (OpenAlex, Crossref, Semantic Scholar), open access aggregators (CORE, Unpaywall). Use for searching research papers, finding citations, looking up articles by DOI or PMID, retrieving abstracts or full text, checking open access availability, exploring citation graphs, and systematic literature searches
- **Research Grants** - Write competitive research proposals for NSF, NIH, DOE, DARPA, and Taiwan NSTC. Features agency-specific formatting, review criteria understanding, budget preparation, broader impacts statements, significance narratives, innovation sections, and compliance with submission requirements (including PAPPG 24-1 and current NIH salary-cap guidance). Covers project descriptions, specific aims, technical narratives, milestone plans, budget justifications, and biosketches. Optional figures via the scientific-schematics skill. Use cases: federal grant applications, resubmissions with reviewer response, multi-institutional collaborations, and preliminary data sections
- **Research Lookup** - Look up current research information using Perplexity's Sonar Pro Search or Sonar Reasoning Pro models through OpenRouter. Intelligently selects models based on query complexity. Provides access to current academic literature, recent studies, technical documentation, and general research information with proper citations. Use cases: finding latest research, literature verification, gathering background research, finding citation sources, and staying current with emerging trends
- **Scholar Evaluation** - Apply the ScholarEval framework to systematically evaluate scholarly and research work. Provides structured evaluation methodology based on peer-reviewed research assessment criteria for analyzing academic papers, research proposals, literature reviews, and scholarly writing across multiple quality dimensions. Use cases: evaluating research papers for quality and rigor, assessing methodology design, scoring data analysis approaches, benchmarking research quality, and assessing publication readiness
### Regulatory & Standards Compliance
- **ISO 13485 Certification** - Comprehensive toolkit for preparing ISO 13485:2016 certification documentation for medical device Quality Management Systems. Provides gap analysis of existing documentation, templates for all mandatory documents, compliance checklists, and step-by-step documentation creation. Covers 31 required procedures including Quality Manuals, Medical Device Files, and work instructions. Use cases: starting ISO 13485 certification process, conducting gap analysis, creating or updating QMS documentation, preparing for certification audits, transitioning from FDA QSR to QMSR, and harmonizing with EU MDR requirements
## Scientific Thinking & Analysis
### Analysis & Methodology
- **Experimental Design** - Design studies before data collection: choosing a design, randomization (simple, block, stratified, cluster), blocking and stratification, factorial and fractional-factorial designs (DOE), screening (Plackett-Burman), response-surface designs (central composite, Box-Behnken), Latin hypercube, crossover/repeated-measures/split-plot/Latin-square, sequential and adaptive designs, and avoiding pseudoreplication. Includes seeded allocation-schedule and DOE-matrix generators (numpy, pandas, pyDOE3). For sample size see Statistical Power; for analysis see Statistical Analysis
- **Exploratory Data Analysis** - Comprehensive EDA toolkit with automated statistics, visualizations, and insights for any tabular dataset
- **Hypothesis Generation** - Structured frameworks for generating and evaluating scientific hypotheses
- **Literature Review** - Systematic literature search and review toolkit with support for multiple scientific databases (PubMed, bioRxiv, Google Scholar), citation management with multiple citation styles (APA, AMA, Vancouver, Chicago, IEEE, Nature, Science), citation verification and deduplication, search strategies (Boolean operators, MeSH terms, field tags), PDF report generation with formatted references, and comprehensive templates for conducting systematic reviews following PRISMA guidelines
- **Peer Review** - Comprehensive toolkit for conducting high-quality scientific peer review with structured evaluation of methodology, statistics, reproducibility, ethics, and presentation across all scientific disciplines
- **Scientific Brainstorming** - Conversational brainstorming partner for generating novel research ideas, exploring connections, challenging assumptions, and developing creative approaches through structured ideation workflows
- **Scientific Critical Thinking** - Tools and approaches for rigorous scientific reasoning and evaluation
- **Scientific Visualization** - Best practices and templates for creating publication-quality scientific figures with matplotlib and seaborn, including statistical plots with automatic confidence intervals, colorblind-safe palettes, multi-panel figures, heatmaps, and journal-specific formatting
- **Scientific Writing** - Comprehensive toolkit for writing, structuring, and formatting scientific research papers using IMRAD format, multiple citation styles (APA, AMA, Vancouver, Chicago, IEEE), reporting guidelines (CONSORT, STROBE, PRISMA), effective figures and tables, field-specific terminology, venue-specific structure expectations, and core writing principles for clarity, conciseness, and accuracy across all scientific disciplines
- **Statistical Analysis** - Guided hypothesis testing, assumption checks, effect sizes, power analysis, and APA reporting (Pingouin, SciPy, statsmodels, PyMC). For dedicated model APIs see statsmodels and pymc skills; for in-depth sample-size/power planning see Statistical Power; for designing the study see Experimental Design
- **Statistical Power** - Sample-size and statistical power analysis for planning studies: required n, minimum detectable effect (MDE), and power curves for t-tests, ANOVA, proportions, correlation, chi-square, and regression via a unified closed-form interface, plus a Monte Carlo harness for designs with no formula (logistic/Poisson regression, mixed models, cluster-randomized trials, survival). Covers effect-size choice (SESOI), and adjustments for multiplicity, dropout, and clustering (statsmodels, scipy, pingouin, numpy). For laying out the design see Experimental Design; for post-collection analysis see Statistical Analysis
### Decision & Scenario Analysis
- **Consciousness Council** - Run a multi-perspective Mind Council deliberation on any question, decision, or creative challenge. Use for diverse viewpoints, tough decisions, devil's advocate analysis, exploring problems from multiple angles, or when facing dilemmas, trade-offs, and complex choices with no obvious answer
- **DHDNA Profiler** - Extract cognitive patterns and thinking fingerprints from any text. Analyze how someone thinks, understand cognitive style, profile writing or speech patterns, compare thinking styles between people, and gain deeper insight into an author's reasoning patterns, decision-making style, or cognitive signature
- **What-If Oracle** - Run structured What-If scenario analysis with 4–6 branch possibility exploration (best, likely, worst, wild card). Use for speculative what-if questions, strategic forks, contingency planning, and stress-testing decisions before committing
### Web Search & Information Retrieval
- **Exa Search** - Web search and URL content extraction via the Exa API. Use for high-quality web search tuned for scientific and technical content, scholarly filtering via `category="research paper"` plus academic domain allowlists, and batch URL extraction
- **Parallel Web** - Search the web, extract URL content, and run deep research using the Parallel Chat API and Extract API. Use for web searches, research queries, and general information gathering with synthesized summaries and citations
pyproject.toml
+16
View File
@@ -0,0 +1,16 @@
[project]
name = "scientific-agent-skills"
version = "2.53.0"
description = "A set of ready to use Agent Skills for research, science, engineering, analysis, finance and writing."
readme = "README.md"
requires-python = ">=3.13"
dependencies = [
"cisco-ai-skill-scanner>=2.0.8",
"firecrawl-py>=4.9.0",
"python-dotenv>=1.0.0",
]
[dependency-groups]
dev = [
"skills-ref @ git+https://github.com/agentskills/agentskills.git#subdirectory=skills-ref",
]
scan_pr_skills.py
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Scan a subset of skills (those changed in a pull request) and emit a PR comment.
Usage:
python scan_pr_skills.py [--output FILE] [--fail-on SEVERITY] SKILL_DIR ...
Each ``SKILL_DIR`` should be a directory containing a ``SKILL.md``. Directories
without a ``SKILL.md`` (e.g. deleted skills) are skipped gracefully.
The script writes a markdown report intended to be posted as a sticky comment
on the pull request. It exits non-zero when any scanned skill has a finding at
``--fail-on`` severity or higher (default: ``CRITICAL``).
"""
from __future__ import annotations
import argparse
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from dotenv import load_dotenv
from skill_scanner.core.loader import SkillLoadError
from skill_scanner.core.models import Report
from scan_skills import build_scanner, severity_badge
load_dotenv()
SEVERITY_ORDER = ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO", "SAFE"]
COMMENT_MARKER = "<!-- skill-security-scan -->"
def _sev_str(obj) -> str:
sev = getattr(obj, "max_severity", None) or getattr(obj, "severity", None)
if sev is None:
return "SAFE"
return sev.value if hasattr(sev, "value") else str(sev)
def scan_skill_dirs(scanner, skill_dirs: list[Path]) -> Report:
report = Report()
loaded_skills = []
for skill_dir in skill_dirs:
name = skill_dir.name
print(f" Scanning {name} ...", end="", flush=True)
t0 = time.time()
try:
skill = scanner.loader.load_skill(skill_dir)
result = scanner._scan_single_skill(skill, skill_dir)
report.add_scan_result(result)
loaded_skills.append(skill)
elapsed = time.time() - t0
n = len(result.findings)
tag = severity_badge(_sev_str(result))
print(f" {tag} — {n} finding{'s' if n != 1 else ''} ({elapsed:.1f}s)")
except SkillLoadError as exc:
elapsed = time.time() - t0
print(f" ⚠️ SKIP ({exc}) ({elapsed:.1f}s)")
report.skills_skipped.append({"skill": str(skill_dir), "reason": str(exc)})
except Exception as exc: # pragma: no cover - defensive
elapsed = time.time() - t0
print(f" ❌ ERROR ({exc}) ({elapsed:.1f}s)")
report.skills_skipped.append({"skill": str(skill_dir), "reason": str(exc)})
if len(loaded_skills) > 1:
print("\n Running cross-skill overlap analysis ...", end="", flush=True)
t0 = time.time()
try:
from skill_scanner.core.analyzers.cross_skill_scanner import CrossSkillScanner
overlap = scanner._check_description_overlap(loaded_skills) or []
cross = CrossSkillScanner().analyze_skill_set(loaded_skills) or []
all_cross = [*overlap, *cross]
if scanner.policy.disabled_rules:
all_cross = [f for f in all_cross if f.rule_id not in scanner.policy.disabled_rules]
if all_cross:
scanner._apply_severity_overrides(all_cross)
report.add_cross_skill_findings(all_cross)
elapsed = time.time() - t0
print(f" {len(all_cross)} finding{'s' if len(all_cross) != 1 else ''} ({elapsed:.1f}s)")
except Exception as exc: # pragma: no cover - defensive
print(f" error: {exc}")
return report
def _loc(finding) -> str | None:
if not finding.file_path:
return None
loc = finding.file_path
if finding.line_number:
loc += f":{finding.line_number}"
return loc
def format_comment(report: Report, scanned_dirs: list[Path]) -> str:
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
lines: list[str] = [COMMENT_MARKER, "## 🛡️ Skill Security Scan", ""]
lines.append(f"_Generated at {now}_")
lines.append("")
if not scanned_dirs:
lines.append("No skill directories with a `SKILL.md` were changed in this PR — nothing to scan.")
return "\n".join(lines)
lines.append(f"**Skills scanned:** {report.total_skills_scanned} ")
lines.append(f"**Total findings:** {report.total_findings} ")
lines.append(
f"**Critical:** {report.critical_count} | "
f"**High:** {report.high_count} | "
f"**Safe:** {report.safe_count}/{report.total_skills_scanned}"
)
lines.append("")
if report.scan_results:
lines.append("| Skill | Severity | Findings | Safe |")
lines.append("|-------|----------|----------|------|")
sorted_results = sorted(
report.scan_results,
key=lambda r: SEVERITY_ORDER.index(_sev_str(r)),
)
for result in sorted_results:
sev = _sev_str(result)
safe = "✅" if result.is_safe else "❌"
lines.append(
f"| `{result.skill_name}` | {severity_badge(sev)} | {len(result.findings)} | {safe} |"
)
lines.append("")
flagged = [r for r in sorted_results if r.findings]
if flagged:
lines.append("### Findings")
lines.append("")
for result in flagged:
sev = _sev_str(result)
n = len(result.findings)
lines.append(
f"<details><summary><code>{result.skill_name}</code> — "
f"{severity_badge(sev)} ({n} finding{'s' if n != 1 else ''})</summary>"
)
lines.append("")
for finding in result.findings:
fsev = _sev_str(finding)
lines.append(
f"- **{severity_badge(fsev)}** `{finding.rule_id}` — {finding.title}"
)
if finding.description:
lines.append(f" > {finding.description}")
loc = _loc(finding)
if loc:
lines.append(f" > File: `{loc}`")
if finding.remediation:
lines.append(f" > **Remediation:** {finding.remediation}")
lines.append("")
lines.append("</details>")
lines.append("")
if report.skills_skipped:
lines.append("### Skipped")
lines.append("")
for entry in report.skills_skipped:
lines.append(f"- `{entry['skill']}`: {entry['reason']}")
lines.append("")
return "\n".join(lines)
def _should_block(report: Report, fail_on: str) -> tuple[bool, str | None]:
if fail_on == "NEVER":
return False, None
blocking_ranks = SEVERITY_ORDER[: SEVERITY_ORDER.index(fail_on) + 1]
for result in report.scan_results:
sev = _sev_str(result)
if sev in blocking_ranks:
return True, f"{sev} finding(s) in {result.skill_name}"
return False, None
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("skill_dirs", nargs="*", help="Skill directories to scan")
parser.add_argument(
"--output",
default="pr_scan_comment.md",
help="Path to write the PR comment markdown (default: pr_scan_comment.md)",
)
parser.add_argument(
"--fail-on",
default="CRITICAL",
choices=["CRITICAL", "HIGH", "MEDIUM", "LOW", "NEVER"],
help="Exit non-zero if any scanned skill has a finding at this severity or higher",
)
args = parser.parse_args()
scan_targets: list[Path] = []
for raw in args.skill_dirs:
p = Path(raw)
if not p.is_dir():
print(f" SKIP {raw} (not a directory)")
continue
if not (p / "SKILL.md").exists():
print(f" SKIP {raw} (no SKILL.md)")
continue
scan_targets.append(p)
if not scan_targets:
print("No skill directories to scan — writing no-op comment.")
md = format_comment(Report(), [])
Path(args.output).write_text(md)
return 0
print("Building scanner (LLM + behavioral + trigger + balanced policy)...")
scanner = build_scanner()
print(f"Analyzers: {scanner.list_analyzers()}\n")
print(f"Scanning {len(scan_targets)} skill(s):")
for d in scan_targets:
print(f" - {d}")
print()
report = scan_skill_dirs(scanner, scan_targets)
print(
f"\nResults: {report.total_skills_scanned} skills, {report.total_findings} findings "
f"(Critical: {report.critical_count} High: {report.high_count} Safe: {report.safe_count})"
)
md = format_comment(report, scan_targets)
Path(args.output).write_text(md)
print(f"Comment written to {args.output}")
blocked, reason = _should_block(report, args.fail_on)
if blocked:
print(f"\n❌ Blocking: {reason} (--fail-on {args.fail_on})")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
scan_skills.py
+224
View File
@@ -0,0 +1,224 @@
#!/usr/bin/env python3
"""Scan all skills for security issues and produce SECURITY.md."""
import os
import time
from datetime import datetime, timezone
from pathlib import Path
from dotenv import load_dotenv
from skill_scanner import SkillScanner
from skill_scanner.core.analyzers import (
BehavioralAnalyzer,
LLMAnalyzer,
TriggerAnalyzer,
)
from skill_scanner.core.loader import SkillLoadError
from skill_scanner.core.models import Report
from skill_scanner.core.scan_policy import ScanPolicy
load_dotenv()
SKILLS_DIR = "skills"
OUTPUT_FILE = "SECURITY.md"
def build_scanner() -> SkillScanner:
policy = ScanPolicy.from_preset("balanced")
policy.llm_analysis.max_instruction_body_chars = 75_000
policy.llm_analysis.max_referenced_file_chars = 75_000
policy.llm_analysis.max_code_file_chars = 75_000
policy.llm_analysis.max_total_prompt_chars = 500_000
llm_model = os.getenv("SKILL_SCANNER_LLM_MODEL", "anthropic/claude-sonnet-4-6")
llm_key = os.getenv("SKILL_SCANNER_LLM_API_KEY")
scanner = SkillScanner(
analyzers=[
BehavioralAnalyzer(),
TriggerAnalyzer(),
LLMAnalyzer(model=llm_model, api_key=llm_key, policy=policy),
],
policy=policy,
)
return scanner
def severity_badge(sev: str) -> str:
icons = {
"CRITICAL": "🔴",
"HIGH": "🟠",
"MEDIUM": "🟡",
"LOW": "🔵",
"INFO": "⚪",
"SAFE": "🟢",
}
return f"{icons.get(sev, '⚫')} {sev}"
def generate_report(report) -> str:
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
lines: list[str] = []
lines.append("# Security Scan Report")
lines.append("")
lines.append(f"**Generated:** {now} ")
lines.append(f"**Skills scanned:** {report.total_skills_scanned} ")
lines.append(f"**Total findings:** {report.total_findings} ")
lines.append(f"**Critical:** {report.critical_count} | **High:** {report.high_count} | **Safe skills:** {report.safe_count}/{report.total_skills_scanned}")
lines.append("")
# Summary table
lines.append("## Summary")
lines.append("")
lines.append("| Skill | Severity | Findings | Safe | Duration |")
lines.append("|-------|----------|----------|------|----------|")
sorted_results = sorted(
report.scan_results,
key=lambda r: ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO", "SAFE"].index(
r.max_severity.value if hasattr(r.max_severity, "value") else str(r.max_severity)
),
)
for result in sorted_results:
sev = result.max_severity.value if hasattr(result.max_severity, "value") else str(result.max_severity)
safe = "✅" if result.is_safe else "❌"
duration = f"{result.scan_duration_seconds:.1f}s"
lines.append(f"| {result.skill_name} | {severity_badge(sev)} | {len(result.findings)} | {safe} | {duration} |")
lines.append("")
# Per-skill details (only for skills with findings)
flagged = [r for r in sorted_results if r.findings]
if flagged:
lines.append("## Detailed Findings")
lines.append("")
for result in flagged:
sev = result.max_severity.value if hasattr(result.max_severity, "value") else str(result.max_severity)
lines.append(f"### {result.skill_name} — {severity_badge(sev)}")
lines.append("")
for finding in result.findings:
fsev = finding.severity.value if hasattr(finding.severity, "value") else str(finding.severity)
lines.append(f"- **{severity_badge(fsev)}** `{finding.rule_id}` — {finding.title}")
if finding.description:
lines.append(f" > {finding.description}")
if finding.file_path:
loc = finding.file_path
if finding.line_number:
loc += f":{finding.line_number}"
lines.append(f" > File: `{loc}`")
if finding.remediation:
lines.append(f" > **Remediation:** {finding.remediation}")
lines.append("")
else:
lines.append("## Detailed Findings")
lines.append("")
lines.append("No findings to report — all skills passed.")
lines.append("")
return "\n".join(lines)
def scan_with_progress(scanner: SkillScanner, skills_dir: str) -> Report:
"""Run scan_directory logic with per-skill progress output."""
base = Path(skills_dir)
if not base.exists():
raise FileNotFoundError(f"Directory does not exist: {base}")
skill_dirs = sorted(
{p.parent for p in base.rglob("SKILL.md")},
key=lambda p: p.name,
)
total = len(skill_dirs)
if total == 0:
print(" No skills found.")
return Report()
print(f" Found {total} skills to scan\n")
report = Report()
loaded_skills = []
scan_start = time.time()
width = len(str(total))
longest_name = 0
for i, skill_dir in enumerate(skill_dirs, 1):
name = skill_dir.name
longest_name = max(longest_name, len(name))
counter = f"[{i:>{width}}/{total}]"
print(f" {counter} {name} ...", end="", flush=True)
t0 = time.time()
try:
skill = scanner.loader.load_skill(skill_dir)
result = scanner._scan_single_skill(skill, skill_dir)
report.add_scan_result(result)
loaded_skills.append(skill)
elapsed = time.time() - t0
sev = result.max_severity.value if hasattr(result.max_severity, "value") else str(result.max_severity)
tag = severity_badge(sev)
n_findings = len(result.findings)
detail = f"{n_findings} finding{'s' if n_findings != 1 else ''}" if n_findings else ""
print(f"\r {counter} {name:{longest_name}} {tag:18} {detail:20} ({elapsed:.1f}s)")
except SkillLoadError as e:
elapsed = time.time() - t0
print(f"\r {counter} {name:{longest_name}} ⚠️ SKIP ({e}) ({elapsed:.1f}s)")
report.skills_skipped.append({"skill": str(skill_dir), "reason": str(e)})
except Exception as e:
elapsed = time.time() - t0
print(f"\r {counter} {name:{longest_name}} ❌ ERROR ({e}) ({elapsed:.1f}s)")
report.skills_skipped.append({"skill": str(skill_dir), "reason": str(e)})
wall = time.time() - scan_start
if len(loaded_skills) > 1:
print("\n Running cross-skill overlap analysis ...", end="", flush=True)
t0 = time.time()
try:
overlap = scanner._check_description_overlap(loaded_skills)
from skill_scanner.core.analyzers.cross_skill_scanner import CrossSkillScanner
cross = CrossSkillScanner().analyze_skill_set(loaded_skills)
all_cross = [*list(overlap or []), *list(cross or [])]
if scanner.policy.disabled_rules:
all_cross = [f for f in all_cross if f.rule_id not in scanner.policy.disabled_rules]
if all_cross:
scanner._apply_severity_overrides(all_cross)
report.add_cross_skill_findings(all_cross)
elapsed = time.time() - t0
print(f" {len(all_cross)} finding{'s' if len(all_cross) != 1 else ''} ({elapsed:.1f}s)")
except Exception as e:
print(f" error: {e}")
print(f"\n Done in {wall:.1f}s")
return report
def main():
print("Building scanner (LLM + behavioral + trigger + balanced policy)...")
scanner = build_scanner()
print(f"Analyzers: {scanner.list_analyzers()}\n")
print(f"Scanning {SKILLS_DIR}/...")
report = scan_with_progress(scanner, SKILLS_DIR)
print(f"\nResults: {report.total_skills_scanned} skills, {report.total_findings} findings")
print(f" Critical: {report.critical_count} High: {report.high_count} Safe: {report.safe_count}")
md = generate_report(report)
with open(OUTPUT_FILE, "w") as f:
f.write(md)
print(f"\nReport written to {OUTPUT_FILE}")
if __name__ == "__main__":
main()
skills/adaptyv/SKILL.md
+239
View File
@@ -0,0 +1,239 @@
---
name: adaptyv
author: "K-Dense, Inc."
description: "How to use the Adaptyv Bio Foundry API and Python SDK for protein experiment design, submission, and results retrieval. Use this skill whenever the user mentions Adaptyv, Foundry API, protein binding assays, protein screening experiments, BLI/SPR assays, thermostability assays, or wants to submit protein sequences for experimental characterization. Also trigger when code imports `adaptyv`, `adaptyv_sdk`, or `FoundryClient`, or references `foundry-api-public.adaptyvbio.com`."
license: MIT
compatibility: Requires Python 3.10+, an Adaptyv Foundry account, and an API key from foundry.adaptyvbio.com. Install adaptyv-sdk from GitHub with uv pip install.
metadata: {"version": "1.2", "skill-author": "K-Dense Inc."}
---
# Adaptyv Bio Foundry API
Adaptyv Bio is a cloud lab that turns protein sequences into experimental data. Users submit amino acid sequences via API or UI; Adaptyv's automated lab runs assays (binding, thermostability, expression, fluorescence) and delivers results in ~21 days.
**Official docs:** [docs.adaptyvbio.com/api-reference](https://docs.adaptyvbio.com/api-reference) · [llms.txt index](https://docs.adaptyvbio.com/llms.txt) · [OpenAPI spec](https://foundry-api-public.adaptyvbio.com/api/v1/openapi.json)
## Quick Start
**Base URL:** `https://foundry-api-public.adaptyvbio.com/api/v1`
**Authentication:** Bearer token in the `Authorization` header. Tokens are obtained from [foundry.adaptyvbio.com](https://foundry.adaptyvbio.com/) sidebar.
When writing code, always read the API key from the environment variable `ADAPTYV_API_KEY` or from a `.env` file — never hardcode tokens. Check for a `.env` file in the project root first; if one exists, use a library like `python-dotenv` to load it.
The [official API docs](https://docs.adaptyvbio.com/api-reference/api-introduction) use `FOUNDRY_API_TOKEN` in curl examples; that is the same bearer token — prefer `ADAPTYV_API_KEY` in Python and new shell scripts for consistency with the SDK.
```bash
export ADAPTYV_API_KEY="abs0_..."
curl https://foundry-api-public.adaptyvbio.com/api/v1/targets?limit=3 \
-H "Authorization: Bearer $ADAPTYV_API_KEY"
```
Every request except `GET /openapi.json` requires authentication. Store tokens in environment variables or `.env` files — never commit them to source control.
## Python SDK
**Version note:** `adaptyv-sdk` **0.1.0** (beta) is not yet on PyPI — install from GitHub:
```bash
uv pip install "git+https://github.com/adaptyvbio/adaptyv-sdk.git"
```
In a project with `pyproject.toml`:
```bash
uv add "adaptyv-sdk @ git+https://github.com/adaptyvbio/adaptyv-sdk.git"
```
**Environment variables** (set in shell or `.env` file):
```bash
ADAPTYV_API_KEY=your_api_key
ADAPTYV_API_URL=https://foundry-api-public.adaptyvbio.com/api/v1
ADAPTYV_ORGANIZATION_ID=your_org_id # optional
```
The `@lab.experiment` decorator and `FoundryClient` both read `ADAPTYV_API_KEY` and `ADAPTYV_API_URL` from the environment when not passed explicitly.
### Decorator Pattern
```python
from adaptyv import lab
@lab.experiment(target="PD-L1", experiment_type="screening", method="bli")
def design_binders():
return {"design_a": "MVKVGVNG...", "design_b": "MKVLVAG..."}
result = design_binders()
print(f"Experiment: {result.experiment_url}")
```
### Client Pattern
```python
import os
from adaptyv import FoundryClient
client = FoundryClient(
api_key=os.environ["ADAPTYV_API_KEY"],
base_url=os.environ.get(
"ADAPTYV_API_URL",
"https://foundry-api-public.adaptyvbio.com/api/v1",
),
)
# Browse targets
targets = client.targets.list(search="EGFR", selfservice_only=True)
# Estimate cost
estimate = client.experiments.cost_estimate({
"experiment_spec": {
"experiment_type": "screening",
"method": "bli",
"target_id": "target-uuid",
"sequences": {"seq1": "EVQLVESGGGLVQ..."},
"n_replicates": 3
}
})
# Create and submit
exp = client.experiments.create({...})
client.experiments.submit(exp.experiment_id)
# Later: retrieve results
results = client.experiments.get_results(exp.experiment_id)
```
## Experiment Types
| Type | Method | Measures | Requires Target |
|---|---|---|---|
| `affinity` | `bli` or `spr` | KD, kon, koff kinetics | Yes |
| `screening` | `bli` or `spr` | Yes/no binding | Yes |
| `thermostability` | — | Melting temperature (Tm) | No |
| `expression` | — | Expression yield | No |
| `fluorescence` | — | Fluorescence intensity | No |
## Experiment Lifecycle
```
Draft → WaitingForConfirmation → QuoteSent → WaitingForMaterials → InQueue → InProduction → DataAnalysis → InReview → Done
```
| Status | Who Acts | Description |
|---|---|---|
| `Draft` | You | Editable, no cost commitment |
| `WaitingForConfirmation` | Adaptyv | Under review, quote being prepared |
| `QuoteSent` | You | Review and confirm the quote |
| `WaitingForMaterials` | Adaptyv | Gene fragments and target ordered |
| `InQueue` | Adaptyv | Materials arrived, queued for lab |
| `InProduction` | Adaptyv | Assay running |
| `DataAnalysis` | Adaptyv | Raw data processing and QC |
| `InReview` | Adaptyv | Final validation |
| `Done` | You | Results available |
| `Canceled` | Either | Experiment canceled |
The `results_status` field on an experiment tracks: `none`, `partial`, or `all`.
## Common Workflows
### 1. Submit a Binding Screen (Step by Step)
```python
# 1. Find a target
targets = client.targets.list(search="EGFR", selfservice_only=True)
target_id = targets.items[0].id
# 2. Preview cost
estimate = client.experiments.cost_estimate({
"experiment_spec": {
"experiment_type": "screening",
"method": "bli",
"target_id": target_id,
"sequences": {"seq1": "EVQLVESGGGLVQ...", "seq2": "MKVLVAG..."},
"n_replicates": 3
}
})
# 3. Create experiment (starts as Draft)
exp = client.experiments.create({
"name": "EGFR binder screen batch 1",
"experiment_spec": {
"experiment_type": "screening",
"method": "bli",
"target_id": target_id,
"sequences": {"seq1": "EVQLVESGGGLVQ...", "seq2": "MKVLVAG..."},
"n_replicates": 3
}
})
# 4. Submit for review
client.experiments.submit(exp.experiment_id)
# 5. Poll or use webhooks until Done
# 6. Retrieve results
results = client.experiments.get_results(exp.experiment_id)
```
### 2. Automated Pipeline (Skip Draft + Auto-Accept Quote)
```python
exp = client.experiments.create({
"name": "Auto pipeline run",
"experiment_spec": {...},
"skip_draft": True,
"auto_accept_quote": True,
"webhook_url": "https://my-server.com/webhook"
})
# Webhook fires on each status transition; poll or wait for Done
```
### 3. Using Webhooks
Pass `webhook_url` when creating an experiment. Adaptyv POSTs to that URL on every status transition with the experiment ID, previous status, and new status.
## Sequences
- Simple format: `{"seq1": "EVQLVESGGGLVQPGGSLRLSCAAS"}`
- Rich format: `{"seq1": {"aa_string": "EVQLVESGGGLVQ...", "control": false, "metadata": {"type": "scfv"}}}`
- Multi-chain: use colon separator — `"MVLS:EVQL"`
- Valid amino acids: A, C, D, E, F, G, H, I, K, L, M, N, P, Q, R, S, T, V, W, Y (case-insensitive, stored uppercase)
- Sequences can only be added to experiments in `Draft` status
## Filtering, Sorting, and Pagination
All list endpoints support pagination (`limit` 1-100, default 50; `offset`), search (free-text on name fields), and sorting.
**Filtering** uses s-expression syntax via the `filter` query parameter:
- Comparison: `eq(field,value)`, `neq`, `gt`, `gte`, `lt`, `lte`, `contains(field,substring)`
- Range/set: `between(field,lo,hi)`, `in(field,v1,v2,...)`
- Logic: `and(expr1,expr2,...)`, `or(...)`, `not(expr)`
- Null: `is_null(field)`, `is_not_null(field)`
- JSONB: `at(field,key)` — e.g., `eq(at(metadata,score),42)`
- Cast: `float()`, `int()`, `text()`, `timestamp()`, `date()`
**Sorting** uses `asc(field)` or `desc(field)`, comma-separated (max 8):
```
sort=desc(created_at),asc(name)
```
**Example:** `filter=and(gte(created_at,2026-01-01),eq(status,done))`
## Error Handling
All errors return:
```json
{
"error": "Human-readable description",
"request_id": "req_019462a4-b1c2-7def-8901-23456789abcd"
}
```
The `request_id` is also in the `x-request-id` response header — include it when contacting support.
## Token Management
Tokens use Biscuit-based cryptographic attenuation. You can create restricted tokens scoped by organization, resource type, actions (read/create/update), and expiry via `POST /tokens/attenuate`. Revoking a token (`POST /tokens/revoke`) revokes it and all its descendants.
## Detailed API Reference
For the full list of all 32 endpoints with request/response schemas, read `references/api-endpoints.md`.
skills/adaptyv/references/api-endpoints.md
+690
View File
@@ -0,0 +1,690 @@
# Adaptyv Bio Foundry API — Complete Endpoint Reference
Base URL: `https://foundry-api-public.adaptyvbio.com/api/v1`
OpenAPI spec: `GET /openapi.json`
## Table of Contents
- [Experiments](#experiments)
- [Sequences](#sequences)
- [Results](#results)
- [Targets](#targets)
- [Quotes](#quotes)
- [Tokens](#tokens)
- [Updates](#updates)
- [Feedback](#feedback)
---
## Experiments
### POST /experiments — Create experiment
Creates a new experiment. Starts in `Draft` status by default.
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `name` | string | Yes | Human-readable name |
| `experiment_spec` | ExperimentSpec | Yes | Experiment definition (see below) |
| `skip_draft` | boolean | No (default false) | Bypass Draft, go straight to WaitingForConfirmation |
| `auto_accept_quote` | boolean | No (default false) | Auto-accept quote and create invoice |
| `webhook_url` | string/null | No | URL for status-change POST notifications |
**ExperimentSpec:**
| Field | Type | Required | Description |
|---|---|---|---|
| `experiment_type` | string | Yes | `affinity`, `screening`, `thermostability`, `fluorescence`, or `expression` |
| `method` | string | Required for binding types | `bli` or `spr` |
| `target_id` | uuid | Required for binding types | Target UUID from catalog |
| `sequences` | object | Yes | Map of name → amino acid string or rich object |
| `n_replicates` | integer | Recommended (default 3) | Technical replicates (min 1) |
| `antigen_concentrations` | number[] | No (affinity only) | Defaults to `[1000.0, 316.2, 100.0, 31.6, 0.0]` nM |
| `parameters` | object | No | Experiment-specific settings |
**Field requirements by experiment type:**
| Field | Affinity | Screening | Thermostability | Fluorescence | Expression |
|---|---|---|---|---|---|
| `experiment_type` | required | required | required | required | required |
| `method` | required | required | — | — | — |
| `target_id` | required | required | — | — | — |
| `sequences` | required | required | required | required | required |
| `n_replicates` | recommended | recommended | optional | optional | optional |
| `antigen_concentrations` | optional | — | — | — | — |
**Response (201):**
| Field | Type | Description |
|---|---|---|
| `experiment_id` | string | UUID of new experiment |
| `error` | string/null | Error message if validation fails |
| `stripe_hosted_invoice_url` | string/null | Present when `auto_accept_quote` created an invoice |
| `stripe_invoice_id` | string/null | Stripe invoice ID |
**Status codes:** 201, 400, 401, 403, 404
---
### GET /experiments — List experiments
Lists experiments accessible to caller, sorted by creation date (newest first).
**Query params:** `limit`, `offset`, `filter`, `search`, `sort`
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Unique identifier |
| `code` | string | e.g., "EXP-2024-001" |
| `name` | string/null | Human-readable name |
| `status` | ExperimentStatus | Current lifecycle status |
| `experiment_type` | ExperimentType | affinity/screening/thermostability/fluorescence/expression |
| `results_status` | ResultsStatus | none/partial/all |
| `created_at` | datetime | ISO 8601 |
| `experiment_url` | string | URL to Foundry portal |
| `stripe_invoice_url` | string/null | Invoice URL |
| `stripe_quote_url` | string/null | Quote URL |
**Status codes:** 200, 401
---
### GET /experiments/{experiment_id} — Get experiment
Returns full metadata for a single experiment.
**Path param:** `experiment_id` (uuid)
**Response:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Unique identifier |
| `code` | string | Experiment code |
| `status` | ExperimentStatus | Current status |
| `experiment_spec` | ExperimentSpec | Full experiment definition |
| `results_status` | ResultsStatus | none/partial/all |
| `created_at` | datetime | ISO 8601 |
| `experiment_url` | string | Portal URL |
| `costs` | object | Cost breakdown |
**Status codes:** 200, 401, 404, 500
---
### PATCH /experiments/{experiment_id} — Update experiment
Modify an existing experiment. Draft experiments allow full edits; after quote generation, only `name`, `description`, and `webhook_url` are editable.
**Path param:** `experiment_id` (uuid)
**Request body:** All fields optional — only provided fields are updated.
**Status codes:** 200, 400, 401, 404, 409
---
### POST /experiments/{experiment_id}/submit — Submit experiment
Submits a draft experiment for review. Advances from `Draft` to `WaitingForConfirmation`.
**Path param:** `experiment_id` (uuid)
**Response:**
| Field | Type | Description |
|---|---|---|
| `experiment_id` | string | Experiment UUID |
**Status codes:** 200, 401, 403, 404, 409, 500
---
### POST /experiments/cost-estimate — Estimate cost
Calculates cost without creating an experiment.
**Request body:**
```json
{
"experiment_spec": {
"experiment_type": "screening",
"method": "bli",
"target_id": "...",
"sequences": {"seq1": "MKTL..."},
"n_replicates": 3
}
}
```
**Response:**
| Field | Type | Description |
|---|---|---|
| `pricing_version` | string | e.g., "v1_2026-01-20" |
| `assay` | object | Per-type costs with base and replicate pricing |
| `materials` | object | Target material costs (binding experiments) |
| `total_cents` | integer | Sum in USD cents |
All prices exclude VAT; taxes calculated at invoicing. Targets without self-service pricing return incomplete estimates.
**Status codes:** 200, 400, 401
---
### GET /experiments/{experiment_id}/quote — Get quote
Returns quote metadata (totals, currency, status, expiration).
**Path param:** `experiment_id` (uuid)
**Response:**
| Field | Type | Description |
|---|---|---|
| `experiment_id` | string | Experiment UUID |
| `stripe_quote_url` | string | Stripe quote URL |
| `amount_total` | int64 | Total in smallest currency unit |
| `amount_subtotal` | int64 | Subtotal |
| `currency` | string | ISO currency code (e.g., "usd") |
| `status` | string | Quote status |
| `expires_at` | datetime/null | Expiration time |
**Status codes:** 200, 401, 403, 404, 500
---
### GET /experiments/{experiment_id}/quote/pdf — Get quote PDF
Returns the quote as a PDF file (`application/pdf`).
**Path param:** `experiment_id` (uuid)
**Status codes:** 200, 401, 403, 404, 500
---
### POST /experiments/{experiment_id}/quote/confirm — Accept quote (by experiment)
Accepts Stripe quote, creates draft invoice, transitions to `WaitingForMaterials`.
**Path param:** `experiment_id` (uuid)
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `purchase_order_number` | string/null | No | PO number for your records |
| `notes` | string/null | No | Reserved |
**Response:**
| Field | Type | Description |
|---|---|---|
| `id` | string | Quote ID |
| `status` | StripeQuoteStatus | New status |
| `hosted_invoice_url` | string/null | Stripe payment URL |
| `invoice_id` | string/null | Generated invoice ID |
**Status codes:** 200, 401, 403, 404, 409
---
### GET /experiments/{experiment_id}/invoice — Get invoice
Returns invoice metadata including hosted payment URL.
**Path param:** `experiment_id` (uuid)
**Status codes:** 200, 401, 403, 404, 500
---
### GET /experiments/{experiment_id}/results — List results for experiment
Returns all analysis results for a specific experiment.
**Path param:** `experiment_id` (uuid)
**Query params:** `limit`, `offset`, `filter`, `sort`
**Status codes:** 200, 400, 401, 403, 404
---
### GET /experiments/{experiment_id}/sequences — List sequences for experiment
Returns all sequences for a specific experiment, sorted newest first.
**Path param:** `experiment_id` (uuid)
**Query params:** `limit`, `offset`, `search`, `sort`
**Status codes:** 200, 400, 401, 403, 404
---
### GET /experiments/{experiment_id}/updates — List experiment updates
Returns updates for one experiment, oldest first. Types: `status_change`, `progress`, `error`.
**Path param:** `experiment_id` (uuid)
**Query params:** `limit`, `offset`, `filter`, `sort`
Filter example: `filter=eq(type,status_change)`
---
## Sequences
### GET /sequences — List sequences
Returns sequences from all experiments, sorted newest first.
**Query params:** `limit`, `offset`, `search`, `sort`, `experiment_id` (filter by experiment UUID)
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Unique identifier |
| `name` | string/null | Optional name |
| `aa_preview` | string/null | Truncated preview (first 50 chars) |
| `length` | int32 | Sequence length in amino acids |
| `experiment_id` | uuid | Parent experiment |
| `experiment_code` | string | Human-readable experiment code |
| `is_control` | boolean | Whether this is a control |
| `created_at` | datetime | Creation timestamp |
**Status codes:** 200, 401
---
### GET /sequences/{sequence_id} — Get sequence
Returns full details including complete amino acid string.
**Path param:** `sequence_id` (uuid)
**Response:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Unique identifier |
| `aa_string` | string/null | Complete amino acid sequence |
| `length` | int32 | Length in amino acids |
| `is_control` | boolean | Control flag |
| `metadata` | object | Sequence-level annotations |
| `experiment` | object | Parent experiment reference |
| `created_at` | datetime | Creation timestamp |
**Status codes:** 200, 401, 403, 404, 500
---
### POST /sequences — Add sequences to experiment
Appends sequences to a **Draft** experiment identified by its human-readable code.
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `experiment_code` | string | Yes | e.g., "PROJ-001" |
| `sequences` | array | Yes | Array of sequence entries |
**Each sequence entry:**
| Field | Type | Required | Description |
|---|---|---|---|
| `aa_string` | string | Yes | Amino acid sequence |
| `name` | string | No | Human-readable name |
| `control` | boolean | No | Whether this is a control |
| `metadata` | object | No | Annotations |
**Response (201):**
| Field | Type | Description |
|---|---|---|
| `added_count` | int32 | Number of sequences added |
| `experiment_id` | string | Experiment UUID |
| `experiment_code` | string | Experiment code |
| `sequence_ids` | array | IDs of added sequences |
**Status codes:** 201, 400, 404, 409 (experiment not in Draft), 500
---
## Results
### GET /results — List results
Lists completed analysis results, sorted newest first. Results appear when `results_status` reaches `partial` or `all`.
**Query params:** `limit`, `offset`, `filter`, `search`, `sort`
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Result identifier |
| `title` | string | Human-readable title |
| `experiment_id` | uuid | Associated experiment |
| `result_type` | string | e.g., "affinity", "thermostability" |
| `summary` | array | Key results (type-specific, see below) |
| `metadata` | object | Extended metadata (e.g., instrument info) |
| `data_package_url` | string/null | Download URL for raw data package |
| `created_at` | datetime | When result was generated |
**AffinityResult summary fields:** `kd_mean`, `kd_std`, `kon_mean`, `kon_log_std`, `koff_mean`, `koff_std`, `replicates` (array with per-replicate `kd`, `kon`, `koff`, `binding_strength`, `kon_method`, `koff_method`, `replicate` index), `sequence`, `target_id`
**ThermostabilityResult summary fields:** Tm values and melting curves
**Status codes:** 200, 401
---
### GET /results/{result_id} — Get result
Returns detailed result data including full summary array.
**Path param:** `result_id` (uuid)
**Status codes:** 200, 401, 403, 404, 500
---
## Targets
### GET /targets — List targets
Lists validated antigens available for experiments.
**Query params:**
| Parameter | Type | Description |
|---|---|---|
| `limit` | int | Max items (1-100, default 50) |
| `offset` | int | Skip count |
| `search` | string | Free-text search on product name |
| `sort` | string | Sort expression |
| `selfservice_only` | boolean | Only targets with self-service pricing |
| `show_conjugated` | boolean | Include conjugated targets (default: unconjugated only) |
| `detailed` | boolean | Populate `details` block with enrichment data |
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Target UUID (use as `experiment_spec.target_id`) |
| `name` | string | Target name |
| `vendor_name` | string | Vendor name |
| `catalog_number` | string | Vendor catalog/SKU number |
| `url` | string | Target URL |
| `pricing` | object/null | Self-service pricing (null = custom quote required) |
| `details` | object/null | Enrichment data (gene names, structures, sequence, bioactivity) |
**Status codes:** 200, 401
---
### GET /targets/{target_id} — Get target
Returns catalog record for a single target.
**Path param:** `target_id` (uuid)
**Status codes:** 200, 400, 401, 403, 404, 500
---
### POST /targets/request-custom — Submit custom target request
Submit a new custom target for staff review. At least one of `sequence` or `pdb_id` must be provided.
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `name` | string | Yes | Display name |
| `product_id` | string | Yes | Must be unique within organization |
| `sequence` | string/null | At least one | Amino acid sequence |
| `pdb_id` | string/null | At least one | PDB identifier |
| `pdb_file` | string/null | No | PDB file content |
| `molecular_weight` | number/null | No | Weight in kDa |
| `note` | string/null | No | Additional notes |
**Status codes:** 201, 400, 401, 403, 500
---
### GET /targets/request-custom — List custom target requests
Returns custom target requests for your organization, sorted newest first.
**Query params:** `limit`, `offset`, `filter`, `sort`
Filter example: `filter=eq(status,pending_review)`
---
### GET /targets/request-custom/{request_id} — Get custom target request
**Path param:** `request_id` (uuid)
**Response:**
| Field | Type | Description |
|---|---|---|
| `id` | uuid | Request identifier |
| `name` | string | Target name |
| `product_id` | string | Your product ID |
| `status` | string | e.g., "pending_review" |
| `material_id` | string/null | Linked catalog ID if approved |
| `molecular_weight` | number/null | Weight in kDa |
| `note` | string/null | User notes |
| `created_at` | datetime | Created |
| `updated_at` | datetime | Last updated |
**Status codes:** 200, 401, 403, 404, 500
---
## Quotes
### GET /quotes — List quotes
Returns all quotes for caller's organization.
**Query params:** `limit`, `offset`, `filter`, `sort`
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | string | Quote identifier |
| `quote_number` | string | Human-readable quote number |
| `organization_id` | uuid | Organization |
| `amount_cents` | int | Amount in cents |
| `currency` | string | ISO 4217 code |
| `status` | StripeQuoteStatus | Quote status |
| `valid_until` | datetime | Expiration |
| `created_at` | datetime | Creation timestamp |
---
### GET /quotes/{quote_id} — Get quote
Returns full quote document with itemized pricing.
**Path param:** `quote_id` (string, e.g., "qt_1Abc2DefGhi")
**Response:**
| Field | Type | Description |
|---|---|---|
| `id` | string | Quote identifier |
| `quote_number` | string | Reference number |
| `organization_id` | uuid | Organization |
| `organization_name` | string | Organization name |
| `line_items` | array | Itemized pricing |
| `subtotal_cents` | int | Subtotal in cents |
| `tax_cents` | int | Tax in cents |
| `total_cents` | int | Total in cents |
| `currency` | string | ISO 4217 |
| `status` | StripeQuoteStatus | Current status |
| `valid_until` | datetime | Expiration |
| `notes` | string | Special pricing info |
| `terms_and_conditions` | string | Terms |
| `stripe_quote_url` | string | Stripe URL |
| `created_at` | datetime | Created |
**Status codes:** 200, 401, 403, 404, 500
---
### POST /quotes/{quote_id}/confirm — Accept quote
Finalizes quote, creates draft invoice, advances experiment to `WaitingForMaterials`.
**Path param:** `quote_id` (string)
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `purchase_order_number` | string/null | No | PO number |
| `notes` | string/null | No | Reserved |
**Response:** `id`, `status`, `hosted_invoice_url`, `invoice_id`
**Status codes:** 200, 403, 404, 409, 500
---
### POST /quotes/{quote_id}/reject — Reject quote
Cancels quote; linked experiment reverts to `Draft`.
**Path param:** `quote_id` (string)
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `reason` | QuoteRejectionReason | Yes | Primary reason |
| `feedback` | string/null | No | Additional feedback |
**Response:** `id`, `status` (canceled)
**Status codes:** 200, 403, 404, 409, 500
---
## Tokens
### GET /tokens — List tokens
Returns all tokens (root and attenuated) the caller owns.
**Query params:** `limit`, `offset`
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | string | Token identifier |
| `name` | string | Human-readable label |
| `kind` | string | "root" or "attenuated" |
| `created_at` | datetime | Created |
| `expires_at` | datetime/null | Expiration (null = no expiry) |
| `revoked_at` | datetime/null | Revocation timestamp |
| `parent_token_id` | string/null | Parent (null for root) |
| `root_token_id` | string/null | Root of derivation tree |
| `attenuation_spec` | object/null | Restrictions (null for root) |
---
### POST /tokens/attenuate — Attenuate token
Creates a restricted version of an existing token using Biscuit cryptographic attenuation.
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `token` | string | Yes | Existing token (`abs0_{slug}{biscuit_base64}`) |
| `attenuation` | AttenuationSpec | Yes | Restrictions to apply |
| `name` | string | Yes | Human-readable label |
| `attenuated_parent_token_id` | uuid/null | No | Parent ID for chained attenuation |
**Restriction types:** Organization, Resource (experiments/results), Action (read/create/update), Expiry
**Response (201):** `id` (database ID), `token` (new attenuated token string)
**Status codes:** 201, 400, 401, 403
---
### POST /tokens/revoke — Revoke token and lineage
Revokes the calling token's root and all attenuated descendants. Idempotent.
**Response:**
| Field | Type | Description |
|---|---|---|
| `token_id` | string | Root token ID revoked |
| `revoked_at` | datetime | Revocation timestamp |
| `children_revoked` | int64 | Child tokens newly revoked |
**Status codes:** 200, 403, 404
---
## Updates
### GET /updates — List updates
Returns the experiment update feed (newest first): status changes, progress, errors.
**Query params:** `limit`, `offset`, `filter`, `sort`
**Filter examples:**
- `filter=eq(experiment_id,<uuid>)`
- `filter=in(experiment_id,uuid1,uuid2)`
- `filter=eq(type,status_change)`
**Response item:**
| Field | Type | Description |
|---|---|---|
| `id` | string | Update identifier |
| `experiment_id` | uuid | Associated experiment |
| `experiment_code` | string | Human-readable code |
| `name` | string | Update description |
| `timestamp` | datetime | When the update occurred |
---
## Feedback
### POST /feedback/submit — Submit feedback
For bug reports, feature requests, or general feedback.
**Request body:**
| Field | Type | Required | Description |
|---|---|---|---|
| `request_uuid` | uuid | Yes | UUID from the problematic API request |
| `feedback_type` | FeedbackType | Yes | `feature_request`, `feedback`, or `bug_report` |
| `title` | string/null | No | Short title |
| `json_body` | object/null | At least one | Structured error details |
| `human_note` | string/null | At least one | Free-form description |
**Response (201):** `reference` (feedback reference), `message` (confirmation)
**Status codes:** 201, 400, 401, 500
skills/aeon/SKILL.md
+400
View File
@@ -0,0 +1,400 @@
---
name: aeon
description: This skill should be used for time series machine learning tasks including classification, regression, clustering, forecasting, anomaly detection, segmentation, and similarity search. Use when working with temporal data, sequential patterns, or time-indexed observations requiring specialized algorithms beyond standard ML approaches. Particularly suited for univariate and multivariate time series analysis with scikit-learn compatible APIs.
license: BSD-3-Clause license
allowed-tools: Read Write Edit Bash
compatibility: Requires Python 3.10+ and the aeon package (uv pip install). Optional aeon[all_extras] for deep learning and extended dependencies.
metadata: {"version": "1.0", "skill-author": "K-Dense Inc."}
---
# Aeon Time Series Machine Learning
## Overview
Aeon is a scikit-learn compatible Python toolkit for time series machine learning ([aeon-toolkit.org](https://www.aeon-toolkit.org/)). It provides algorithms across classification, regression, clustering, forecasting, anomaly detection, segmentation, similarity search, distances, transformations, benchmarking, and visualization — with a consistent estimator API.
**Version note:** Examples target **aeon 1.x** (stable docs: v1.4.0, March 2026). The v1.0 release reworked forecasting and transformations; import paths differ from aeon 0.x/sktime-era code.
## When to Use This Skill
Apply this skill when:
- Classifying or predicting from time series data
- Detecting anomalies or change points in temporal sequences
- Clustering similar time series patterns
- Forecasting future values
- Finding repeated patterns (motifs) or unusual subsequences (discords)
- Comparing time series with specialized distance metrics
- Extracting features from temporal data
## Installation
Requires **Python 3.10+** (3.11+ recommended). Pin a 1.x release for reproducibility:
```bash
uv pip install "aeon>=1.4,<2"
```
For deep learning forecasters/classifiers and other optional estimators:
```bash
uv pip install "aeon[all_extras]>=1.4,<2"
```
On zsh, quote the extras: `uv pip install "aeon[all_extras]>=1.4,<2"`.
### Experimental modules
Upstream treats **forecasting**, **anomaly_detection**, **segmentation**, **similarity_search**, and **visualisation** as experimental — interfaces may change between minor releases. Prefer stable modules (classification, regression, clustering, distances, transformations) for production pipelines unless you need these tasks.
## Core Capabilities
### 1. Time Series Classification
Categorize time series into predefined classes. See `references/classification.md` for complete algorithm catalog.
**Quick Start:**
```python
from aeon.classification.convolution_based import RocketClassifier
from aeon.datasets import load_classification
# Load data
X_train, y_train = load_classification("GunPoint", split="train")
X_test, y_test = load_classification("GunPoint", split="test")
# Train classifier
clf = RocketClassifier(n_kernels=10000)
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
```
**Algorithm Selection:**
- **Speed + Performance**: `MiniRocketClassifier`, `Arsenal`
- **Maximum Accuracy**: `HIVECOTEV2`, `InceptionTimeClassifier`
- **Interpretability**: `ShapeletTransformClassifier`, `Catch22Classifier`
- **Small Datasets**: `KNeighborsTimeSeriesClassifier` with DTW distance
### 2. Time Series Regression
Predict continuous values from time series. See `references/regression.md` for algorithms.
**Quick Start:**
```python
from aeon.regression.convolution_based import RocketRegressor
from aeon.datasets import load_regression
X_train, y_train = load_regression("Covid3Month", split="train")
X_test, y_test = load_regression("Covid3Month", split="test")
reg = RocketRegressor()
reg.fit(X_train, y_train)
predictions = reg.predict(X_test)
```
### 3. Time Series Clustering
Group similar time series without labels. See `references/clustering.md` for methods.
**Quick Start:**
```python
from aeon.clustering import TimeSeriesKMeans
clusterer = TimeSeriesKMeans(
n_clusters=3,
distance="dtw",
averaging_method="ba"
)
labels = clusterer.fit_predict(X_train)
centers = clusterer.cluster_centers_
```
### 4. Forecasting
Predict future time series values (experimental module in aeon 1.x). See `references/forecasting.md` for forecasters.
**Quick Start:**
```python
import numpy as np
from aeon.forecasting import NaiveForecaster
from aeon.forecasting.stats import ARIMA
y_train = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])
# Set horizon in the constructor; predict passes the series to forecast from
naive = NaiveForecaster(strategy="last", horizon=5)
naive.fit(y_train)
y_pred = naive.predict(y_train)
# ARIMA uses p/d/q (not order=); multi-step via iterative_forecast
arima = ARIMA(p=1, d=1, q=1)
arima.fit(y_train)
y_pred = arima.iterative_forecast(y_train, prediction_horizon=5)
```
### 5. Anomaly Detection
Identify unusual patterns or outliers. See `references/anomaly_detection.md` for detectors.
**Quick Start:**
```python
from aeon.anomaly_detection import STOMP
detector = STOMP(window_size=50)
anomaly_scores = detector.fit_predict(y)
# Higher scores indicate anomalies
threshold = np.percentile(anomaly_scores, 95)
anomalies = anomaly_scores > threshold
```
### 6. Segmentation
Partition time series into regions with change points. See `references/segmentation.md`.
**Quick Start:**
```python
from aeon.segmentation import ClaSPSegmenter
segmenter = ClaSPSegmenter()
change_points = segmenter.fit_predict(y)
```
### 7. Similarity Search
Find similar patterns within or across time series. See `references/similarity_search.md`.
**Quick Start:**
```python
from aeon.similarity_search import StompMotif
# Find recurring patterns
motif_finder = StompMotif(window_size=50, k=3)
motifs = motif_finder.fit_predict(y)
```
## Feature Extraction and Transformations
Transform time series for feature engineering. See `references/transformations.md`.
**ROCKET Features:**
```python
from aeon.transformations.collection.convolution_based import RocketTransformer
rocket = RocketTransformer()
X_features = rocket.fit_transform(X_train)
# Use features with any sklearn classifier
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier()
clf.fit(X_features, y_train)
```
**Statistical Features:**
```python
from aeon.transformations.collection.feature_based import Catch22
catch22 = Catch22()
X_features = catch22.fit_transform(X_train)
```
**Preprocessing:**
```python
from aeon.transformations.collection import MinMaxScaler, Normalizer
scaler = Normalizer() # Z-normalization
X_normalized = scaler.fit_transform(X_train)
```
## Distance Metrics
Specialized temporal distance measures. See `references/distances.md` for complete catalog.
**Usage:**
```python
from aeon.distances import dtw_distance, dtw_pairwise_distance
# Single distance
distance = dtw_distance(x, y, window=0.1)
# Pairwise distances
distance_matrix = dtw_pairwise_distance(X_train)
# Use with classifiers
from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier
clf = KNeighborsTimeSeriesClassifier(
n_neighbors=5,
distance="dtw",
distance_params={"window": 0.2}
)
```
**Available Distances:**
- **Elastic**: DTW, DDTW, WDTW, ERP, EDR, LCSS, TWE, MSM
- **Lock-step**: Euclidean, Manhattan, Minkowski
- **Shape-based**: Shape DTW, SBD
## Deep Learning Networks
Neural architectures for time series. See `references/networks.md`.
**Architectures:**
- Convolutional: `FCNClassifier`, `ResNetClassifier`, `InceptionTimeClassifier`
- Recurrent: `RecurrentNetwork`, `TCNNetwork`
- Autoencoders: `AEFCNClusterer`, `AEResNetClusterer`
**Usage:**
```python
from aeon.classification.deep_learning import InceptionTimeClassifier
clf = InceptionTimeClassifier(n_epochs=100, batch_size=32)
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
```
## Datasets and Benchmarking
Load standard benchmarks and evaluate performance. See `references/datasets_benchmarking.md`.
**Load Datasets:**
```python
from aeon.datasets import load_classification, load_gunpoint, load_regression
# Classification (generic loader or dataset-specific helper)
X_train, y_train = load_classification("GunPoint", split="train")
X_train, y_train = load_gunpoint(split="train") # same UCR dataset
# Regression
X_train, y_train = load_regression("Covid3Month", split="train")
```
**Benchmarking:**
```python
from aeon.benchmarking import get_estimator_results
# Compare with published results
published = get_estimator_results("ROCKET", "GunPoint")
```
## Common Workflows
### Classification Pipeline
```python
from aeon.transformations.collection import Normalizer
from aeon.classification.convolution_based import RocketClassifier
from sklearn.pipeline import Pipeline
pipeline = Pipeline([
('normalize', Normalizer()),
('classify', RocketClassifier())
])
pipeline.fit(X_train, y_train)
accuracy = pipeline.score(X_test, y_test)
```
### Feature Extraction + Traditional ML
```python
from aeon.transformations.collection import RocketTransformer
from sklearn.ensemble import GradientBoostingClassifier
# Extract features
rocket = RocketTransformer()
X_train_features = rocket.fit_transform(X_train)
X_test_features = rocket.transform(X_test)
# Train traditional ML
clf = GradientBoostingClassifier()
clf.fit(X_train_features, y_train)
predictions = clf.predict(X_test_features)
```
### Anomaly Detection with Visualization
```python
from aeon.anomaly_detection import STOMP
import matplotlib.pyplot as plt
detector = STOMP(window_size=50)
scores = detector.fit_predict(y)
plt.figure(figsize=(15, 5))
plt.subplot(2, 1, 1)
plt.plot(y, label='Time Series')
plt.subplot(2, 1, 2)
plt.plot(scores, label='Anomaly Scores', color='red')
plt.axhline(np.percentile(scores, 95), color='k', linestyle='--')
plt.show()
```
## Best Practices
### Data Preparation
1. **Normalize**: Most algorithms benefit from z-normalization
```python
from aeon.transformations.collection import Normalizer
normalizer = Normalizer()
X_train = normalizer.fit_transform(X_train)
X_test = normalizer.transform(X_test)
```
2. **Handle Missing Values**: Impute before analysis
```python
from aeon.transformations.collection import SimpleImputer
imputer = SimpleImputer(strategy='mean')
X_train = imputer.fit_transform(X_train)
```
3. **Check Data Format**: Collections use `(n_cases, n_channels, n_timepoints)`; single series use `(n_channels, n_timepoints)` (see [data format](https://www.aeon-toolkit.org/en/stable/api_reference/data_format.html))
### Model Selection
1. **Start Simple**: Begin with ROCKET variants before deep learning
2. **Use Validation**: Split training data for hyperparameter tuning
3. **Compare Baselines**: Test against simple methods (1-NN Euclidean, Naive)
4. **Consider Resources**: ROCKET for speed, deep learning if GPU available
### Algorithm Selection Guide
**For Fast Prototyping:**
- Classification: `MiniRocketClassifier`
- Regression: `MiniRocketRegressor`
- Clustering: `TimeSeriesKMeans` with Euclidean
**For Maximum Accuracy:**
- Classification: `HIVECOTEV2`, `InceptionTimeClassifier`
- Regression: `InceptionTimeRegressor`
- Forecasting: `AutoARIMA`, `AutoETS`, `TCNForecaster` (requires `[all_extras]` for deep learning)
**For Interpretability:**
- Classification: `ShapeletTransformClassifier`, `Catch22Classifier`
- Features: `Catch22`, `TSFresh`
**For Small Datasets:**
- Distance-based: `KNeighborsTimeSeriesClassifier` with DTW
- Avoid: Deep learning (requires large data)
## Reference Documentation
Detailed information available in `references/`:
- `classification.md` - All classification algorithms
- `regression.md` - Regression methods
- `clustering.md` - Clustering algorithms
- `forecasting.md` - Forecasting approaches
- `anomaly_detection.md` - Anomaly detection methods
- `segmentation.md` - Segmentation algorithms
- `similarity_search.md` - Pattern matching and motif discovery
- `transformations.md` - Feature extraction and preprocessing
- `distances.md` - Time series distance metrics
- `networks.md` - Deep learning architectures
- `datasets_benchmarking.md` - Data loading and evaluation tools
## Additional Resources
- Documentation: https://www.aeon-toolkit.org/
- GitHub: https://github.com/aeon-toolkit/aeon
- Examples: https://www.aeon-toolkit.org/en/stable/examples.html
- API Reference: https://www.aeon-toolkit.org/en/stable/api_reference.html
skills/aeon/references/anomaly_detection.md
+154
View File
@@ -0,0 +1,154 @@
# Anomaly Detection
Aeon provides anomaly detection methods for identifying unusual patterns in time series at both series and collection levels.
## Collection Anomaly Detectors
Detect anomalous time series within a collection:
- `ClassificationAdapter` - Adapts classifiers for anomaly detection
- Train on normal data, flag outliers during prediction
- **Use when**: Have labeled normal data, want classification-based approach
- `OutlierDetectionAdapter` - Wraps sklearn outlier detectors
- Works with IsolationForest, LOF, OneClassSVM
- **Use when**: Want to use sklearn anomaly detectors on collections
## Series Anomaly Detectors
Detect anomalous points or subsequences within a single time series.
### Distance-Based Methods
Use similarity metrics to identify anomalies:
- `CBLOF` - Cluster-Based Local Outlier Factor
- Clusters data, identifies outliers based on cluster properties
- **Use when**: Anomalies form sparse clusters
- `KMeansAD` - K-means based anomaly detection
- Distance to nearest cluster center indicates anomaly
- **Use when**: Normal patterns cluster well
- `LeftSTAMPi` - Left STAMP incremental
- Matrix profile for online anomaly detection
- **Use when**: Streaming data, need online detection
- `STOMP` - Scalable Time series Ordered-search Matrix Profile
- Computes matrix profile for subsequence anomalies
- **Use when**: Discord discovery, motif detection
- `MERLIN` - Matrix profile-based method
- Efficient matrix profile computation
- **Use when**: Large time series, need scalability
- `LOF` - Local Outlier Factor adapted for time series
- Density-based outlier detection
- **Use when**: Anomalies in low-density regions
- `ROCKAD` - ROCKET-based semi-supervised detection
- Uses ROCKET features for anomaly identification
- **Use when**: Have some labeled data, want feature-based approach
### Distribution-Based Methods
Analyze statistical distributions:
- `COPOD` - Copula-Based Outlier Detection
- Models marginal and joint distributions
- **Use when**: Multi-dimensional time series, complex dependencies
- `DWT_MLEAD` - Discrete Wavelet Transform Multi-Level Anomaly Detection
- Decomposes series into frequency bands
- **Use when**: Anomalies at specific frequencies
### Isolation-Based Methods
Use isolation principles:
- `IsolationForest` - Random forest-based isolation
- Anomalies easier to isolate than normal points
- **Use when**: High-dimensional data, no assumptions about distribution
- `OneClassSVM` - Support vector machine for novelty detection
- Learns boundary around normal data
- **Use when**: Well-defined normal region, need robust boundary
- `STRAY` - Streaming Robust Anomaly Detection
- Robust to data distribution changes
- **Use when**: Streaming data, distribution shifts
### External Library Integration
- `PyODAdapter` - Bridges PyOD library to aeon
- Access 40+ PyOD anomaly detectors
- **Use when**: Need specific PyOD algorithm
## Quick Start
```python
from aeon.anomaly_detection import STOMP
import numpy as np
# Create time series with anomaly
y = np.concatenate([
np.sin(np.linspace(0, 10, 100)),
[5.0], # Anomaly spike
np.sin(np.linspace(10, 20, 100))
])
# Detect anomalies
detector = STOMP(window_size=10)
anomaly_scores = detector.fit_predict(y)
# Higher scores indicate more anomalous points
threshold = np.percentile(anomaly_scores, 95)
anomalies = anomaly_scores > threshold
```
## Point vs Subsequence Anomalies
- **Point anomalies**: Single unusual values
- Use: COPOD, DWT_MLEAD, IsolationForest
- **Subsequence anomalies** (discords): Unusual patterns
- Use: STOMP, LeftSTAMPi, MERLIN
- **Collective anomalies**: Groups of points forming unusual pattern
- Use: Matrix profile methods, clustering-based
## Evaluation Metrics
Specialized metrics for anomaly detection:
```python
from aeon.benchmarking.metrics.anomaly_detection import (
range_precision,
range_recall,
range_f_score,
roc_auc_score
)
# Range-based metrics account for window detection
precision = range_precision(y_true, y_pred, alpha=0.5)
recall = range_recall(y_true, y_pred, alpha=0.5)
f1 = range_f_score(y_true, y_pred, alpha=0.5)
```
## Algorithm Selection
- **Speed priority**: KMeansAD, IsolationForest
- **Accuracy priority**: STOMP, COPOD
- **Streaming data**: LeftSTAMPi, STRAY
- **Discord discovery**: STOMP, MERLIN
- **Multi-dimensional**: COPOD, PyODAdapter
- **Semi-supervised**: ROCKAD, OneClassSVM
- **No training data**: IsolationForest, STOMP
## Best Practices
1. **Normalize data**: Many methods sensitive to scale
2. **Choose window size**: For matrix profile methods, window size critical
3. **Set threshold**: Use percentile-based or domain-specific thresholds
4. **Validate results**: Visualize detections to verify meaningfulness
5. **Handle seasonality**: Detrend/deseasonalize before detection
skills/aeon/references/classification.md
+144
View File
@@ -0,0 +1,144 @@
# Time Series Classification
Aeon provides 13 categories of time series classifiers with scikit-learn compatible APIs.
## Convolution-Based Classifiers
Apply random convolutional transformations for efficient feature extraction:
- `Arsenal` - Ensemble of ROCKET classifiers with varied kernels
- `HydraClassifier` - Multi-resolution convolution with dilation
- `RocketClassifier` - Random convolution kernels with ridge regression
- `MiniRocketClassifier` - Simplified ROCKET variant for speed
- `MultiRocketClassifier` - Combines multiple ROCKET variants
**Use when**: Need fast, scalable classification with strong performance across diverse datasets.
## Deep Learning Classifiers
Neural network architectures optimized for temporal sequences:
- `FCNClassifier` - Fully convolutional network
- `ResNetClassifier` - Residual networks with skip connections
- `InceptionTimeClassifier` - Multi-scale inception modules
- `TimeCNNClassifier` - Standard CNN for time series
- `MLPClassifier` - Multi-layer perceptron baseline
- `EncoderClassifier` - Generic encoder wrapper
- `DisjointCNNClassifier` - Shapelet-focused architecture
**Use when**: Large datasets available, need end-to-end learning, or complex temporal patterns.
## Dictionary-Based Classifiers
Transform time series into symbolic representations:
- `BOSSEnsemble` - Bag-of-SFA-Symbols with ensemble voting
- `TemporalDictionaryEnsemble` - Multiple dictionary methods combined
- `WEASEL` - Word ExtrAction for time SEries cLassification
- `MrSEQLClassifier` - Multiple symbolic sequence learning
**Use when**: Need interpretable models, sparse patterns, or symbolic reasoning.
## Distance-Based Classifiers
Leverage specialized time series distance metrics:
- `KNeighborsTimeSeriesClassifier` - k-NN with temporal distances (DTW, LCSS, ERP, etc.)
- `ElasticEnsemble` - Combines multiple elastic distance measures
- `ProximityForest` - Tree ensemble using distance-based splits
**Use when**: Small datasets, need similarity-based classification, or interpretable decisions.
## Feature-Based Classifiers
Extract statistical and signature features before classification:
- `Catch22Classifier` - 22 canonical time-series characteristics
- `TSFreshClassifier` - Automated feature extraction via tsfresh
- `SignatureClassifier` - Path signature transformations
- `SummaryClassifier` - Summary statistics extraction
- `FreshPRINCEClassifier` - Combines multiple feature extractors
**Use when**: Need interpretable features, domain expertise available, or feature engineering approach.
## Interval-Based Classifiers
Extract features from random or supervised intervals:
- `CanonicalIntervalForestClassifier` - Random interval features with decision trees
- `DrCIFClassifier` - Diverse Representation CIF with catch22 features
- `TimeSeriesForestClassifier` - Random intervals with summary statistics
- `RandomIntervalClassifier` - Simple interval-based approach
- `RandomIntervalSpectralEnsembleClassifier` - Spectral features from intervals
- `SupervisedTimeSeriesForest` - Supervised interval selection
**Use when**: Discriminative patterns occur in specific time windows.
## Shapelet-Based Classifiers
Identify discriminative subsequences (shapelets):
- `ShapeletTransformClassifier` - Discovers and uses discriminative shapelets
- `LearningShapeletClassifier` - Learns shapelets via gradient descent
- `SASTClassifier` - Scalable approximate shapelet transform
- `RDSTClassifier` - Random dilated shapelet transform
**Use when**: Need interpretable discriminative patterns or phase-invariant features.
## Hybrid Classifiers
Combine multiple classification paradigms:
- `HIVECOTEV1` - Hierarchical Vote Collective of Transformation-based Ensembles (version 1)
- `HIVECOTEV2` - Enhanced version with updated components
**Use when**: Maximum accuracy required, computational resources available.
## Early Classification
Make predictions before observing entire time series:
- `TEASER` - Two-tier Early and Accurate Series Classifier
- `ProbabilityThresholdEarlyClassifier` - Prediction when confidence exceeds threshold
**Use when**: Real-time decisions needed, or observations have cost.
## Ordinal Classification
Handle ordered class labels:
- `OrdinalTDE` - Temporal dictionary ensemble for ordinal outputs
**Use when**: Classes have natural ordering (e.g., severity levels).
## Composition Tools
Build custom pipelines and ensembles:
- `ClassifierPipeline` - Chain transformers with classifiers
- `WeightedEnsembleClassifier` - Weighted combination of classifiers
- `SklearnClassifierWrapper` - Adapt sklearn classifiers for time series
## Quick Start
```python
from aeon.classification.convolution_based import RocketClassifier
from aeon.datasets import load_classification
# Load data
X_train, y_train = load_classification("GunPoint", split="train")
X_test, y_test = load_classification("GunPoint", split="test")
# Train and predict
clf = RocketClassifier()
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
```
## Algorithm Selection
- **Speed priority**: MiniRocketClassifier, Arsenal
- **Accuracy priority**: HIVECOTEV2, InceptionTimeClassifier
- **Interpretability**: ShapeletTransformClassifier, Catch22Classifier
- **Small data**: KNeighborsTimeSeriesClassifier, Distance-based methods
- **Large data**: Deep learning classifiers, ROCKET variants
skills/aeon/references/clustering.md
+123
View File
@@ -0,0 +1,123 @@
# Time Series Clustering
Aeon provides clustering algorithms adapted for temporal data with specialized distance metrics and averaging methods.
## Partitioning Algorithms
Standard k-means/k-medoids adapted for time series:
- `TimeSeriesKMeans` - K-means with temporal distance metrics (DTW, Euclidean, etc.)
- `TimeSeriesKMedoids` - Uses actual time series as cluster centers
- `TimeSeriesKShape` - Shape-based clustering algorithm
- `TimeSeriesKernelKMeans` - Kernel-based variant for nonlinear patterns
**Use when**: Known number of clusters, spherical cluster shapes expected.
## Large Dataset Methods
Efficient clustering for large collections:
- `TimeSeriesCLARA` - Clustering Large Applications with sampling
- `TimeSeriesCLARANS` - Randomized search variant of CLARA
**Use when**: Dataset too large for standard k-medoids, need scalability.
## Elastic Distance Clustering
Specialized for alignment-based similarity:
- `KASBA` - K-means with shift-invariant elastic averaging
- `ElasticSOM` - Self-organizing map using elastic distances
**Use when**: Time series have temporal shifts or warping.
## Spectral Methods
Graph-based clustering:
- `KSpectralCentroid` - Spectral clustering with centroid computation
**Use when**: Non-convex cluster shapes, need graph-based approach.
## Deep Learning Clustering
Neural network-based clustering with auto-encoders:
- `AEFCNClusterer` - Fully convolutional auto-encoder
- `AEResNetClusterer` - Residual network auto-encoder
- `AEDCNNClusterer` - Dilated CNN auto-encoder
- `AEDRNNClusterer` - Dilated RNN auto-encoder
- `AEBiGRUClusterer` - Bidirectional GRU auto-encoder
- `AEAttentionBiGRUClusterer` - Attention-enhanced BiGRU auto-encoder
**Use when**: Large datasets, need learned representations, or complex patterns.
## Feature-Based Clustering
Transform to feature space before clustering:
- `Catch22Clusterer` - Clusters on 22 canonical features
- `SummaryClusterer` - Uses summary statistics
- `TSFreshClusterer` - Automated tsfresh features
**Use when**: Raw time series not informative, need interpretable features.
## Composition
Build custom clustering pipelines:
- `ClustererPipeline` - Chain transformers with clusterers
## Averaging Methods
Compute cluster centers for time series:
- `mean_average` - Arithmetic mean
- `ba_average` - Barycentric averaging with DTW
- `kasba_average` - Shift-invariant averaging
- `shift_invariant_average` - General shift-invariant method
**Use when**: Need representative cluster centers for visualization or initialization.
## Quick Start
```python
from aeon.clustering import TimeSeriesKMeans
from aeon.datasets import load_classification
# Load data (using classification data for clustering)
X_train, _ = load_classification("GunPoint", split="train")
# Cluster time series
clusterer = TimeSeriesKMeans(
n_clusters=3,
distance="dtw", # Use DTW distance
averaging_method="ba" # Barycentric averaging
)
labels = clusterer.fit_predict(X_train)
centers = clusterer.cluster_centers_
```
## Algorithm Selection
- **Speed priority**: TimeSeriesKMeans with Euclidean distance
- **Temporal alignment**: KASBA, TimeSeriesKMeans with DTW
- **Large datasets**: TimeSeriesCLARA, TimeSeriesCLARANS
- **Complex patterns**: Deep learning clusterers
- **Interpretability**: Catch22Clusterer, SummaryClusterer
- **Non-convex clusters**: KSpectralCentroid
## Distance Metrics
Compatible distance metrics include:
- Euclidean, Manhattan, Minkowski (lock-step)
- DTW, DDTW, WDTW (elastic with alignment)
- ERP, EDR, LCSS (edit-based)
- MSM, TWE (specialized elastic)
## Evaluation
Use clustering metrics from sklearn or aeon benchmarking:
- Silhouette score
- Davies-Bouldin index
- Calinski-Harabasz index
skills/aeon/references/datasets_benchmarking.md
+392
View File
@@ -0,0 +1,392 @@
# Datasets and Benchmarking
Aeon provides comprehensive tools for loading datasets and benchmarking time series algorithms.
From **aeon 1.4** onward, most classification and regression archives are hosted on **Zenodo** (including the relaunched Multiverse multivariate classification archive). Loaders download on first use; cache location follows aeon defaults.
## Dataset Loading
### Task-Specific Loaders
**Classification Datasets**:
```python
from aeon.datasets import load_classification
# Load train/test split (or use load_gunpoint for this benchmark)
from aeon.datasets import load_gunpoint
X_train, y_train = load_classification("GunPoint", split="train")
X_test, y_test = load_classification("GunPoint", split="test")
# X_train, y_train = load_gunpoint(split="train")
# Load entire dataset
X, y = load_classification("GunPoint")
```
**Regression Datasets**:
```python
from aeon.datasets import load_regression
X_train, y_train = load_regression("Covid3Month", split="train")
X_test, y_test = load_regression("Covid3Month", split="test")
# Bulk download
from aeon.datasets import download_all_regression
download_all_regression() # Downloads Monash TSER archive
```
**Forecasting Datasets**:
```python
from aeon.datasets import load_forecasting
# Load from forecastingdata.org
y, X = load_forecasting("airline", return_X_y=True)
```
**Anomaly Detection Datasets**:
```python
from aeon.datasets import load_anomaly_detection
X, y = load_anomaly_detection("NAB_realKnownCause")
```
### File Format Loaders
**Load from .ts files**:
```python
from aeon.datasets import load_from_ts_file
X, y = load_from_ts_file("path/to/data.ts")
```
**Load from .tsf files**:
```python
from aeon.datasets import load_from_tsf_file
df, metadata = load_from_tsf_file("path/to/data.tsf")
```
**Load from ARFF files**:
```python
from aeon.datasets import load_from_arff_file
X, y = load_from_arff_file("path/to/data.arff")
```
**Load from TSV files**:
```python
from aeon.datasets import load_from_tsv_file
data = load_from_tsv_file("path/to/data.tsv")
```
**Load TimeEval CSV**:
```python
from aeon.datasets import load_from_timeeval_csv_file
X, y = load_from_timeeval_csv_file("path/to/timeeval.csv")
```
### Writing Datasets
**Write to .ts format**:
```python
from aeon.datasets import write_to_ts_file
write_to_ts_file(X, "output.ts", y=y, problem_name="MyDataset")
```
**Write to ARFF format**:
```python
from aeon.datasets import write_to_arff_file
write_to_arff_file(X, "output.arff", y=y)
```
## Built-in Datasets
Aeon includes several benchmark datasets for quick testing:
### Classification
- `ArrowHead` - Shape classification
- `GunPoint` - Gesture recognition
- `ItalyPowerDemand` - Energy demand
- `BasicMotions` - Motion classification
- And 100+ more from UCR/UEA archives
### Regression
- `Covid3Month` - COVID forecasting
- Various datasets from Monash TSER archive
### Segmentation
- Time series segmentation datasets
- Human activity data
- Sensor data collections
### Special Collections
- `RehabPile` - Rehabilitation data (classification & regression)
## Dataset Metadata
Get information about datasets:
```python
from aeon.datasets import get_dataset_meta_data
metadata = get_dataset_meta_data("GunPoint")
print(metadata)
# {'n_train': 50, 'n_test': 150, 'length': 150, 'n_classes': 2, ...}
```
## Benchmarking Tools
### Loading Published Results
Access pre-computed benchmark results:
```python
from aeon.benchmarking import get_estimator_results
# Get results for specific algorithm on dataset
results = get_estimator_results(
estimator_name="ROCKET",
dataset_name="GunPoint"
)
# Get all available estimators for a dataset
estimators = get_available_estimators("GunPoint")
```
### Resampling Strategies
Create reproducible train/test splits:
```python
from aeon.benchmarking import stratified_resample
# Stratified resampling maintaining class distribution
X_train, X_test, y_train, y_test = stratified_resample(
X, y,
random_state=42,
test_size=0.3
)
```
### Performance Metrics
Specialized metrics for time series tasks:
**Anomaly Detection Metrics**:
```python
from aeon.benchmarking.metrics.anomaly_detection import (
range_precision,
range_recall,
range_f_score,
range_roc_auc_score
)
# Range-based metrics for window detection
precision = range_precision(y_true, y_pred, alpha=0.5)
recall = range_recall(y_true, y_pred, alpha=0.5)
f1 = range_f_score(y_true, y_pred, alpha=0.5)
auc = range_roc_auc_score(y_true, y_scores)
```
**Clustering Metrics**:
```python
from aeon.benchmarking.metrics.clustering import clustering_accuracy
# Clustering accuracy with label matching
accuracy = clustering_accuracy(y_true, y_pred)
```
**Segmentation Metrics**:
```python
from aeon.benchmarking.metrics.segmentation import (
count_error,
hausdorff_error
)
# Number of change points difference
count_err = count_error(y_true, y_pred)
# Maximum distance between predicted and true change points
hausdorff_err = hausdorff_error(y_true, y_pred)
```
### Statistical Testing
Post-hoc analysis for algorithm comparison:
```python
from aeon.benchmarking import (
nemenyi_test,
wilcoxon_test
)
# Nemenyi test for multiple algorithms
results = nemenyi_test(scores_matrix, alpha=0.05)
# Pairwise Wilcoxon signed-rank test
stat, p_value = wilcoxon_test(scores_alg1, scores_alg2)
```
## Benchmark Collections
### UCR/UEA Time Series Archives
Access to comprehensive benchmark repositories:
```python
# Classification: 112 univariate + 30 multivariate datasets
X_train, y_train = load_classification("Chinatown", split="train")
# Automatically downloads from timeseriesclassification.com
```
### Monash Forecasting Archive
```python
# Load forecasting datasets
y = load_forecasting("nn5_daily", return_X_y=False)
```
### Published Benchmark Results
Pre-computed results from major competitions:
- 2017 Univariate Bake-off
- 2021 Multivariate Classification
- 2023 Univariate Bake-off
## Workflow Example
Complete benchmarking workflow:
```python
from aeon.datasets import load_classification
from aeon.classification.convolution_based import RocketClassifier
from aeon.benchmarking import get_estimator_results
from sklearn.metrics import accuracy_score
import numpy as np
# Load dataset
dataset_name = "GunPoint"
X_train, y_train = load_classification(dataset_name, split="train")
X_test, y_test = load_classification(dataset_name, split="test")
# Train model
clf = RocketClassifier(n_kernels=10000, random_state=42)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
# Evaluate
accuracy = accuracy_score(y_test, y_pred)
print(f"Accuracy: {accuracy:.4f}")
# Compare with published results
published = get_estimator_results("ROCKET", dataset_name)
print(f"Published ROCKET accuracy: {published['accuracy']:.4f}")
```
## Best Practices
### 1. Use Standard Splits
For reproducibility, use provided train/test splits:
```python
# Good: Use standard splits
X_train, y_train = load_classification("GunPoint", split="train")
X_test, y_test = load_classification("GunPoint", split="test")
# Avoid: Creating custom splits
X, y = load_classification("GunPoint")
X_train, X_test, y_train, y_test = train_test_split(X, y)
```
### 2. Set Random Seeds
Ensure reproducibility:
```python
clf = RocketClassifier(random_state=42)
results = stratified_resample(X, y, random_state=42)
```
### 3. Report Multiple Metrics
Don't rely on single metric:
```python
from sklearn.metrics import accuracy_score, f1_score, precision_score
accuracy = accuracy_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred, average='weighted')
precision = precision_score(y_test, y_pred, average='weighted')
```
### 4. Cross-Validation
For robust evaluation on small datasets:
```python
from sklearn.model_selection import cross_val_score
scores = cross_val_score(
clf, X_train, y_train,
cv=5,
scoring='accuracy'
)
print(f"CV Accuracy: {scores.mean():.4f} (+/- {scores.std():.4f})")
```
### 5. Compare Against Baselines
Always compare with simple baselines:
```python
from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier
# Simple baseline: 1-NN with Euclidean distance
baseline = KNeighborsTimeSeriesClassifier(n_neighbors=1, distance="euclidean")
baseline.fit(X_train, y_train)
baseline_acc = baseline.score(X_test, y_test)
print(f"Baseline: {baseline_acc:.4f}")
print(f"Your model: {accuracy:.4f}")
```
### 6. Statistical Significance
Test if improvements are statistically significant:
```python
from aeon.benchmarking import wilcoxon_test
# Run on multiple datasets
accuracies_alg1 = [0.85, 0.92, 0.78, 0.88]
accuracies_alg2 = [0.83, 0.90, 0.76, 0.86]
stat, p_value = wilcoxon_test(accuracies_alg1, accuracies_alg2)
if p_value < 0.05:
print("Difference is statistically significant")
```
## Dataset Discovery
Find datasets matching criteria:
```python
# List all available classification datasets
from aeon.datasets import get_available_datasets
datasets = get_available_datasets("classification")
print(f"Found {len(datasets)} classification datasets")
# Filter by properties
univariate_datasets = [
d for d in datasets
if get_dataset_meta_data(d)['n_channels'] == 1
]
```
skills/aeon/references/distances.md
+256
View File
@@ -0,0 +1,256 @@
# Distance Metrics
Aeon provides specialized distance functions for measuring similarity between time series, compatible with both aeon and scikit-learn estimators.
## Distance Categories
### Elastic Distances
Allow flexible temporal alignment between series:
**Dynamic Time Warping Family:**
- `dtw` - Classic Dynamic Time Warping
- `ddtw` - Derivative DTW (compares derivatives)
- `wdtw` - Weighted DTW (penalizes warping by location)
- `wddtw` - Weighted Derivative DTW
- `shape_dtw` - Shape-based DTW
**Edit-Based:**
- `erp` - Edit distance with Real Penalty
- `edr` - Edit Distance on Real sequences
- `lcss` - Longest Common SubSequence
- `twe` - Time Warp Edit distance
**Specialized:**
- `msm` - Move-Split-Merge distance
- `adtw` - Amerced DTW
- `sbd` - Shape-Based Distance
**Use when**: Time series may have temporal shifts, speed variations, or phase differences.
### Lock-Step Distances
Compare time series point-by-point without alignment:
- `euclidean` - Euclidean distance (L2 norm)
- `manhattan` - Manhattan distance (L1 norm)
- `minkowski` - Generalized Minkowski distance (Lp norm)
- `squared` - Squared Euclidean distance
**Use when**: Series already aligned, need computational speed, or no temporal warping expected.
## Usage Patterns
### Computing Single Distance
```python
from aeon.distances import dtw_distance
# Distance between two time series
distance = dtw_distance(x, y)
# With window constraint (Sakoe-Chiba band)
distance = dtw_distance(x, y, window=0.1)
```
### Pairwise Distance Matrix
```python
from aeon.distances import dtw_pairwise_distance
# All pairwise distances in collection
X = [series1, series2, series3, series4]
distance_matrix = dtw_pairwise_distance(X)
# Cross-collection distances
distance_matrix = dtw_pairwise_distance(X_train, X_test)
```
### Cost Matrix and Alignment Path
```python
from aeon.distances import dtw_cost_matrix, dtw_alignment_path
# Get full cost matrix
cost_matrix = dtw_cost_matrix(x, y)
# Get optimal alignment path
path = dtw_alignment_path(x, y)
# Returns indices: [(0,0), (1,1), (2,1), (2,2), ...]
```
### Using with Estimators
```python
from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier
# Use DTW distance in classifier
clf = KNeighborsTimeSeriesClassifier(
n_neighbors=5,
distance="dtw",
distance_params={"window": 0.2}
)
clf.fit(X_train, y_train)
```
## Distance Parameters
### Window Constraints
Limit warping path deviation (improves speed and prevents pathological warping):
```python
# Sakoe-Chiba band: window as fraction of series length
dtw_distance(x, y, window=0.1) # Allow 10% deviation
# Itakura parallelogram: slopes constrain path
dtw_distance(x, y, itakura_max_slope=2.0)
```
### Normalization
Control whether to z-normalize series before distance computation:
```python
# Most elastic distances support normalization
distance = dtw_distance(x, y, normalize=True)
```
### Distance-Specific Parameters
```python
# ERP: penalty for gaps
distance = erp_distance(x, y, g=0.5)
# TWE: stiffness and penalty parameters
distance = twe_distance(x, y, nu=0.001, lmbda=1.0)
# LCSS: epsilon threshold for matching
distance = lcss_distance(x, y, epsilon=0.5)
```
## Algorithm Selection
### By Use Case:
**Temporal misalignment**: DTW, DDTW, WDTW
**Speed variations**: DTW with window constraint
**Shape similarity**: Shape DTW, SBD
**Edit operations**: ERP, EDR, LCSS
**Derivative matching**: DDTW
**Computational speed**: Euclidean, Manhattan
**Outlier robustness**: Manhattan, LCSS
### By Computational Cost:
**Fastest**: Euclidean (O(n))
**Fast**: Constrained DTW (O(nw) where w is window)
**Medium**: Full DTW (O(n²))
**Slower**: Complex elastic distances (ERP, TWE, MSM)
## Quick Reference Table
| Distance | Alignment | Speed | Robustness | Interpretability |
|----------|-----------|-------|------------|------------------|
| Euclidean | Lock-step | Very Fast | Low | High |
| DTW | Elastic | Medium | Medium | Medium |
| DDTW | Elastic | Medium | High | Medium |
| WDTW | Elastic | Medium | Medium | Medium |
| ERP | Edit-based | Slow | High | Low |
| LCSS | Edit-based | Slow | Very High | Low |
| Shape DTW | Elastic | Medium | Medium | High |
## Best Practices
### 1. Normalization
Most distances sensitive to scale; normalize when appropriate:
```python
from aeon.transformations.collection import Normalizer
normalizer = Normalizer()
X_normalized = normalizer.fit_transform(X)
```
### 2. Window Constraints
For DTW variants, use window constraints for speed and better generalization:
```python
# Start with 10-20% window
distance = dtw_distance(x, y, window=0.1)
```
### 3. Series Length
- Equal-length required: Most lock-step distances
- Unequal-length supported: Elastic distances (DTW, ERP, etc.)
### 4. Multivariate Series
Most distances support multivariate time series:
```python
# x.shape = (n_channels, n_timepoints)
distance = dtw_distance(x_multivariate, y_multivariate)
```
### 5. Performance Optimization
- Use numba-compiled implementations (default in aeon)
- Consider lock-step distances if alignment not needed
- Use windowed DTW instead of full DTW
- Precompute distance matrices for repeated use
### 6. Choosing the Right Distance
```python
# Quick decision tree:
if series_aligned:
use_distance = "euclidean"
elif need_speed:
use_distance = "dtw" # with window constraint
elif temporal_shifts_expected:
use_distance = "dtw" or "shape_dtw"
elif outliers_present:
use_distance = "lcss" or "manhattan"
elif derivatives_matter:
use_distance = "ddtw" or "wddtw"
```
## Integration with scikit-learn
Aeon distances work with sklearn estimators:
```python
from sklearn.neighbors import KNeighborsClassifier
from aeon.distances import dtw_pairwise_distance
# Precompute distance matrix
X_train_distances = dtw_pairwise_distance(X_train)
# Use with sklearn
clf = KNeighborsClassifier(metric='precomputed')
clf.fit(X_train_distances, y_train)
```
## Available Distance Functions
Get list of all available distances:
```python
from aeon.distances import get_distance_function_names
print(get_distance_function_names())
# ['dtw', 'ddtw', 'wdtw', 'euclidean', 'erp', 'edr', ...]
```
Retrieve specific distance function:
```python
from aeon.distances import get_distance_function
distance_func = get_distance_function("dtw")
result = distance_func(x, y, window=0.1)
```
skills/aeon/references/forecasting.md
+109
View File
@@ -0,0 +1,109 @@
# Time Series Forecasting
The `aeon.forecasting` module provides forecasters for univariate and multivariate series. In aeon **1.x**, forecasting was rebuilt on array-native `BaseForecaster` estimators (replacing the old sktime-style `fh` API). The module is marked **experimental** — expect API evolution between releases.
Import paths (aeon 1.4+):
- `from aeon.forecasting import NaiveForecaster, RegressionForecaster`
- `from aeon.forecasting.stats import ARIMA, AutoARIMA, ETS, AutoETS, Theta, TAR, AutoTAR, TVP`
- `from aeon.forecasting.deep_learning import TCNForecaster, DeepARForecaster`
List all forecasters: `aeon.utils.discovery.all_estimators(type_filter="forecaster")`.
## Naive and Baseline Methods
- `NaiveForecaster` — `strategy` in `"last"`, `"mean"`, `"seasonal_last"`; set `horizon` and `seasonal_period` in the constructor
- **Use when**: Establishing baselines or simple patterns
## Statistical Models
- `ARIMA` / `AutoARIMA` — `p`, `d`, `q` orders (not `order=(p,d,q)`); supports exogenous variables via `exog`
- `ETS` / `AutoETS` — exponential smoothing (native implementations in aeon 1.4+)
- `Theta` — classical Theta method
- `TAR` / `AutoTAR` — threshold autoregressive models for regime switching
- `TVP` — time-varying parameter (Kalman-style) models
## Deep Learning Forecasters
Requires `aeon[all_extras]` (PyTorch stack):
- `TCNForecaster` — temporal convolutional network
- `DeepARForecaster` — probabilistic RNN forecaster (replaces legacy `DeepARNetwork` naming)
## Regression-Based Forecasting
- `RegressionForecaster` — sliding `window` over history, `horizon` steps ahead, any sklearn/aeon regressor
## Quick Start
```python
import numpy as np
from aeon.forecasting import NaiveForecaster
from aeon.forecasting.stats import ARIMA, AutoETS
y = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])
# Naive — horizon is a constructor argument; predict(y) forecasts from series y
naive = NaiveForecaster(strategy="last", horizon=3)
naive.fit(y)
pred_naive = naive.predict(y)
# ARIMA — one-step by default; multi-step via iterative_forecast
arima = ARIMA(p=1, d=1, q=1)
arima.fit(y)
pred_arima = arima.iterative_forecast(y, prediction_horizon=3)
# Auto model selection
auto_ets = AutoETS(horizon=3)
auto_ets.fit(y)
pred_ets = auto_ets.predict(y)
```
## Forecasting Horizon
In aeon 1.x, set `horizon` on the estimator (number of steps ahead). `predict(y)` returns the forecast `horizon` steps beyond the end of `y`.
Multi-step strategies:
- **`iterative_forecast(y, prediction_horizon)`** — reuse one fitted model, feed predictions back (ARIMA, many stats models)
- **`direct_forecast(y, prediction_horizon)`** — refit per horizon (requires `capability:horizon` tag; e.g. `RegressionForecaster`)
- **`NaiveForecaster`** — set `horizon>1` directly when `strategy` supports it
There is no `ForecastingHorizon` / `fh=[1,2,3]` API in aeon 1.x.
## Model Selection
- **Baseline**: `NaiveForecaster(strategy="seasonal_last", seasonal_period=12, horizon=h)`
- **Linear / stationary**: `ARIMA`, `AutoARIMA`
- **Trend + seasonality**: `ETS`, `AutoETS`
- **Regime changes**: `TAR`, `AutoTAR`
- **Complex patterns**: `TCNForecaster`, `RegressionForecaster` with aeon regressors
- **Probabilistic**: `DeepARForecaster`
## Evaluation Metrics
Use scikit-learn or standard numpy metrics on hold-out forecasts:
```python
from sklearn.metrics import mean_absolute_error, mean_squared_error
mae = mean_absolute_error(y_true, y_pred)
mse = mean_squared_error(y_true, y_pred)
```
## Exogenous Variables
Pass aligned exogenous arrays as `exog` (not `X`):
```python
forecaster.fit(y_train, exog=exog_train)
y_pred = forecaster.predict(y_test, exog=exog_test)
```
## Base Classes
- `BaseForecaster` — `horizon`, `axis`, `fit`, `predict`, `forecast`
- `DirectForecastingMixin` / `IterativeForecastingMixin` — multi-step helpers
- `BaseDeepForecaster` — deep learning forecasters
Extend `BaseForecaster` for custom forecasters.
skills/aeon/references/networks.md
+289
View File
@@ -0,0 +1,289 @@
# Deep Learning Networks
Aeon provides neural network architectures specifically designed for time series tasks. These networks serve as building blocks for classification, regression, clustering, and forecasting.
## Core Network Architectures
### Convolutional Networks
**FCNNetwork** - Fully Convolutional Network
- Three convolutional blocks with batch normalization
- Global average pooling for dimensionality reduction
- **Use when**: Need simple yet effective CNN baseline
**ResNetNetwork** - Residual Network
- Residual blocks with skip connections
- Prevents vanishing gradients in deep networks
- **Use when**: Deep networks needed, training stability important
**InceptionNetwork** - Inception Modules
- Multi-scale feature extraction with parallel convolutions
- Different kernel sizes capture patterns at various scales
- **Use when**: Patterns exist at multiple temporal scales
**TimeCNNNetwork** - Standard CNN
- Basic convolutional architecture
- **Use when**: Simple CNN sufficient, interpretability valued
**DisjointCNNNetwork** - Separate Pathways
- Disjoint convolutional pathways
- **Use when**: Different feature extraction strategies needed
**DCNNNetwork** - Dilated CNN
- Dilated convolutions for large receptive fields
- **Use when**: Long-range dependencies without many layers
### Recurrent Networks
**RecurrentNetwork** - RNN/LSTM/GRU
- Configurable cell type (RNN, LSTM, GRU)
- Sequential modeling of temporal dependencies
- **Use when**: Sequential dependencies critical, variable-length series
### Temporal Convolutional Network
**TCNNetwork** - Temporal Convolutional Network
- Dilated causal convolutions
- Large receptive field without recurrence
- **Use when**: Long sequences, need parallelizable architecture
### Multi-Layer Perceptron
**MLPNetwork** - Basic Feedforward
- Simple fully-connected layers
- Flattens time series before processing
- **Use when**: Baseline needed, computational limits, or simple patterns
## Encoder-Based Architectures
Networks designed for representation learning and clustering.
### Autoencoder Variants
**EncoderNetwork** - Generic Encoder
- Flexible encoder structure
- **Use when**: Custom encoding needed
**AEFCNNetwork** - FCN-based Autoencoder
- Fully convolutional encoder-decoder
- **Use when**: Need convolutional representation learning
**AEResNetNetwork** - ResNet Autoencoder
- Residual blocks in encoder-decoder
- **Use when**: Deep autoencoding with skip connections
**AEDCNNNetwork** - Dilated CNN Autoencoder
- Dilated convolutions for compression
- **Use when**: Need large receptive field in autoencoder
**AEDRNNNetwork** - Dilated RNN Autoencoder
- Dilated recurrent connections
- **Use when**: Sequential patterns with long-range dependencies
**AEBiGRUNetwork** - Bidirectional GRU
- Bidirectional recurrent encoding
- **Use when**: Context from both directions helpful
**AEAttentionBiGRUNetwork** - Attention + BiGRU
- Attention mechanism on BiGRU outputs
- **Use when**: Need to focus on important time steps
## Specialized Architectures
**LITENetwork** - Lightweight Inception Time Ensemble
- Efficient inception-based architecture
- LITEMV variant for multivariate series
- **Use when**: Need efficiency with strong performance
**DeepARForecaster** - Probabilistic forecasting (use via `aeon.forecasting.deep_learning`)
- Autoregressive RNN for forecasting
- Produces probabilistic predictions
- **Use when**: Need forecast uncertainty quantification
## Usage with Estimators
Networks are typically used within estimators, not directly:
```python
from aeon.classification.deep_learning import FCNClassifier
from aeon.regression.deep_learning import ResNetRegressor
from aeon.clustering.deep_learning import AEFCNClusterer
# Classification with FCN
clf = FCNClassifier(n_epochs=100, batch_size=16)
clf.fit(X_train, y_train)
# Regression with ResNet
reg = ResNetRegressor(n_epochs=100)
reg.fit(X_train, y_train)
# Clustering with autoencoder
clusterer = AEFCNClusterer(n_clusters=3, n_epochs=100)
labels = clusterer.fit_predict(X_train)
```
## Custom Network Configuration
Many networks accept configuration parameters:
```python
# Configure FCN layers
clf = FCNClassifier(
n_epochs=200,
batch_size=32,
kernel_size=[7, 5, 3], # Kernel sizes for each layer
n_filters=[128, 256, 128], # Filters per layer
learning_rate=0.001
)
```
## Base Classes
- `BaseDeepLearningNetwork` - Abstract base for all networks
- `BaseDeepRegressor` - Base for deep regression
- `BaseDeepClassifier` - Base for deep classification
- `BaseDeepForecaster` - Base for deep forecasting
Extend these to implement custom architectures.
## Training Considerations
### Hyperparameters
Key hyperparameters to tune:
- `n_epochs` - Training iterations (50-200 typical)
- `batch_size` - Samples per batch (16-64 typical)
- `learning_rate` - Step size (0.0001-0.01)
- Network-specific: layers, filters, kernel sizes
### Callbacks
Many networks support callbacks for training monitoring:
```python
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
clf = FCNClassifier(
n_epochs=200,
callbacks=[
EarlyStopping(patience=20, restore_best_weights=True),
ReduceLROnPlateau(patience=10, factor=0.5)
]
)
```
### GPU Acceleration
Deep learning networks benefit from GPU:
```python
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0' # Use first GPU
# Networks automatically use GPU if available
clf = InceptionTimeClassifier(n_epochs=100)
clf.fit(X_train, y_train)
```
## Architecture Selection
### By Task:
**Classification**: InceptionNetwork, ResNetNetwork, FCNNetwork
**Regression**: InceptionNetwork, ResNetNetwork, TCNNetwork
**Forecasting**: TCNForecaster, DeepARForecaster, RecurrentNetwork
**Clustering**: AEFCNNetwork, AEResNetNetwork, AEAttentionBiGRUNetwork
### By Data Characteristics:
**Long sequences**: TCNNetwork, DCNNNetwork (dilated convolutions)
**Short sequences**: MLPNetwork, FCNNetwork
**Multivariate**: InceptionNetwork, FCNNetwork, LITENetwork
**Variable length**: RecurrentNetwork with masking
**Multi-scale patterns**: InceptionNetwork
### By Computational Resources:
**Limited compute**: MLPNetwork, LITENetwork
**Moderate compute**: FCNNetwork, TimeCNNNetwork
**High compute available**: InceptionNetwork, ResNetNetwork
**GPU available**: Any deep network (major speedup)
## Best Practices
### 1. Data Preparation
Normalize input data:
```python
from aeon.transformations.collection import Normalizer
normalizer = Normalizer()
X_train_norm = normalizer.fit_transform(X_train)
X_test_norm = normalizer.transform(X_test)
```
### 2. Training/Validation Split
Use validation set for early stopping:
```python
from sklearn.model_selection import train_test_split
X_train_fit, X_val, y_train_fit, y_val = train_test_split(
X_train, y_train, test_size=0.2, stratify=y_train
)
clf = FCNClassifier(n_epochs=200)
clf.fit(X_train_fit, y_train_fit, validation_data=(X_val, y_val))
```
### 3. Start Simple
Begin with simpler architectures before complex ones:
1. Try MLPNetwork or FCNNetwork first
2. If insufficient, try ResNetNetwork or InceptionNetwork
3. Consider ensembles if single models insufficient
### 4. Hyperparameter Tuning
Use grid search or random search:
```python
from sklearn.model_selection import GridSearchCV
param_grid = {
'n_epochs': [100, 200],
'batch_size': [16, 32],
'learning_rate': [0.001, 0.0001]
}
clf = FCNClassifier()
grid = GridSearchCV(clf, param_grid, cv=3)
grid.fit(X_train, y_train)
```
### 5. Regularization
Prevent overfitting:
- Use dropout (if network supports)
- Early stopping
- Data augmentation (if available)
- Reduce model complexity
### 6. Reproducibility
Set random seeds:
```python
import numpy as np
import random
import tensorflow as tf
seed = 42
np.random.seed(seed)
random.seed(seed)
tf.random.set_seed(seed)
```
skills/aeon/references/regression.md
+118
View File
@@ -0,0 +1,118 @@
# Time Series Regression
Aeon provides time series regressors across 9 categories for predicting continuous values from temporal sequences.
## Convolution-Based Regressors
Apply convolutional kernels for feature extraction:
- `HydraRegressor` - Multi-resolution dilated convolutions
- `RocketRegressor` - Random convolutional kernels
- `MiniRocketRegressor` - Simplified ROCKET for speed
- `MultiRocketRegressor` - Combined ROCKET variants
- `MultiRocketHydraRegressor` - Merges ROCKET and Hydra approaches
**Use when**: Need fast regression with strong baseline performance.
## Deep Learning Regressors
Neural architectures for end-to-end temporal regression:
- `FCNRegressor` - Fully convolutional network
- `ResNetRegressor` - Residual blocks with skip connections
- `InceptionTimeRegressor` - Multi-scale inception modules
- `TimeCNNRegressor` - Standard CNN architecture
- `RecurrentRegressor` - RNN/LSTM/GRU variants
- `MLPRegressor` - Multi-layer perceptron
- `EncoderRegressor` - Generic encoder wrapper
- `LITERegressor` - Lightweight inception time ensemble
- `DisjointCNNRegressor` - Specialized CNN architecture
**Use when**: Large datasets, complex patterns, or need feature learning.
## Distance-Based Regressors
k-nearest neighbors with temporal distance metrics:
- `KNeighborsTimeSeriesRegressor` - k-NN with DTW, LCSS, ERP, or other distances
**Use when**: Small datasets, local similarity patterns, or interpretable predictions.
## Feature-Based Regressors
Extract statistical features before regression:
- `Catch22Regressor` - 22 canonical time-series characteristics
- `FreshPRINCERegressor` - Pipeline combining multiple feature extractors
- `SummaryRegressor` - Summary statistics features
- `TSFreshRegressor` - Automated tsfresh feature extraction
**Use when**: Need interpretable features or domain-specific feature engineering.
## Hybrid Regressors
Combine multiple approaches:
- `RISTRegressor` - Randomized Interval-Shapelet Transformation
**Use when**: Benefit from combining interval and shapelet methods.
## Interval-Based Regressors
Extract features from time intervals:
- `CanonicalIntervalForestRegressor` - Random intervals with decision trees
- `DrCIFRegressor` - Diverse Representation CIF
- `TimeSeriesForestRegressor` - Random interval ensemble
- `RandomIntervalRegressor` - Simple interval-based approach
- `RandomIntervalSpectralEnsembleRegressor` - Spectral interval features
- `QUANTRegressor` - Quantile-based interval features
**Use when**: Predictive patterns occur in specific time windows.
## Shapelet-Based Regressors
Use discriminative subsequences for prediction:
- `RDSTRegressor` - Random Dilated Shapelet Transform
**Use when**: Need phase-invariant discriminative patterns.
## Composition Tools
Build custom regression pipelines:
- `RegressorPipeline` - Chain transformers with regressors
- `RegressorEnsemble` - Weighted ensemble with learnable weights
- `SklearnRegressorWrapper` - Adapt sklearn regressors for time series
## Utilities
- `DummyRegressor` - Baseline strategies (mean, median)
- `BaseRegressor` - Abstract base for custom regressors
- `BaseDeepRegressor` - Base for deep learning regressors
## Quick Start
```python
from aeon.regression.convolution_based import RocketRegressor
from aeon.datasets import load_regression
# Load data
X_train, y_train = load_regression("Covid3Month", split="train")
X_test, y_test = load_regression("Covid3Month", split="test")
# Train and predict
reg = RocketRegressor()
reg.fit(X_train, y_train)
predictions = reg.predict(X_test)
```
## Algorithm Selection
- **Speed priority**: MiniRocketRegressor
- **Accuracy priority**: InceptionTimeRegressor, MultiRocketHydraRegressor
- **Interpretability**: Catch22Regressor, SummaryRegressor
- **Small data**: KNeighborsTimeSeriesRegressor
- **Large data**: Deep learning regressors, ROCKET variants
- **Interval patterns**: DrCIFRegressor, CanonicalIntervalForestRegressor
skills/aeon/references/segmentation.md
+163
View File
@@ -0,0 +1,163 @@
# Time Series Segmentation
Aeon provides algorithms to partition time series into regions with distinct characteristics, identifying change points and boundaries.
## Segmentation Algorithms
### Binary Segmentation
- `BinSegmenter` - Recursive binary segmentation
- Iteratively splits series at most significant change points
- Parameters: `n_segments`, `cost_function`
- **Use when**: Known number of segments, hierarchical structure
### Classification-Based
- `ClaSPSegmenter` - Classification Score Profile
- Uses classification performance to identify boundaries
- Discovers segments where classification distinguishes neighbors
- **Use when**: Segments have different temporal patterns
### Fast Pattern-Based
- `FLUSSSegmenter` - Fast Low-cost Unipotent Semantic Segmentation
- Efficient semantic segmentation using arc crossings
- Based on matrix profile
- **Use when**: Large time series, need speed and pattern discovery
### Information Theory
- `InformationGainSegmenter` - Information gain maximization
- Finds boundaries maximizing information gain
- **Use when**: Statistical differences between segments
### Gaussian Modeling
- `GreedyGaussianSegmenter` - Greedy Gaussian approximation
- Models segments as Gaussian distributions
- Incrementally adds change points
- **Use when**: Segments follow Gaussian distributions
### Hierarchical Agglomerative
- `EAggloSegmenter` - Bottom-up merging approach
- Estimates change points via agglomeration
- **Use when**: Want hierarchical segmentation structure
### Hidden Markov Models
- `HMMSegmenter` - HMM with Viterbi decoding
- Probabilistic state-based segmentation
- **Use when**: Segments represent hidden states
### Dimensionality-Based
- `HidalgoSegmenter` - Heterogeneous Intrinsic Dimensionality Algorithm
- Detects changes in local dimensionality
- **Use when**: Dimensionality shifts between segments
### Baseline
- `RandomSegmenter` - Random change point generation
- **Use when**: Need null hypothesis baseline
## Quick Start
```python
from aeon.segmentation import ClaSPSegmenter
import numpy as np
# Create time series with regime changes
y = np.concatenate([
np.sin(np.linspace(0, 10, 100)), # Segment 1
np.cos(np.linspace(0, 10, 100)), # Segment 2
np.sin(2 * np.linspace(0, 10, 100)) # Segment 3
])
# Segment the series
segmenter = ClaSPSegmenter()
change_points = segmenter.fit_predict(y)
print(f"Detected change points: {change_points}")
```
## Output Format
Segmenters return change point indices:
```python
# change_points = [100, 200] # Boundaries between segments
# This divides series into: [0:100], [100:200], [200:end]
```
## Algorithm Selection
- **Speed priority**: FLUSSSegmenter, BinSegmenter
- **Accuracy priority**: ClaSPSegmenter, HMMSegmenter
- **Known segment count**: BinSegmenter with n_segments parameter
- **Unknown segment count**: ClaSPSegmenter, InformationGainSegmenter
- **Pattern changes**: FLUSSSegmenter, ClaSPSegmenter
- **Statistical changes**: InformationGainSegmenter, GreedyGaussianSegmenter
- **State transitions**: HMMSegmenter
## Common Use Cases
### Regime Change Detection
Identify when time series behavior fundamentally changes:
```python
from aeon.segmentation import InformationGainSegmenter
segmenter = InformationGainSegmenter(k=3) # Up to 3 change points
change_points = segmenter.fit_predict(stock_prices)
```
### Activity Segmentation
Segment sensor data into activities:
```python
from aeon.segmentation import ClaSPSegmenter
segmenter = ClaSPSegmenter()
boundaries = segmenter.fit_predict(accelerometer_data)
```
### Seasonal Boundary Detection
Find season transitions in time series:
```python
from aeon.segmentation import HMMSegmenter
segmenter = HMMSegmenter(n_states=4) # 4 seasons
segments = segmenter.fit_predict(temperature_data)
```
## Evaluation Metrics
Use segmentation quality metrics:
```python
from aeon.benchmarking.metrics.segmentation import (
count_error,
hausdorff_error
)
# Count error: difference in number of change points
count_err = count_error(y_true, y_pred)
# Hausdorff: maximum distance between predicted and true points
hausdorff_err = hausdorff_error(y_true, y_pred)
```
## Best Practices
1. **Normalize data**: Ensures change detection not dominated by scale
2. **Choose appropriate metric**: Different algorithms optimize different criteria
3. **Validate segments**: Visualize to verify meaningful boundaries
4. **Handle noise**: Consider smoothing before segmentation
5. **Domain knowledge**: Use expected segment count if known
6. **Parameter tuning**: Adjust sensitivity parameters (thresholds, penalties)
## Visualization
```python
import matplotlib.pyplot as plt
plt.figure(figsize=(12, 4))
plt.plot(y, label='Time Series')
for cp in change_points:
plt.axvline(cp, color='r', linestyle='--', label='Change Point')
plt.legend()
plt.show()
```
skills/aeon/references/similarity_search.md
+187
View File
@@ -0,0 +1,187 @@
# Similarity Search
Aeon provides tools for finding similar patterns within and across time series, including subsequence search, motif discovery, and approximate nearest neighbors.
## Subsequence Nearest Neighbors (SNN)
Find most similar subsequences within a time series.
### MASS Algorithm
- `MassSNN` - Mueen's Algorithm for Similarity Search
- Fast normalized cross-correlation for similarity
- Computes distance profile efficiently
- **Use when**: Need exact nearest neighbor distances, large series
### STOMP-Based Motif Discovery
- `StompMotif` - Discovers recurring patterns (motifs)
- Finds top-k most similar subsequence pairs
- Based on matrix profile computation
- **Use when**: Want to discover repeated patterns
### Brute Force Baseline
- `DummySNN` - Exhaustive distance computation
- Computes all pairwise distances
- **Use when**: Small series, need exact baseline
## Collection-Level Search
Find similar time series across collections.
### Approximate Nearest Neighbors (ANN)
- `RandomProjectionIndexANN` - Locality-sensitive hashing
- Uses random projections with cosine similarity
- Builds index for fast approximate search
- **Use when**: Large collection, speed more important than exactness
## Quick Start: Motif Discovery
```python
from aeon.similarity_search import StompMotif
import numpy as np
# Create time series with repeated patterns
pattern = np.sin(np.linspace(0, 2*np.pi, 50))
y = np.concatenate([
pattern + np.random.normal(0, 0.1, 50),
np.random.normal(0, 1, 100),
pattern + np.random.normal(0, 0.1, 50),
np.random.normal(0, 1, 100)
])
# Find top-3 motifs
motif_finder = StompMotif(window_size=50, k=3)
motifs = motif_finder.fit_predict(y)
# motifs contains indices of motif occurrences
for i, (idx1, idx2) in enumerate(motifs):
print(f"Motif {i+1} at positions {idx1} and {idx2}")
```
## Quick Start: Subsequence Search
```python
from aeon.similarity_search import MassSNN
import numpy as np
# Time series to search within
y = np.sin(np.linspace(0, 20, 500))
# Query subsequence
query = np.sin(np.linspace(0, 2, 50))
# Find nearest subsequences
searcher = MassSNN()
distances = searcher.fit_transform(y, query)
# Find best match
best_match_idx = np.argmin(distances)
print(f"Best match at index {best_match_idx}")
```
## Quick Start: Approximate NN on Collections
```python
from aeon.similarity_search import RandomProjectionIndexANN
from aeon.datasets import load_classification
# Load time series collection
X_train, _ = load_classification("GunPoint", split="train")
# Build index
ann = RandomProjectionIndexANN(n_projections=8, n_bits=4)
ann.fit(X_train)
# Find approximate nearest neighbors
query = X_train[0]
neighbors, distances = ann.kneighbors(query, k=5)
```
## Matrix Profile
The matrix profile is a fundamental data structure for many similarity search tasks:
- **Distance Profile**: Distances from a query to all subsequences
- **Matrix Profile**: Minimum distance for each subsequence to any other
- **Motif**: Pair of subsequences with minimum distance
- **Discord**: Subsequence with maximum minimum distance (anomaly)
```python
from aeon.similarity_search import StompMotif
# Compute matrix profile and find motifs/discords
mp = StompMotif(window_size=50)
mp.fit(y)
# Access matrix profile
profile = mp.matrix_profile_
profile_indices = mp.matrix_profile_index_
# Find discords (anomalies)
discord_idx = np.argmax(profile)
```
## Algorithm Selection
- **Exact subsequence search**: MassSNN
- **Motif discovery**: StompMotif
- **Anomaly detection**: Matrix profile (see anomaly_detection.md)
- **Fast approximate search**: RandomProjectionIndexANN
- **Small data**: DummySNN for exact results
## Use Cases
### Pattern Matching
Find where a pattern occurs in a long series:
```python
# Find heartbeat pattern in ECG data
searcher = MassSNN()
distances = searcher.fit_transform(ecg_data, heartbeat_pattern)
occurrences = np.where(distances < threshold)[0]
```
### Motif Discovery
Identify recurring patterns:
```python
# Find repeated behavioral patterns
motif_finder = StompMotif(window_size=100, k=5)
motifs = motif_finder.fit_predict(activity_data)
```
### Time Series Retrieval
Find similar time series in database:
```python
# Build searchable index
ann = RandomProjectionIndexANN()
ann.fit(time_series_database)
# Query for similar series
neighbors = ann.kneighbors(query_series, k=10)
```
## Best Practices
1. **Window size**: Critical parameter for subsequence methods
- Too small: Captures noise
- Too large: Misses fine-grained patterns
- Rule of thumb: 10-20% of series length
2. **Normalization**: Most methods assume z-normalized subsequences
- Handles amplitude variations
- Focus on shape similarity
3. **Distance metrics**: Different metrics for different needs
- Euclidean: Fast, shape-based
- DTW: Handles temporal warping
- Cosine: Scale-invariant
4. **Exclusion zone**: For motif discovery, exclude trivial matches
- Typically set to 0.5-1.0 × window_size
- Prevents finding overlapping occurrences
5. **Performance**:
- MASS is O(n log n) vs O(n²) brute force
- ANN trades accuracy for speed
- GPU acceleration available for some methods
skills/aeon/references/transformations.md
+246
View File
@@ -0,0 +1,246 @@
# Transformations
Aeon provides extensive transformation capabilities for preprocessing, feature extraction, and representation learning from time series data.
## Transformation Types
Aeon distinguishes between:
- **CollectionTransformers**: Transform multiple time series (collections)
- **SeriesTransformers**: Transform individual time series
## Collection Transformers
### Convolution-Based Feature Extraction
Fast, scalable feature generation using random kernels:
- `RocketTransformer` - Random convolutional kernels
- `MiniRocketTransformer` - Simplified ROCKET for speed
- `MultiRocketTransformer` - Enhanced ROCKET variant
- `HydraTransformer` - Multi-resolution dilated convolutions
- `MultiRocketHydraTransformer` - Combines ROCKET and Hydra
- `ROCKETGPU` - GPU-accelerated variant
**Use when**: Need fast, scalable features for any ML algorithm, strong baseline performance.
### Statistical Feature Extraction
Domain-agnostic features based on time series characteristics:
- `Catch22` - 22 canonical time-series characteristics
- `TSFresh` - Comprehensive automated feature extraction (100+ features)
- `TSFreshRelevant` - Feature extraction with relevance filtering
- `SevenNumberSummary` - Descriptive statistics (mean, std, quantiles)
**Use when**: Need interpretable features, domain-agnostic approach, or feeding traditional ML.
### Dictionary-Based Representations
Symbolic approximations for discrete representations:
- `SAX` - Symbolic Aggregate approXimation
- `PAA` - Piecewise Aggregate Approximation
- `SFA` - Symbolic Fourier Approximation
- `SFAFast` - Optimized SFA
- `SFAWhole` - SFA on entire series (no windowing)
- `BORF` - Bag-of-Receptive-Fields
**Use when**: Need discrete/symbolic representation, dimensionality reduction, interpretability.
### Shapelet-Based Features
Discriminative subsequence extraction:
- `RandomShapeletTransform` - Random discriminative shapelets
- `RandomDilatedShapeletTransform` - Dilated shapelets for multi-scale
- `SAST` - Scalable And Accurate Subsequence Transform
- `RSAST` - Randomized SAST
**Use when**: Need interpretable discriminative patterns, phase-invariant features.
### Interval-Based Features
Statistical summaries from time intervals:
- `RandomIntervals` - Features from random intervals
- `SupervisedIntervals` - Supervised interval selection
- `QUANTTransformer` - Quantile-based interval features
**Use when**: Predictive patterns localized to specific windows.
### Preprocessing Transformations
Data preparation and normalization:
- `MinMaxScaler` - Scale to [0, 1] range
- `Normalizer` - Z-normalization (zero mean, unit variance)
- `Centerer` - Center to zero mean
- `SimpleImputer` - Fill missing values
- `DownsampleTransformer` - Reduce temporal resolution
- `Tabularizer` - Convert time series to tabular format
**Use when**: Need standardization, missing value handling, format conversion.
### Specialized Transformations
Advanced analysis methods:
- `MatrixProfile` - Computes distance profiles for pattern discovery
- `DWTTransformer` - Discrete Wavelet Transform
- `AutocorrelationFunctionTransformer` - ACF computation
- `Dobin` - Distance-based Outlier BasIs using Neighbors
- `SignatureTransformer` - Path signature methods
- `PLATransformer` - Piecewise Linear Approximation
### Class Imbalance Handling
- `ADASYN` - Adaptive Synthetic Sampling
- `SMOTE` - Synthetic Minority Over-sampling
- `OHIT` - Over-sampling with Highly Imbalanced Time series
**Use when**: Classification with imbalanced classes.
### Pipeline Composition
- `CollectionTransformerPipeline` - Chain multiple transformers
## Series Transformers
Transform individual time series (e.g., for preprocessing in forecasting).
### Statistical Analysis
- `AutoCorrelationSeriesTransformer` - Autocorrelation
- `StatsModelsACF` - ACF using statsmodels
- `StatsModelsPACF` - Partial autocorrelation
### Smoothing and Filtering
- `ExponentialSmoothing` - Exponentially weighted moving average
- `MovingAverage` - Simple or weighted moving average
- `SavitzkyGolayFilter` - Polynomial smoothing
- `GaussianFilter` - Gaussian kernel smoothing
- `BKFilter` - Baxter-King bandpass filter
- `DiscreteFourierApproximation` - Fourier-based filtering
**Use when**: Need noise reduction, trend extraction, or frequency filtering.
### Dimensionality Reduction
- `PCASeriesTransformer` - Principal component analysis
- `PlASeriesTransformer` - Piecewise Linear Approximation
### Transformations
- `BoxCoxTransformer` - Variance stabilization
- `LogTransformer` - Logarithmic scaling
- `ClaSPTransformer` - Classification Score Profile
### Pipeline Composition
- `SeriesTransformerPipeline` - Chain series transformers
## Quick Start: Feature Extraction
```python
from aeon.transformations.collection.convolution_based import RocketTransformer
from aeon.classification.sklearn import RotationForest
from aeon.datasets import load_classification
# Load data
X_train, y_train = load_classification("GunPoint", split="train")
X_test, y_test = load_classification("GunPoint", split="test")
# Extract ROCKET features
rocket = RocketTransformer()
X_train_features = rocket.fit_transform(X_train)
X_test_features = rocket.transform(X_test)
# Use with any sklearn classifier
clf = RotationForest()
clf.fit(X_train_features, y_train)
accuracy = clf.score(X_test_features, y_test)
```
## Quick Start: Preprocessing Pipeline
```python
from aeon.transformations.collection import (
MinMaxScaler,
SimpleImputer,
CollectionTransformerPipeline
)
# Build preprocessing pipeline
pipeline = CollectionTransformerPipeline([
('imputer', SimpleImputer(strategy='mean')),
('scaler', MinMaxScaler())
])
X_transformed = pipeline.fit_transform(X_train)
```
## Quick Start: Series Smoothing
```python
from aeon.transformations.series import MovingAverage
# Smooth individual time series
smoother = MovingAverage(window_size=5)
y_smoothed = smoother.fit_transform(y)
```
## Algorithm Selection
### For Feature Extraction:
- **Speed + Performance**: MiniRocketTransformer
- **Interpretability**: Catch22, TSFresh
- **Dimensionality reduction**: PAA, SAX, PCA
- **Discriminative patterns**: Shapelet transforms
- **Comprehensive features**: TSFresh (with longer runtime)
### For Preprocessing:
- **Normalization**: Normalizer, MinMaxScaler
- **Smoothing**: MovingAverage, SavitzkyGolayFilter
- **Missing values**: SimpleImputer
- **Frequency analysis**: DWTTransformer, Fourier methods
### For Symbolic Representation:
- **Fast approximation**: PAA
- **Alphabet-based**: SAX
- **Frequency-based**: SFA, SFAFast
## Best Practices
1. **Fit on training data only**: Avoid data leakage
```python
transformer.fit(X_train)
X_train_tf = transformer.transform(X_train)
X_test_tf = transformer.transform(X_test)
```
2. **Pipeline composition**: Chain transformers for complex workflows
```python
pipeline = CollectionTransformerPipeline([
('imputer', SimpleImputer()),
('scaler', Normalizer()),
('features', RocketTransformer())
])
```
3. **Feature selection**: TSFresh can generate many features; consider selection
```python
from sklearn.feature_selection import SelectKBest
selector = SelectKBest(k=100)
X_selected = selector.fit_transform(X_features, y)
```
4. **Memory considerations**: Some transformers memory-intensive on large datasets
- Use MiniRocket instead of ROCKET for speed
- Consider downsampling for very long series
- Use ROCKETGPU for GPU acceleration
5. **Domain knowledge**: Choose transformations matching domain:
- Periodic data: Fourier-based methods
- Noisy data: Smoothing filters
- Spike detection: Wavelet transforms
skills/anndata/SKILL.md
+429
View File
@@ -0,0 +1,429 @@
---
name: anndata
description: Data structure for annotated matrices in single-cell analysis. Use when working with .h5ad files or integrating with the scverse ecosystem. This is the data format skill—for analysis workflows use scanpy; for probabilistic models use scvi-tools; for population-scale queries use cellxgene-census.
license: BSD-3-Clause license
allowed-tools: Read Write Edit Bash
compatibility: Requires Python 3.11+ and uv. Examples target AnnData 0.12.16, with experimental APIs clearly marked where used.
metadata: {"version": "1.1", "skill-author": "K-Dense Inc."}
---
# AnnData
## Overview
AnnData is a Python package for handling annotated data matrices, storing experimental measurements (X) alongside observation metadata (obs), variable metadata (var), and multi-dimensional annotations (obsm, varm, obsp, varp, uns). Originally designed for single-cell genomics through Scanpy, it now serves as a general-purpose framework for any annotated data requiring efficient storage, manipulation, and analysis.
## When to Use This Skill
Use this skill when:
- Creating, reading, or writing AnnData objects
- Working with h5ad, zarr, or other genomics data formats
- Performing single-cell RNA-seq analysis
- Managing large datasets with sparse matrices or backed mode
- Concatenating multiple datasets or experimental batches
- Subsetting, filtering, or transforming annotated data
- Integrating with scanpy, scvi-tools, or other scverse ecosystem tools
## Installation
Requires Python 3.11+. Current stable release: 0.12.16 (released 2026-05-18).
```bash
uv pip install "anndata==0.12.16"
# Lazy I/O and dask-backed operations
uv pip install "anndata[dask,lazy]==0.12.16"
# Development / docs (contributors)
uv pip install "anndata[dev,test,doc]==0.12.16"
```
Use unpinned installs only when intentionally tracking the latest compatible release.
Current API notes:
- Use `anndata.io` for non-native `read_*` and `write_*` helpers. Top-level `anndata.read_h5ad` and `anndata.read_zarr` remain supported.
- Avoid deprecated APIs: `ad.read`, `AnnData.concatenate()`, `AnnData.*_keys()`, and `anndata.__version__`. Prefer `ad.read_h5ad`, `ad.concat`, mapping `.keys()`, and `importlib.metadata.version("anndata")`.
- Treat `anndata.experimental` APIs as useful but unstable. Prefer them for large-data workflows only when their current caveats are acceptable.
## Quick Start
### Creating an AnnData object
```python
import anndata as ad
import numpy as np
import pandas as pd
# Minimal creation
X = np.random.rand(100, 2000) # 100 cells × 2000 genes
adata = ad.AnnData(X)
# With metadata
obs = pd.DataFrame({
'cell_type': ['T cell', 'B cell'] * 50,
'sample': ['A', 'B'] * 50
}, index=[f'cell_{i}' for i in range(100)])
var = pd.DataFrame({
'gene_name': [f'Gene_{i}' for i in range(2000)]
}, index=[f'ENSG{i:05d}' for i in range(2000)])
adata = ad.AnnData(X=X, obs=obs, var=var)
```
### Reading data
```python
# Native formats (read_h5ad/read_zarr remain at top-level)
adata = ad.read_h5ad('data.h5ad')
adata = ad.read_h5ad('large_data.h5ad', backed='r') # lazy load for large files
adata = ad.read_zarr('data.zarr')
# Other formats: prefer anndata.io (top-level imports are deprecated)
from anndata.io import read_csv, read_loom, read_mtx
adata = read_csv('data.csv')
adata = read_loom('data.loom')
# 10X Genomics: use scanpy (not anndata) — see scanpy skill
import scanpy as sc
adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5')
adata = sc.read_10x_mtx('filtered_feature_bc_matrix/')
```
### Writing data
```python
# Write h5ad file
adata.write_h5ad('output.h5ad')
# Write with compression
adata.write_h5ad('output.h5ad', compression='gzip')
# Write other formats
adata.write_zarr('output.zarr')
adata.write_csvs('output_dir/')
```
### Basic operations
```python
# Subset by conditions
t_cells = adata[adata.obs['cell_type'] == 'T cell']
# Subset by indices
subset = adata[0:50, 0:100]
# Add metadata
adata.obs['quality_score'] = np.random.rand(adata.n_obs)
adata.var['highly_variable'] = np.random.rand(adata.n_vars) > 0.8
# Access dimensions
print(f"{adata.n_obs} observations × {adata.n_vars} variables")
```
## Core Capabilities
### 1. Data Structure
Understand the AnnData object structure including X, obs, var, layers, obsm, varm, obsp, varp, uns, and raw components.
**See**: `references/data_structure.md` for comprehensive information on:
- Core components (X, obs, var, layers, obsm, varm, obsp, varp, uns, raw)
- Creating AnnData objects from various sources
- Accessing and manipulating data components
- Memory-efficient practices
### 2. Input/Output Operations
Read and write data in various formats with support for compression, backed mode, and cloud storage.
**See**: `references/io_operations.md` for details on:
- Native formats (h5ad, zarr)
- Alternative formats (CSV, MTX, Loom, 10X, Excel)
- Backed mode for large datasets
- Remote data access
- Format conversion
- Performance optimization
Common commands:
```python
from anndata.io import read_mtx
# Read/write h5ad
adata = ad.read_h5ad('data.h5ad', backed='r')
adata.write_h5ad('output.h5ad', compression='gzip')
# 10X Genomics (via scanpy)
import scanpy as sc
adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5')
# Read MTX format
adata = read_mtx('matrix.mtx').T
```
### 3. Concatenation
Combine multiple AnnData objects along observations or variables with flexible join strategies.
**See**: `references/concatenation.md` for comprehensive coverage of:
- Basic concatenation (axis=0 for observations, axis=1 for variables)
- Join types (inner, outer)
- Merge strategies (same, unique, first, only)
- Tracking data sources with labels
- Lazy concatenation (AnnCollection)
- On-disk concatenation for large datasets
Common commands:
```python
# Concatenate observations (combine samples)
adata = ad.concat(
[adata1, adata2, adata3],
axis=0,
join='inner',
label='batch',
keys=['batch1', 'batch2', 'batch3']
)
# Concatenate variables (combine modalities)
adata = ad.concat([adata_rna, adata_protein], axis=1)
# Lazy collection over backed AnnData objects (experimental)
from anndata.experimental import AnnCollection
backed_adatas = [
ad.read_h5ad(path, backed='r')
for path in ['data1.h5ad', 'data2.h5ad']
]
collection = AnnCollection(
backed_adatas,
join_obs='outer',
join_vars='inner',
label='dataset'
)
```
### 4. Data Manipulation
Transform, subset, filter, and reorganize data efficiently.
**See**: `references/manipulation.md` for detailed guidance on:
- Subsetting (by indices, names, boolean masks, metadata conditions)
- Transposition
- Copying (full copies vs views)
- Renaming (observations, variables, categories)
- Type conversions (strings to categoricals, sparse/dense)
- Adding/removing data components
- Reordering
- Quality control filtering
Common commands:
```python
# Subset by metadata
filtered = adata[adata.obs['quality_score'] > 0.8]
hv_genes = adata[:, adata.var['highly_variable']]
# Transpose
adata_T = adata.T
# Copy vs view
view = adata[0:100, :] # View (lightweight reference)
copy = adata[0:100, :].copy() # Independent copy
# Convert strings to categoricals
adata.strings_to_categoricals()
```
### 5. Best Practices
Follow recommended patterns for memory efficiency, performance, and reproducibility.
**See**: `references/best_practices.md` for guidelines on:
- Memory management (sparse matrices, categoricals, backed mode)
- Views vs copies
- Data storage optimization
- Performance optimization
- Working with raw data
- Metadata management
- Reproducibility
- Error handling
- Integration with other tools
- Common pitfalls and solutions
Key recommendations:
```python
# Use sparse matrices for sparse data
from scipy.sparse import csr_matrix
adata.X = csr_matrix(adata.X)
# Convert strings to categoricals
adata.strings_to_categoricals()
# Use backed mode for large files
adata = ad.read_h5ad('large.h5ad', backed='r')
# Store raw before filtering
adata.raw = adata.copy()
adata = adata[:, adata.var['highly_variable']]
```
## Integration with Scverse Ecosystem
AnnData serves as the foundational data structure for the scverse ecosystem:
### Scanpy (Single-cell analysis)
```python
import scanpy as sc
# Preprocessing
sc.pp.filter_cells(adata, min_genes=200)
sc.pp.normalize_total(adata, target_sum=1e4)
sc.pp.log1p(adata)
sc.pp.highly_variable_genes(adata, n_top_genes=2000)
# Dimensionality reduction
sc.pp.pca(adata, n_comps=50)
sc.pp.neighbors(adata, n_neighbors=15)
sc.tl.umap(adata)
sc.tl.leiden(adata)
# Visualization
sc.pl.umap(adata, color=['cell_type', 'leiden'])
```
### Muon (Multimodal data)
```python
import muon as mu
# Combine RNA and protein data
mdata = mu.MuData({'rna': adata_rna, 'protein': adata_protein})
```
### PyTorch integration
```python
from anndata.experimental import AnnLoader
# Create DataLoader for deep learning
dataloader = AnnLoader(adata, batch_size=128, shuffle=True)
for batch in dataloader:
X = batch.X
# Train model
```
## Common Workflows
### Single-cell RNA-seq analysis
```python
import anndata as ad
import scanpy as sc
# 1. Load data (10X via scanpy; anndata handles h5ad/zarr natively)
adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5')
# 2. Quality control
adata.obs['n_genes'] = (adata.X > 0).sum(axis=1)
adata.obs['n_counts'] = adata.X.sum(axis=1)
adata = adata[adata.obs['n_genes'] > 200]
adata = adata[adata.obs['n_counts'] < 50000]
# 3. Store raw
adata.raw = adata.copy()
# 4. Normalize and filter
sc.pp.normalize_total(adata, target_sum=1e4)
sc.pp.log1p(adata)
sc.pp.highly_variable_genes(adata, n_top_genes=2000)
adata = adata[:, adata.var['highly_variable']]
# 5. Save processed data
adata.write_h5ad('processed.h5ad')
```
### Batch integration
```python
# Load multiple batches
adata1 = ad.read_h5ad('batch1.h5ad')
adata2 = ad.read_h5ad('batch2.h5ad')
adata3 = ad.read_h5ad('batch3.h5ad')
# Concatenate with batch labels
adata = ad.concat(
[adata1, adata2, adata3],
label='batch',
keys=['batch1', 'batch2', 'batch3'],
join='inner'
)
# Apply batch correction
import scanpy as sc
sc.pp.combat(adata, key='batch')
# Continue analysis
sc.pp.pca(adata)
sc.pp.neighbors(adata)
sc.tl.umap(adata)
```
### Working with large datasets
```python
# Open in backed mode
adata = ad.read_h5ad('100GB_dataset.h5ad', backed='r')
# Filter based on metadata (no data loading)
high_quality = adata[adata.obs['quality_score'] > 0.8]
# Load filtered subset
adata_subset = high_quality.to_memory()
# Process subset
process(adata_subset)
# Or process in chunks
chunk_size = 1000
for i in range(0, adata.n_obs, chunk_size):
chunk = adata[i:i+chunk_size, :].to_memory()
process(chunk)
```
## Troubleshooting
### Out of memory errors
Use backed mode or convert to sparse matrices:
```python
# Backed mode
adata = ad.read_h5ad('file.h5ad', backed='r')
# Sparse matrices
from scipy.sparse import csr_matrix
adata.X = csr_matrix(adata.X)
```
### Slow file reading
Use compression and appropriate formats:
```python
# Optimize for storage
adata.strings_to_categoricals()
adata.write_h5ad('file.h5ad', compression='gzip')
# Use Zarr for cloud storage; v3 writes are opt-in in anndata 0.12
import anndata as ad
ad.settings.zarr_write_format = 3
ad.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format
adata.write_zarr('file.zarr', chunks=(1000, 1000))
```
### Index alignment issues
Always align external data on index:
```python
# Wrong
adata.obs['new_col'] = external_data['values']
# Correct
adata.obs['new_col'] = external_data.set_index('cell_id').loc[adata.obs_names, 'values']
```
## Additional Resources
- **Official documentation**: https://anndata.readthedocs.io/
- **Scanpy tutorials**: https://scanpy.readthedocs.io/
- **Scverse ecosystem**: https://scverse.org/
- **GitHub repository**: https://github.com/scverse/anndata
skills/anndata/references/best_practices.md
+532
View File
@@ -0,0 +1,532 @@
# Best Practices
Guidelines for efficient and effective use of AnnData.
## Memory Management
### Use sparse matrices for sparse data
```python
import numpy as np
from scipy.sparse import csr_matrix, csc_matrix
import anndata as ad
# Check data sparsity
data = np.random.rand(1000, 2000)
sparsity = 1 - np.count_nonzero(data) / data.size
print(f"Sparsity: {sparsity:.2%}")
# Convert to sparse if >50% zeros (anndata 0.12+ requires csr or csc)
if sparsity > 0.5:
adata = ad.AnnData(X=csr_matrix(data))
else:
adata = ad.AnnData(X=data)
# Benefits: 10-100x memory reduction for sparse genomics data
```
### Convert strings to categoricals
```python
# Inefficient: string columns use lots of memory
adata.obs['cell_type'] = ['Type_A', 'Type_B', 'Type_C'] * 333 + ['Type_A']
# Efficient: convert to categorical
adata.obs['cell_type'] = adata.obs['cell_type'].astype('category')
# Convert all string columns
adata.strings_to_categoricals()
# Benefits: 10-50x memory reduction for repeated strings
```
### Use backed mode for large datasets
```python
# Don't load entire dataset into memory
adata = ad.read_h5ad('large_dataset.h5ad', backed='r')
# Work with metadata
filtered = adata[adata.obs['quality'] > 0.8]
# Load only filtered subset
adata_subset = filtered.to_memory()
# Benefits: Work with datasets larger than RAM
```
## Views vs Copies
### Understanding views
```python
# Subsetting creates a view by default
subset = adata[0:100, :]
print(subset.is_view) # True
# Views don't copy data (memory efficient)
# But modifications can affect original
# Check if object is a view
if adata.is_view:
adata = adata.copy() # Make independent
```
### When to use views
```python
# Good: Read-only operations on subsets
mean_expr = adata[adata.obs['cell_type'] == 'T cell'].X.mean()
# Good: Temporary analysis
temp_subset = adata[:100, :]
result = analyze(temp_subset.X)
```
### When to use copies
```python
# Create independent copy for modifications
adata_filtered = adata[keep_cells, :].copy()
# Safe to modify without affecting original
adata_filtered.obs['new_column'] = values
# Always copy when:
# - Storing subset for later use
# - Modifying subset data
# - Passing to function that modifies data
```
## Data Storage Best Practices
### Choose the right format
**H5AD (HDF5) - Default choice**
```python
adata.write_h5ad('data.h5ad', compression='gzip')
```
- Fast random access
- Supports backed mode
- Good compression
- Best for: Most use cases
**Zarr - Cloud and parallel access**
```python
import anndata
# Default is Zarr v2; opt into v3 for cloud workflows (anndata 0.12+)
anndata.settings.zarr_write_format = 3
anndata.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format
adata.write_zarr('data.zarr', chunks=(100, 100))
```
- Excellent for cloud storage (S3, GCS)
- Supports parallel I/O and opt-in Zarr v3 sharding (0.12+)
- Good compression
- Best for: Large datasets, cloud workflows, parallel processing
**CSV - Interoperability**
```python
adata.write_csvs('output_dir/')
```
- Human readable
- Compatible with all tools
- Large file sizes, slow
- Best for: Sharing with non-Python tools, small datasets
### Optimize file size
```python
# Before saving, optimize:
# 1. Convert to sparse if appropriate
from scipy.sparse import csr_matrix, issparse
if not issparse(adata.X):
density = np.count_nonzero(adata.X) / adata.X.size
if density < 0.5:
adata.X = csr_matrix(adata.X)
# 2. Convert strings to categoricals
adata.strings_to_categoricals()
# 3. Use compression
adata.write_h5ad('data.h5ad', compression='gzip', compression_opts=9)
# Typical results: 5-20x file size reduction
```
## Backed Mode Strategies
### Read-only analysis
```python
# Open in read-only backed mode
adata = ad.read_h5ad('data.h5ad', backed='r')
# Perform filtering without loading data
high_quality = adata[adata.obs['quality_score'] > 0.8]
# Load only filtered data
adata_filtered = high_quality.to_memory()
```
### Read-write modifications
```python
# Open in read-write backed mode
adata = ad.read_h5ad('data.h5ad', backed='r+')
# Modify X (persisted to disk)
adata.X[0, 0] = 0
# Metadata changes are not persisted from backed mode; load and write a new file
adata_memory = adata.to_memory()
adata_memory.obs['new_annotation'] = values
adata_memory.write_h5ad('data_with_annotations.h5ad')
```
### Chunked processing
```python
# Process large dataset in chunks
adata = ad.read_h5ad('huge_dataset.h5ad', backed='r')
results = []
chunk_size = 1000
for i in range(0, adata.n_obs, chunk_size):
chunk = adata[i:i+chunk_size, :].to_memory()
result = process(chunk)
results.append(result)
final_result = combine(results)
```
## Performance Optimization
### Subsetting performance
```python
# Fast: Boolean indexing with arrays
mask = np.array(adata.obs['quality'] > 0.5)
subset = adata[mask, :]
# Slow: Boolean indexing with Series (creates view chain)
subset = adata[adata.obs['quality'] > 0.5, :]
# Fastest: Integer indices
indices = np.where(adata.obs['quality'] > 0.5)[0]
subset = adata[indices, :]
```
### Avoid repeated subsetting
```python
# Inefficient: Multiple subset operations
for cell_type in ['A', 'B', 'C']:
subset = adata[adata.obs['cell_type'] == cell_type]
process(subset)
# Efficient: Group and process
groups = adata.obs.groupby('cell_type').groups
for cell_type, indices in groups.items():
subset = adata[indices, :]
process(subset)
```
### Use chunked operations for large matrices
```python
# Process X in chunks
for chunk in adata.chunked_X(chunk_size=1000):
result = compute(chunk)
# More memory efficient than loading full X
```
## Working with Raw Data
### Store raw before filtering
```python
# Original data with all genes
adata = ad.AnnData(X=counts)
# Store raw before filtering
adata.raw = adata.copy()
# Filter to highly variable genes
adata = adata[:, adata.var['highly_variable']]
# Later: access original data
original_expression = adata.raw.X
all_genes = adata.raw.var_names
```
### When to use raw
```python
# Use raw for:
# - Differential expression on filtered genes
# - Visualization of specific genes not in filtered set
# - Accessing original counts after normalization
# Access raw data
if adata.raw is not None:
gene_expr = adata.raw[:, 'GENE_NAME'].X
else:
gene_expr = adata[:, 'GENE_NAME'].X
```
## Metadata Management
### Naming conventions
```python
# Consistent naming improves usability
# Observation metadata (obs):
# - cell_id, sample_id
# - cell_type, tissue, condition
# - n_genes, n_counts, percent_mito
# - cluster, leiden, louvain
# Variable metadata (var):
# - gene_id, gene_name
# - highly_variable, n_cells
# - mean_expression, dispersion
# Embeddings (obsm):
# - X_pca, X_umap, X_tsne
# - X_diffmap, X_draw_graph_fr
# Follow conventions from scanpy/scverse ecosystem
```
### Document metadata
```python
# Store metadata descriptions in uns
adata.uns['metadata_descriptions'] = {
'cell_type': 'Cell type annotation from automated clustering',
'quality_score': 'QC score from scrublet (0-1, higher is better)',
'batch': 'Experimental batch identifier'
}
# Store processing history
adata.uns['processing_steps'] = [
'Raw counts loaded from 10X',
'Filtered: n_genes > 200, n_counts < 50000',
'Normalized to 10000 counts per cell',
'Log transformed'
]
```
## Reproducibility
### Set random seeds
```python
import numpy as np
# Set seed for reproducible results
np.random.seed(42)
# Document in uns
adata.uns['random_seed'] = 42
```
### Store parameters
```python
# Store analysis parameters in uns
adata.uns['pca'] = {
'n_comps': 50,
'svd_solver': 'arpack',
'random_state': 42
}
adata.uns['neighbors'] = {
'n_neighbors': 15,
'n_pcs': 50,
'metric': 'euclidean',
'method': 'umap'
}
```
### Version tracking
```python
import sys
from importlib.metadata import version
# Store package versions (anndata.__version__ deprecated in 0.12.3)
adata.uns['versions'] = {
'anndata': version('anndata'),
'scanpy': version('scanpy'),
'numpy': version('numpy'),
'python': sys.version,
}
```
## Error Handling
### Check data validity
```python
# Verify dimensions
assert adata.n_obs == len(adata.obs)
assert adata.n_vars == len(adata.var)
assert adata.X.shape == (adata.n_obs, adata.n_vars)
# Check for NaN values
has_nan = np.isnan(adata.X.data).any() if issparse(adata.X) else np.isnan(adata.X).any()
if has_nan:
print("Warning: Data contains NaN values")
# Check for negative values (if counts expected)
has_negative = (adata.X.data < 0).any() if issparse(adata.X) else (adata.X < 0).any()
if has_negative:
print("Warning: Data contains negative values")
```
### Validate metadata
```python
# Check for missing values
missing_obs = adata.obs.isnull().sum()
if missing_obs.any():
print("Missing values in obs:")
print(missing_obs[missing_obs > 0])
# Verify indices are unique
assert adata.obs_names.is_unique, "Observation names not unique"
assert adata.var_names.is_unique, "Variable names not unique"
# Check metadata alignment
assert len(adata.obs) == adata.n_obs
assert len(adata.var) == adata.n_vars
```
## Integration with Other Tools
### Scanpy integration
```python
import scanpy as sc
# AnnData is native format for scanpy
sc.pp.filter_cells(adata, min_genes=200)
sc.pp.filter_genes(adata, min_cells=3)
sc.pp.normalize_total(adata, target_sum=1e4)
sc.pp.log1p(adata)
sc.pp.highly_variable_genes(adata)
sc.pp.pca(adata)
sc.pp.neighbors(adata)
sc.tl.umap(adata)
```
### Pandas integration
```python
import pandas as pd
# Convert to DataFrame
df = adata.to_df()
# Create from DataFrame
adata = ad.AnnData(df)
# Work with metadata as DataFrames
adata.obs = adata.obs.merge(external_metadata, left_index=True, right_index=True)
```
### PyTorch integration
```python
from anndata.experimental import AnnLoader
# Create PyTorch DataLoader
dataloader = AnnLoader(adata, batch_size=128, shuffle=True)
# Iterate in training loop
for batch in dataloader:
X = batch.X
# Train model on batch
```
## Common Pitfalls
### Pitfall 1: Modifying views
```python
# Wrong: Modifying view can affect original
subset = adata[:100, :]
subset.X = new_data # May modify adata.X!
# Correct: Copy before modifying
subset = adata[:100, :].copy()
subset.X = new_data # Independent copy
```
### Pitfall 2: Index misalignment
```python
# Wrong: Assuming order matches
external_data = pd.read_csv('data.csv')
adata.obs['new_col'] = external_data['values'] # May misalign!
# Correct: Align on index
adata.obs['new_col'] = external_data.set_index('cell_id').loc[adata.obs_names, 'values']
```
### Pitfall 3: Mixing sparse and dense
```python
# Wrong: Converting sparse to dense uses huge memory
result = adata.X + 1 # Converts sparse to dense!
# Correct: Use sparse operations
from scipy.sparse import issparse
if issparse(adata.X):
result = adata.X.copy()
result.data += 1
```
### Pitfall 4: Not handling views
```python
# Wrong: Assuming subset is independent
subset = adata[mask, :]
del adata # subset may become invalid!
# Correct: Copy when needed
subset = adata[mask, :].copy()
del adata # subset remains valid
```
### Pitfall 5: Ignoring memory constraints
```python
# Wrong: Loading huge dataset into memory
adata = ad.read_h5ad('100GB_file.h5ad') # OOM error!
# Correct: Use backed mode
adata = ad.read_h5ad('100GB_file.h5ad', backed='r')
subset = adata[adata.obs['keep']].to_memory()
```
## Workflow Example
Complete best-practices workflow:
```python
import anndata as ad
import numpy as np
from scipy.sparse import csr_matrix, issparse
# 1. Load with backed mode if large
adata = ad.read_h5ad('data.h5ad', backed='r')
# 2. Quick metadata check without loading data
print(f"Dataset: {adata.n_obs} cells × {adata.n_vars} genes")
# 3. Filter based on metadata
high_quality = adata[adata.obs['quality_score'] > 0.8]
# 4. Load filtered subset to memory
adata = high_quality.to_memory()
# 5. Convert to optimal storage types (csr/csc sparse only since 0.12)
adata.strings_to_categoricals()
if not issparse(adata.X):
density = np.count_nonzero(adata.X) / adata.X.size
if density < 0.5:
adata.X = csr_matrix(adata.X)
# 6. Store raw before filtering genes
adata.raw = adata.copy()
# 7. Filter to highly variable genes
adata = adata[:, adata.var['highly_variable']].copy()
# 8. Document processing
adata.uns['processing'] = {
'filtered': 'quality_score > 0.8',
'n_hvg': adata.n_vars,
'date': '2025-11-03'
}
# 9. Save optimized
adata.write_h5ad('processed.h5ad', compression='gzip')
```
skills/anndata/references/concatenation.md
+399
View File
@@ -0,0 +1,399 @@
# Concatenating AnnData Objects
Combine multiple AnnData objects along either observations or variables axis.
## Basic Concatenation
### Concatenate along observations (stack cells/samples)
```python
import anndata as ad
import numpy as np
# Create multiple AnnData objects
adata1 = ad.AnnData(X=np.random.rand(100, 50))
adata2 = ad.AnnData(X=np.random.rand(150, 50))
adata3 = ad.AnnData(X=np.random.rand(200, 50))
# Concatenate along observations (axis=0, default)
adata_combined = ad.concat([adata1, adata2, adata3], axis=0)
print(adata_combined.shape) # (450, 50)
```
### Concatenate along variables (stack genes/features)
```python
# Create objects with same observations, different variables
adata1 = ad.AnnData(X=np.random.rand(100, 50))
adata2 = ad.AnnData(X=np.random.rand(100, 30))
adata3 = ad.AnnData(X=np.random.rand(100, 70))
# Concatenate along variables (axis=1)
adata_combined = ad.concat([adata1, adata2, adata3], axis=1)
print(adata_combined.shape) # (100, 150)
```
## Join Types
### Inner join (intersection)
Keep only variables/observations present in all objects.
```python
import pandas as pd
# Create objects with different variables
adata1 = ad.AnnData(
X=np.random.rand(100, 50),
var=pd.DataFrame(index=[f'Gene_{i}' for i in range(50)])
)
adata2 = ad.AnnData(
X=np.random.rand(150, 60),
var=pd.DataFrame(index=[f'Gene_{i}' for i in range(10, 70)])
)
# Inner join: only genes 10-49 are kept (overlap)
adata_inner = ad.concat([adata1, adata2], join='inner')
print(adata_inner.n_vars) # 40 genes (overlap)
```
### Outer join (union)
Keep all variables/observations, filling missing values.
```python
# Outer join: all genes are kept
adata_outer = ad.concat([adata1, adata2], join='outer')
print(adata_outer.n_vars) # 70 genes (union)
# Missing values are filled with appropriate defaults:
# - 0 for sparse matrices
# - NaN for dense matrices
```
### Fill values for outer joins
```python
# Specify fill value for missing data
adata_filled = ad.concat([adata1, adata2], join='outer', fill_value=0)
```
## Tracking Data Sources
### Add batch labels
```python
# Label which object each observation came from
adata_combined = ad.concat(
[adata1, adata2, adata3],
label='batch', # Column name for labels
keys=['batch1', 'batch2', 'batch3'] # Labels for each object
)
print(adata_combined.obs['batch'].value_counts())
# batch1 100
# batch2 150
# batch3 200
```
### Automatic batch labels
```python
# If keys not provided, uses integer indices
adata_combined = ad.concat(
[adata1, adata2, adata3],
label='dataset'
)
# dataset column contains: 0, 1, 2
```
## Merge Strategies
Control how metadata from different objects is combined using the `merge` parameter.
### merge=None (default for observations)
Exclude metadata on non-concatenation axis.
```python
# When concatenating observations, var metadata must match
adata1.var['gene_type'] = 'protein_coding'
adata2.var['gene_type'] = 'protein_coding'
# var is kept only if identical across all objects
adata_combined = ad.concat([adata1, adata2], merge=None)
```
### merge='same'
Keep metadata that is identical across all objects.
```python
adata1.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25
adata2.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25
adata1.var['type'] = 'protein_coding'
adata2.var['type'] = 'lncRNA' # Different
# 'chromosome' is kept (same), 'type' is excluded (different)
adata_combined = ad.concat([adata1, adata2], merge='same')
```
### merge='unique'
Keep metadata columns where each key has exactly one value.
```python
adata1.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)]
adata2.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)]
# gene_id is kept (unique values for each key)
adata_combined = ad.concat([adata1, adata2], merge='unique')
```
### merge='first'
Take values from the first object containing each key.
```python
adata1.var['description'] = ['Desc1'] * 50
adata2.var['description'] = ['Desc2'] * 50
# Uses descriptions from adata1
adata_combined = ad.concat([adata1, adata2], merge='first')
```
### merge='only'
Keep metadata that appears in only one object.
```python
adata1.var['adata1_specific'] = [1] * 50
adata2.var['adata2_specific'] = [2] * 50
# Both metadata columns are kept
adata_combined = ad.concat([adata1, adata2], merge='only')
```
## Handling Index Conflicts
### Make indices unique
```python
import pandas as pd
# Create objects with overlapping observation names
adata1 = ad.AnnData(
X=np.random.rand(3, 10),
obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3'])
)
adata2 = ad.AnnData(
X=np.random.rand(3, 10),
obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3'])
)
# Make indices unique by appending batch keys
adata_combined = ad.concat(
[adata1, adata2],
label='batch',
keys=['batch1', 'batch2'],
index_unique='_' # Separator for making indices unique
)
print(adata_combined.obs_names)
# ['cell_1_batch1', 'cell_2_batch1', 'cell_3_batch1',
# 'cell_1_batch2', 'cell_2_batch2', 'cell_3_batch2']
```
## Concatenating Layers
```python
# Objects with layers
adata1 = ad.AnnData(X=np.random.rand(100, 50))
adata1.layers['normalized'] = np.random.rand(100, 50)
adata1.layers['scaled'] = np.random.rand(100, 50)
adata2 = ad.AnnData(X=np.random.rand(150, 50))
adata2.layers['normalized'] = np.random.rand(150, 50)
adata2.layers['scaled'] = np.random.rand(150, 50)
# Layers are concatenated automatically if present in all objects
adata_combined = ad.concat([adata1, adata2])
print(adata_combined.layers.keys())
# dict_keys(['normalized', 'scaled'])
```
## Concatenating Multi-dimensional Annotations
### obsm/varm
```python
# Objects with embeddings
adata1.obsm['X_pca'] = np.random.rand(100, 50)
adata2.obsm['X_pca'] = np.random.rand(150, 50)
# obsm is concatenated along observation axis
adata_combined = ad.concat([adata1, adata2])
print(adata_combined.obsm['X_pca'].shape) # (250, 50)
```
### obsp/varp (pairwise annotations)
```python
from scipy.sparse import csr_matrix
# Pairwise matrices
adata1.obsp['connectivities'] = csr_matrix((100, 100))
adata2.obsp['connectivities'] = csr_matrix((150, 150))
# By default, obsp is NOT concatenated (set pairwise=True to include)
adata_combined = ad.concat([adata1, adata2])
# adata_combined.obsp is empty
# Include pairwise data (creates block diagonal matrix)
adata_combined = ad.concat([adata1, adata2], pairwise=True)
print(adata_combined.obsp['connectivities'].shape) # (250, 250)
```
## Concatenating uns (unstructured)
Unstructured metadata is merged recursively:
```python
adata1.uns['experiment'] = {'date': '2025-01-01', 'batch': 'A'}
adata2.uns['experiment'] = {'date': '2025-01-01', 'batch': 'B'}
# Using merge='unique' for uns
adata_combined = ad.concat([adata1, adata2], uns_merge='unique')
# 'date' is kept (same value), 'batch' might be excluded (different values)
```
## Lazy Concatenation (AnnCollection)
For very large datasets, use `AnnCollection` to lazily concatenate AnnData objects along the observation axis. This API is experimental; use backed AnnData objects when the inputs are stored in `.h5ad` files.
```python
import anndata as ad
from anndata.experimental import AnnCollection
files = ['data1.h5ad', 'data2.h5ad', 'data3.h5ad']
backed_adatas = [ad.read_h5ad(path, backed='r') for path in files]
collection = AnnCollection(
backed_adatas,
join_obs='outer',
join_vars='inner',
label='dataset',
keys=['dataset1', 'dataset2', 'dataset3']
)
# Access data lazily
print(collection.n_obs) # Total observations
print(collection.obs.head()) # Metadata loaded, not X
# Convert to regular AnnData when needed (loads all data)
adata = collection.to_adata()
```
### Working with AnnCollection
```python
# Subset without loading data
subset = collection[collection.obs['cell_type'] == 'T cell']
# Iterate through datasets
for adata in collection:
print(adata.shape)
# Access specific dataset
first_dataset = collection[0]
```
## Concatenation on Disk
For datasets too large for memory, concatenate directly on disk:
```python
import anndata as ad
from anndata.experimental import concat_on_disk
# Concatenate without loading into memory
concat_on_disk(
['data1.h5ad', 'data2.h5ad', 'data3.h5ad'],
'combined.h5ad',
join='outer'
)
# Load result in backed mode
adata = ad.read_h5ad('combined.h5ad', backed='r')
```
## Common Concatenation Patterns
### Combine technical replicates
```python
# Multiple runs of the same samples
replicates = [adata_run1, adata_run2, adata_run3]
adata_combined = ad.concat(
replicates,
label='technical_replicate',
keys=['rep1', 'rep2', 'rep3'],
join='inner' # Keep only genes measured in all runs
)
```
### Combine batches from experiment
```python
# Different experimental batches
batches = [adata_batch1, adata_batch2, adata_batch3]
adata_combined = ad.concat(
batches,
label='batch',
keys=['batch1', 'batch2', 'batch3'],
join='outer' # Keep all genes
)
# Later: apply batch correction
```
### Merge multi-modal data
```python
# Different measurement modalities (e.g., RNA + protein)
adata_rna = ad.AnnData(X=np.random.rand(100, 2000))
adata_protein = ad.AnnData(X=np.random.rand(100, 50))
# Concatenate along variables to combine modalities
adata_multimodal = ad.concat([adata_rna, adata_protein], axis=1)
# Add labels to distinguish modalities
adata_multimodal.var['modality'] = ['RNA'] * 2000 + ['protein'] * 50
```
## Best Practices
1. **Check compatibility before concatenating**
```python
# Verify shapes are compatible
print([adata.n_vars for adata in [adata1, adata2, adata3]])
# Check variable names match
print([set(adata.var_names) for adata in [adata1, adata2, adata3]])
```
2. **Use appropriate join type**
- `inner`: When you need the same features across all samples (most stringent)
- `outer`: When you want to preserve all features (most inclusive)
3. **Track data sources**
Always use `label` and `keys` to track which observations came from which dataset.
4. **Consider memory usage**
- For large datasets, use `AnnCollection` or `concat_on_disk`
- Consider backed mode for the result
5. **Handle batch effects**
Concatenation combines data but doesn't correct for batch effects. Apply batch correction after concatenation:
```python
# After concatenation, apply batch correction
import scanpy as sc
sc.pp.combat(adata_combined, key='batch')
```
6. **Validate results**
```python
# Check dimensions
print(adata_combined.shape)
# Check batch distribution
print(adata_combined.obs['batch'].value_counts())
# Verify metadata integrity
print(adata_combined.var.head())
print(adata_combined.obs.head())
```
skills/anndata/references/data_structure.md
+314
View File
@@ -0,0 +1,314 @@
# AnnData Object Structure
The AnnData object stores a data matrix with associated annotations, providing a flexible framework for managing experimental data and metadata.
## Core Components
### X (Data Matrix)
The primary data matrix with shape (n_obs, n_vars) storing experimental measurements.
```python
import anndata as ad
import numpy as np
# Create with dense array
adata = ad.AnnData(X=np.random.rand(100, 2000))
# Create with sparse matrix (recommended for large, sparse data)
from scipy.sparse import csr_matrix
sparse_data = csr_matrix(np.random.rand(100, 2000))
adata = ad.AnnData(X=sparse_data)
```
Access data:
```python
# Full matrix (caution with large datasets)
full_data = adata.X
# Single observation
obs_data = adata.X[0, :]
# Single variable across all observations
var_data = adata.X[:, 0]
```
### obs (Observation Annotations)
DataFrame storing metadata about observations (rows). Each row corresponds to one observation in X.
```python
import pandas as pd
# Create AnnData with observation metadata
obs_df = pd.DataFrame({
'cell_type': ['T cell', 'B cell', 'Monocyte'],
'treatment': ['control', 'treated', 'control'],
'timepoint': [0, 24, 24]
}, index=['cell_1', 'cell_2', 'cell_3'])
adata = ad.AnnData(X=np.random.rand(3, 100), obs=obs_df)
# Access observation metadata
print(adata.obs['cell_type'])
print(adata.obs.loc['cell_1'])
```
### var (Variable Annotations)
DataFrame storing metadata about variables (columns). Each row corresponds to one variable in X.
```python
# Create AnnData with variable metadata
var_df = pd.DataFrame({
'gene_name': ['ACTB', 'GAPDH', 'TP53'],
'chromosome': ['7', '12', '17'],
'highly_variable': [True, False, True]
}, index=['ENSG00001', 'ENSG00002', 'ENSG00003'])
adata = ad.AnnData(X=np.random.rand(100, 3), var=var_df)
# Access variable metadata
print(adata.var['gene_name'])
print(adata.var.loc['ENSG00001'])
```
### layers (Alternative Data Representations)
Dictionary storing alternative matrices with the same dimensions as X.
```python
# Store raw counts, normalized data, and scaled data
adata = ad.AnnData(X=np.random.rand(100, 2000))
adata.layers['raw_counts'] = np.random.randint(0, 100, (100, 2000))
adata.layers['normalized'] = adata.X / np.sum(adata.X, axis=1, keepdims=True)
adata.layers['scaled'] = (adata.X - adata.X.mean()) / adata.X.std()
# Access layers
raw_data = adata.layers['raw_counts']
normalized_data = adata.layers['normalized']
```
Common layer uses:
- `raw_counts`: Original count data before normalization
- `normalized`: Log-normalized or TPM values
- `scaled`: Z-scored values for analysis
- `imputed`: Data after imputation
### obsm (Multi-dimensional Observation Annotations)
Dictionary storing multi-dimensional arrays aligned to observations.
```python
# Store PCA coordinates and UMAP embeddings
adata.obsm['X_pca'] = np.random.rand(100, 50) # 50 principal components
adata.obsm['X_umap'] = np.random.rand(100, 2) # 2D UMAP coordinates
adata.obsm['X_tsne'] = np.random.rand(100, 2) # 2D t-SNE coordinates
# Access embeddings
pca_coords = adata.obsm['X_pca']
umap_coords = adata.obsm['X_umap']
```
Common obsm uses:
- `X_pca`: Principal component coordinates
- `X_umap`: UMAP embedding coordinates
- `X_tsne`: t-SNE embedding coordinates
- `X_diffmap`: Diffusion map coordinates
- `protein_expression`: Protein abundance measurements (CITE-seq)
### varm (Multi-dimensional Variable Annotations)
Dictionary storing multi-dimensional arrays aligned to variables.
```python
# Store PCA loadings
adata.varm['PCs'] = np.random.rand(2000, 50) # Loadings for 50 components
adata.varm['gene_modules'] = np.random.rand(2000, 10) # Gene module scores
# Access loadings
pc_loadings = adata.varm['PCs']
```
Common varm uses:
- `PCs`: Principal component loadings
- `gene_modules`: Gene co-expression module assignments
### obsp (Pairwise Observation Relationships)
Dictionary storing sparse matrices representing relationships between observations.
```python
from scipy.sparse import csr_matrix
# Store k-nearest neighbor graph
n_obs = 100
knn_graph = csr_matrix(np.random.rand(n_obs, n_obs) > 0.95)
adata.obsp['connectivities'] = knn_graph
adata.obsp['distances'] = csr_matrix(np.random.rand(n_obs, n_obs))
# Access graphs
knn_connections = adata.obsp['connectivities']
distances = adata.obsp['distances']
```
Common obsp uses:
- `connectivities`: Cell-cell neighborhood graph
- `distances`: Pairwise distances between cells
### varp (Pairwise Variable Relationships)
Dictionary storing sparse matrices representing relationships between variables.
```python
# Store gene-gene correlation matrix
n_vars = 2000
gene_corr = csr_matrix(np.random.rand(n_vars, n_vars) > 0.99)
adata.varp['correlations'] = gene_corr
# Access correlations
gene_correlations = adata.varp['correlations']
```
### uns (Unstructured Annotations)
Dictionary storing arbitrary unstructured metadata.
```python
# Store analysis parameters and results
adata.uns['experiment_date'] = '2025-11-03'
adata.uns['pca'] = {
'variance_ratio': [0.15, 0.10, 0.08],
'params': {'n_comps': 50}
}
adata.uns['neighbors'] = {
'params': {'n_neighbors': 15, 'method': 'umap'},
'connectivities_key': 'connectivities'
}
# Access unstructured data
exp_date = adata.uns['experiment_date']
pca_params = adata.uns['pca']['params']
```
Common uns uses:
- Analysis parameters and settings
- Color palettes for plotting
- Cluster information
- Tool-specific metadata
### raw (Original Data Snapshot)
Optional attribute preserving the original data matrix and variable annotations before filtering.
```python
# Create AnnData and store raw state
adata = ad.AnnData(X=np.random.rand(100, 5000))
adata.var['gene_name'] = [f'Gene_{i}' for i in range(5000)]
# Store raw state before filtering
adata.raw = adata.copy()
# Filter to highly variable genes
highly_variable_mask = np.random.rand(5000) > 0.5
adata = adata[:, highly_variable_mask]
# Access original data
original_matrix = adata.raw.X
original_var = adata.raw.var
```
## Object Properties
```python
# Dimensions
n_observations = adata.n_obs
n_variables = adata.n_vars
shape = adata.shape # (n_obs, n_vars)
# Index information
obs_names = adata.obs_names # Observation identifiers
var_names = adata.var_names # Variable identifiers
# Storage mode
is_view = adata.is_view # True if this is a view of another object
is_backed = adata.isbacked # True if backed by on-disk storage
filename = adata.filename # Path to backing file (if backed)
```
## Creating AnnData Objects
### From arrays and DataFrames
```python
import anndata as ad
import numpy as np
import pandas as pd
# Minimal creation
X = np.random.rand(100, 2000)
adata = ad.AnnData(X)
# With metadata
obs = pd.DataFrame({'cell_type': ['A', 'B'] * 50}, index=[f'cell_{i}' for i in range(100)])
var = pd.DataFrame({'gene_name': [f'Gene_{i}' for i in range(2000)]}, index=[f'ENSG{i:05d}' for i in range(2000)])
adata = ad.AnnData(X=X, obs=obs, var=var)
# With all components
adata = ad.AnnData(
X=X,
obs=obs,
var=var,
layers={'raw': np.random.randint(0, 100, (100, 2000))},
obsm={'X_pca': np.random.rand(100, 50)},
uns={'experiment': 'test'}
)
```
### From DataFrame
```python
# Create from pandas DataFrame (genes as columns, cells as rows)
df = pd.DataFrame(
np.random.rand(100, 50),
columns=[f'Gene_{i}' for i in range(50)],
index=[f'Cell_{i}' for i in range(100)]
)
adata = ad.AnnData(df)
```
## Data Access Patterns
### Vector extraction
```python
# Get observation annotation as array
cell_types = adata.obs_vector('cell_type')
# Get variable values across observations
gene_expression = adata.obs_vector('ACTB') # If ACTB is in var_names
# Get variable annotation as array
gene_names = adata.var_vector('gene_name')
```
### Subsetting
```python
# By index
subset = adata[0:10, 0:100] # First 10 obs, first 100 vars
# By name
subset = adata[['cell_1', 'cell_2'], ['ACTB', 'GAPDH']]
# By boolean mask
high_count_cells = adata.obs['total_counts'] > 1000
subset = adata[high_count_cells, :]
# By observation metadata
t_cells = adata[adata.obs['cell_type'] == 'T cell']
```
## Memory Considerations
The AnnData structure is designed for memory efficiency:
- Sparse matrices reduce memory for sparse data
- Views avoid copying data when possible
- Backed mode enables working with data larger than RAM
- Categorical annotations reduce memory for discrete values
```python
# Convert strings to categoricals (more memory efficient)
adata.obs['cell_type'] = adata.obs['cell_type'].astype('category')
adata.strings_to_categoricals()
# Check if object is a view (doesn't own data)
if adata.is_view:
adata = adata.copy() # Create independent copy
```
skills/anndata/references/io_operations.md
+466
View File
@@ -0,0 +1,466 @@
# Input/Output Operations
AnnData provides comprehensive I/O functionality for reading and writing data in various formats.
Since anndata 0.11, most `read_*` and `write_*` functions live in `anndata.io`. Top-level `read_h5ad` and `read_zarr` remain at `anndata` without deprecation warnings; other top-level imports still work but emit `FutureWarning`.
```python
import anndata as ad
from anndata.io import read_csv, read_mtx, read_loom, read_elem, write_elem
```
Avoid deprecated I/O aliases such as `ad.read`; use `ad.read_h5ad` or `anndata.io.read_h5ad` explicitly.
## Native Formats
### H5AD (HDF5-based)
The recommended native format for AnnData objects, providing efficient storage and fast access.
#### Writing H5AD files
```python
import anndata as ad
# Write to file
adata.write_h5ad('data.h5ad')
# Write with compression
adata.write_h5ad('data.h5ad', compression='gzip')
# Write with specific compression level (0-9, higher = more compression)
adata.write_h5ad('data.h5ad', compression='gzip', compression_opts=9)
```
#### Reading H5AD files
```python
# Read entire file into memory
adata = ad.read_h5ad('data.h5ad')
# Read in backed mode (lazy loading for large files)
adata = ad.read_h5ad('data.h5ad', backed='r') # Read-only
adata = ad.read_h5ad('data.h5ad', backed='r+') # Read-write for X
# Backed mode enables working with datasets larger than RAM
# Only accessed data is loaded into memory
# In backed mode, only X updates are persisted; write a new file for obs/var/uns changes.
```
#### Backed mode operations
```python
# Open in backed mode
adata = ad.read_h5ad('large_dataset.h5ad', backed='r')
# Access metadata without loading X into memory
print(adata.obs.head())
print(adata.var.head())
# Subset operations create views
subset = adata[:100, :500] # View, no data loaded
# Load specific data into memory
X_subset = subset.X[:] # Now loads this subset
# Convert entire backed object to memory
adata_memory = adata.to_memory()
```
### Zarr
Hierarchical array storage format, optimized for cloud storage and parallel I/O.
#### Writing Zarr
```python
# Write to Zarr store
adata.write_zarr('data.zarr')
# Write with specific chunks (important for performance)
adata.write_zarr('data.zarr', chunks=(100, 100))
```
#### Reading Zarr
```python
# Read Zarr store
adata = ad.read_zarr('data.zarr')
```
#### Zarr v3 (anndata 0.12+)
```python
import anndata
# Default writes Zarr v2; opt into v3 and optional auto-sharding
anndata.settings.zarr_write_format = 3
anndata.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format
adata.write_zarr('data.zarr', chunks=(1000, 1000))
```
Zarr v3 writing is available in anndata 0.12, with structured-array exceptions and evolving performance guidance. Consolidated metadata is recommended for remote Zarr stores.
#### Remote Zarr access
Only open remote stores from trusted, expected locations. Prefer allowlisted HTTPS/S3/GCS paths or signed URLs, and avoid asking an agent to fetch arbitrary user-supplied URLs.
```python
import fsspec
# Access Zarr from an expected S3 location
store = fsspec.get_mapper('s3://bucket-name/data.zarr')
adata = ad.read_zarr(store)
# Access Zarr from a trusted HTTPS location
store = fsspec.get_mapper('https://example.com/data.zarr')
adata = ad.read_zarr(store)
```
## Alternative Input Formats
### CSV/TSV
```python
from anndata.io import read_csv
# Read CSV (genes as columns, cells as rows)
adata = read_csv('data.csv')
# Read with custom delimiter
adata = read_csv('data.tsv', delimiter='\t')
# Specify that first column is row names
adata = read_csv('data.csv', first_column_names=True)
```
### Excel
```python
from anndata.io import read_excel
# Read Excel file
adata = read_excel('data.xlsx')
# Read specific sheet
adata = read_excel('data.xlsx', sheet='Sheet1')
```
### Matrix Market (MTX)
Common format for sparse matrices in genomics.
```python
from anndata.io import read_mtx
# Read MTX with associated files
# Requires: matrix.mtx, genes.tsv, barcodes.tsv
adata = read_mtx('matrix.mtx')
# Read with custom gene and barcode files
adata = read_mtx(
'matrix.mtx',
var_names='genes.tsv',
obs_names='barcodes.tsv'
)
# Transpose if needed (MTX often has genes as rows)
adata = adata.T
```
### 10X Genomics formats
10X readers are provided by **scanpy**, not anndata. After loading, the result is a standard `AnnData` object.
```python
import scanpy as sc
# Read 10X h5 format
adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5')
# Read 10X MTX directory
adata = sc.read_10x_mtx('filtered_feature_bc_matrix/')
# Specify genome if multiple present
adata = sc.read_10x_h5('data.h5', genome='GRCh38')
```
### Loom
```python
from anndata.io import read_loom
# Read Loom file
adata = read_loom('data.loom')
# Read with specific observation and variable annotations
adata = read_loom(
'data.loom',
obs_names='CellID',
var_names='Gene'
)
```
### Text files
```python
from anndata.io import read_text
# Read generic text file
adata = read_text('data.txt', delimiter='\t')
# Read with custom parameters
adata = read_text(
'data.txt',
delimiter=',',
first_column_names=True,
dtype='float32'
)
```
### UMI tools
```python
from anndata.io import read_umi_tools
# Read UMI tools format
adata = read_umi_tools('counts.tsv')
```
### HDF5 (generic)
```python
from anndata.io import read_hdf
# Read from HDF5 file (not h5ad format)
adata = read_hdf('data.h5', key='dataset')
```
## Alternative Output Formats
### CSV
```python
# Write to CSV files (creates multiple files)
adata.write_csvs('output_dir/')
# This creates:
# - output_dir/X.csv (expression matrix)
# - output_dir/obs.csv (observation annotations)
# - output_dir/var.csv (variable annotations)
# - output_dir/uns.csv (unstructured annotations, if possible)
# Skip certain components
adata.write_csvs('output_dir/', skip_data=True) # Skip X matrix
```
### Loom
```python
# Write to Loom format
adata.write_loom('output.loom')
```
## Reading Specific Elements
For fine-grained control, read specific elements from an open store:
```python
import h5py
from anndata.io import read_elem
# Read just observation annotations from an h5ad file
with h5py.File('data.h5ad', 'r') as f:
obs = read_elem(f['obs'])
layer = read_elem(f['layers/normalized'])
params = read_elem(f['uns/pca'])
```
## Writing Specific Elements
```python
from anndata.io import write_elem
import h5py
# Write element to existing file
with h5py.File('data.h5ad', 'a') as f:
write_elem(f, 'new_layer', adata.X.copy())
```
## Lazy Operations
For very large datasets, use lazy reading to avoid loading entire datasets. `read_lazy` is experimental and is designed for on-disk or in-cloud AnnData stores, including lazy `obs` and `var` access.
```python
from anndata.experimental import read_lazy
adata = read_lazy('large_data.zarr')
print(adata.obs.head()) # Does not require loading X
```
For element-level control, use `read_elem_lazy` on an open store:
```python
import h5py
from anndata.experimental import read_elem_lazy
# Lazy read from an open store (returns dask-backed array)
with h5py.File('large_data.h5ad', 'r') as f:
X_lazy = read_elem_lazy(f['X'])
subset = X_lazy[:100, :100].compute()
```
## Common I/O Patterns
### Convert between formats
```python
from anndata.io import read_mtx, read_csv
# MTX to H5AD
adata = read_mtx('matrix.mtx').T
adata.write_h5ad('data.h5ad')
# CSV to H5AD
adata = read_csv('data.csv')
adata.write_h5ad('data.h5ad')
# H5AD to Zarr
adata = ad.read_h5ad('data.h5ad')
adata.write_zarr('data.zarr')
```
### Load metadata without data
```python
# Backed mode allows inspecting metadata without loading X
adata = ad.read_h5ad('large_file.h5ad', backed='r')
print(f"Dataset contains {adata.n_obs} observations and {adata.n_vars} variables")
print(adata.obs.columns)
print(adata.var.columns)
# X is not loaded into memory
```
### Update backed data or write a new file
```python
# Open in read-write mode for X updates
adata = ad.read_h5ad('data.h5ad', backed='r+')
# X updates can be persisted in backed mode
adata.X[0, 0] = 0
# Metadata changes are not persisted from backed mode; write a new file instead
adata_memory = adata.to_memory()
adata_memory.obs['new_column'] = values
adata_memory.write_h5ad('data_with_metadata.h5ad')
```
### Download from a trusted URL
Validate remote sources before downloading. Prefer local files or vetted object-store paths over arbitrary URLs.
```python
import anndata as ad
import urllib.request
from urllib.parse import urlparse
url = 'https://example.org/datasets/reference.h5ad'
parsed = urlparse(url)
trusted_hosts = {'example.org'}
if parsed.scheme != 'https' or parsed.netloc not in trusted_hosts:
raise ValueError('Refusing to download from an untrusted host')
urllib.request.urlretrieve(url, 'reference.h5ad')
adata = ad.read_h5ad('reference.h5ad')
```
## Performance Tips
### Reading
- Use `backed='r'` for large files you only need to query
- Use `backed='r+'` only for `X` updates; write a new file for metadata changes
- H5AD format is generally fastest for random access
- Zarr is better for cloud storage and parallel access
- Consider compression for storage, but note it may slow down reading
### Writing
- Use compression for long-term storage: `compression='gzip'` or `compression='lzf'`
- LZF compression is faster but compresses less than GZIP
- For Zarr, tune chunk sizes based on access patterns:
- Larger chunks for sequential reads
- Smaller chunks for random access
- Convert string columns to categorical before writing (smaller files)
### Memory management
```python
# Convert strings to categoricals (reduces file size and memory)
adata.strings_to_categoricals()
adata.write_h5ad('data.h5ad')
# Use sparse matrices for sparse data
from scipy.sparse import csr_matrix
if isinstance(adata.X, np.ndarray):
density = np.count_nonzero(adata.X) / adata.X.size
if density < 0.5: # If more than 50% zeros
adata.X = csr_matrix(adata.X)
```
## Handling Large Datasets
### Strategy 1: Backed mode
```python
# Work with dataset larger than RAM
adata = ad.read_h5ad('100GB_file.h5ad', backed='r')
# Filter based on metadata (fast, no data loading)
filtered = adata[adata.obs['quality_score'] > 0.8]
# Load filtered subset into memory
adata_memory = filtered.to_memory()
```
### Strategy 2: Chunked processing
```python
# Process data in chunks
adata = ad.read_h5ad('large_file.h5ad', backed='r')
chunk_size = 1000
results = []
for i in range(0, adata.n_obs, chunk_size):
chunk = adata[i:i+chunk_size, :].to_memory()
# Process chunk
result = process(chunk)
results.append(result)
```
### Strategy 3: Use AnnCollection
```python
import anndata as ad
from anndata.experimental import AnnCollection
# Create backed objects, then lazily concatenate along observations
adatas = [
ad.read_h5ad(f'dataset_{i}.h5ad', backed='r')
for i in range(10)
]
collection = AnnCollection(
adatas,
join_obs='inner',
join_vars='inner'
)
# Process collection lazily
# Data is loaded only when accessed
```
## Common Issues and Solutions
### Issue: Out of memory when reading
**Solution**: Use backed mode or read in chunks
```python
adata = ad.read_h5ad('file.h5ad', backed='r')
```
### Issue: Slow reading from cloud storage
**Solution**: Use Zarr format with appropriate chunking
```python
adata.write_zarr('data.zarr', chunks=(1000, 1000))
```
### Issue: Large file sizes
**Solution**: Use compression and convert to sparse/categorical
```python
adata.strings_to_categoricals()
from scipy.sparse import csr_matrix
adata.X = csr_matrix(adata.X)
adata.write_h5ad('compressed.h5ad', compression='gzip')
```
### Issue: Cannot modify backed metadata
**Solution**: Load to memory and write a new file. Backed mode only persists updates to `X`.
```python
adata = adata.to_memory()
adata.obs['new_column'] = values
adata.write_h5ad('updated_file.h5ad')
```
skills/anndata/references/manipulation.md
+516
View File
@@ -0,0 +1,516 @@
# Data Manipulation
Operations for transforming, subsetting, and manipulating AnnData objects.
## Subsetting
### By indices
```python
import anndata as ad
import numpy as np
adata = ad.AnnData(X=np.random.rand(1000, 2000))
# Integer indices
subset = adata[0:100, 0:500] # First 100 obs, first 500 vars
# List of indices
obs_indices = [0, 10, 20, 30, 40]
var_indices = [0, 1, 2, 3, 4]
subset = adata[obs_indices, var_indices]
# Single observation or variable
single_obs = adata[0, :]
single_var = adata[:, 0]
```
### By names
```python
import pandas as pd
# Create with named indices
obs_names = [f'cell_{i}' for i in range(1000)]
var_names = [f'gene_{i}' for i in range(2000)]
adata = ad.AnnData(
X=np.random.rand(1000, 2000),
obs=pd.DataFrame(index=obs_names),
var=pd.DataFrame(index=var_names)
)
# Subset by observation names
subset = adata[['cell_0', 'cell_1', 'cell_2'], :]
# Subset by variable names
subset = adata[:, ['gene_0', 'gene_10', 'gene_20']]
# Both axes
subset = adata[['cell_0', 'cell_1'], ['gene_0', 'gene_1']]
```
### By boolean masks
```python
# Create boolean masks
high_count_obs = np.random.rand(1000) > 0.5
high_var_genes = np.random.rand(2000) > 0.7
# Subset using masks
subset = adata[high_count_obs, :]
subset = adata[:, high_var_genes]
subset = adata[high_count_obs, high_var_genes]
```
### By metadata conditions
```python
# Add metadata
adata.obs['cell_type'] = np.random.choice(['A', 'B', 'C'], 1000)
adata.obs['quality_score'] = np.random.rand(1000)
adata.var['highly_variable'] = np.random.rand(2000) > 0.8
# Filter by cell type
t_cells = adata[adata.obs['cell_type'] == 'A']
# Filter by multiple conditions
high_quality_a_cells = adata[
(adata.obs['cell_type'] == 'A') &
(adata.obs['quality_score'] > 0.7)
]
# Filter by variable metadata
hv_genes = adata[:, adata.var['highly_variable']]
# Complex conditions
filtered = adata[
(adata.obs['quality_score'] > 0.5) &
(adata.obs['cell_type'].isin(['A', 'B'])),
adata.var['highly_variable']
]
```
## Transposition
```python
# Transpose AnnData object (swap observations and variables)
adata_T = adata.T
# Shape changes
print(adata.shape) # (1000, 2000)
print(adata_T.shape) # (2000, 1000)
# obs and var are swapped
print(adata.obs.head()) # Observation metadata
print(adata_T.var.head()) # Same data, now as variable metadata
# Useful when data is in opposite orientation
# Common with some file formats where genes are rows
```
## Copying
### Full copy
```python
# Create independent copy
adata_copy = adata.copy()
# Modifications to copy don't affect original
adata_copy.obs['new_column'] = 1
print('new_column' in adata.obs.columns) # False
```
### Shallow copy
```python
# View (doesn't copy data, modifications affect original)
adata_view = adata[0:100, :]
# Check if object is a view
print(adata_view.is_view) # True
# Convert view to independent copy
adata_independent = adata_view.copy()
print(adata_independent.is_view) # False
```
## Renaming
### Rename observations and variables
```python
# Rename all observations
adata.obs_names = [f'new_cell_{i}' for i in range(adata.n_obs)]
# Rename all variables
adata.var_names = [f'new_gene_{i}' for i in range(adata.n_vars)]
# Make names unique (add suffix to duplicates)
adata.obs_names_make_unique()
adata.var_names_make_unique()
```
### Rename categories
```python
# Create categorical column
adata.obs['cell_type'] = pd.Categorical(['A', 'B', 'C'] * 333 + ['A'])
# Rename categories
adata.rename_categories('cell_type', ['Type_A', 'Type_B', 'Type_C'])
# Or using dictionary
adata.rename_categories('cell_type', {
'Type_A': 'T_cell',
'Type_B': 'B_cell',
'Type_C': 'Monocyte'
})
```
## Type Conversions
### Strings to categoricals
```python
# Convert string columns to categorical (more memory efficient)
adata.obs['cell_type'] = ['TypeA', 'TypeB'] * 500
adata.obs['tissue'] = ['brain', 'liver'] * 500
# Convert all string columns to categorical
adata.strings_to_categoricals()
print(adata.obs['cell_type'].dtype) # category
print(adata.obs['tissue'].dtype) # category
```
### Sparse to dense and vice versa
```python
from scipy.sparse import csr_matrix, issparse
# Dense to sparse
if not issparse(adata.X):
adata.X = csr_matrix(adata.X)
# Sparse to dense
if issparse(adata.X):
adata.X = adata.X.toarray()
# Convert layer
adata.layers['normalized'] = csr_matrix(adata.layers['normalized'])
```
## Chunked Operations
Process large datasets in chunks:
```python
# Iterate through data in chunks
chunk_size = 100
for chunk in adata.chunked_X(chunk_size):
# Process chunk
result = process_chunk(chunk)
```
## Extracting Vectors
### Get observation vectors
```python
# Get observation metadata as array
cell_types = adata.obs_vector('cell_type')
# Get gene expression across observations
actb_expression = adata.obs_vector('ACTB') # If ACTB in var_names
```
### Get variable vectors
```python
# Get variable metadata as array
gene_names = adata.var_vector('gene_name')
```
## Adding/Modifying Data
### Add observations
```python
# Create new observations
new_obs = ad.AnnData(X=np.random.rand(100, adata.n_vars))
new_obs.var_names = adata.var_names
# Concatenate with existing
adata_extended = ad.concat([adata, new_obs], axis=0)
```
### Add variables
```python
# Create new variables
new_vars = ad.AnnData(X=np.random.rand(adata.n_obs, 100))
new_vars.obs_names = adata.obs_names
# Concatenate with existing
adata_extended = ad.concat([adata, new_vars], axis=1)
```
### Add metadata columns
```python
# Add observation annotation
adata.obs['new_score'] = np.random.rand(adata.n_obs)
# Add variable annotation
adata.var['new_label'] = ['label'] * adata.n_vars
# Add from external data
external_data = pd.read_csv('metadata.csv', index_col=0)
adata.obs['external_info'] = external_data.loc[adata.obs_names, 'column']
```
### Add layers
```python
# Add new layer
adata.layers['raw_counts'] = np.random.randint(0, 100, adata.shape)
adata.layers['log_transformed'] = np.log1p(adata.X)
# Replace layer
adata.layers['normalized'] = new_normalized_data
```
### Add embeddings
```python
# Add PCA
adata.obsm['X_pca'] = np.random.rand(adata.n_obs, 50)
# Add UMAP
adata.obsm['X_umap'] = np.random.rand(adata.n_obs, 2)
# Add multiple embeddings
adata.obsm['X_tsne'] = np.random.rand(adata.n_obs, 2)
adata.obsm['X_diffmap'] = np.random.rand(adata.n_obs, 10)
```
### Add pairwise relationships
```python
from scipy.sparse import csr_matrix
# Add nearest neighbor graph
n_obs = adata.n_obs
knn_graph = csr_matrix(np.random.rand(n_obs, n_obs) > 0.95)
adata.obsp['connectivities'] = knn_graph
# Add distance matrix
adata.obsp['distances'] = csr_matrix(np.random.rand(n_obs, n_obs))
```
### Add unstructured data
```python
# Add analysis parameters
adata.uns['pca'] = {
'variance': [0.2, 0.15, 0.1],
'variance_ratio': [0.4, 0.3, 0.2],
'params': {'n_comps': 50}
}
# Add color schemes
adata.uns['cell_type_colors'] = ['#FF0000', '#00FF00', '#0000FF']
```
## Removing Data
### Remove observations or variables
```python
# Keep only specific observations
keep_obs = adata.obs['quality_score'] > 0.5
adata = adata[keep_obs, :]
# Remove specific variables
remove_vars = adata.var['low_count']
adata = adata[:, ~remove_vars]
```
### Remove metadata columns
```python
# Remove observation column
adata.obs.drop('unwanted_column', axis=1, inplace=True)
# Remove variable column
adata.var.drop('unwanted_column', axis=1, inplace=True)
```
### Remove layers
```python
# Remove specific layer
del adata.layers['unwanted_layer']
# Remove all layers
adata.layers = {}
```
### Remove embeddings
```python
# Remove specific embedding
del adata.obsm['X_tsne']
# Remove all embeddings
adata.obsm = {}
```
### Remove unstructured data
```python
# Remove specific key
del adata.uns['unwanted_key']
# Remove all unstructured data
adata.uns = {}
```
## Reordering
### Sort observations
```python
# Sort by observation metadata
adata = adata[adata.obs.sort_values('quality_score').index, :]
# Sort by observation names
adata = adata[sorted(adata.obs_names), :]
```
### Sort variables
```python
# Sort by variable metadata
adata = adata[:, adata.var.sort_values('gene_name').index]
# Sort by variable names
adata = adata[:, sorted(adata.var_names)]
```
### Reorder to match external list
```python
# Reorder observations to match external list
desired_order = ['cell_10', 'cell_5', 'cell_20', ...]
adata = adata[desired_order, :]
# Reorder variables
desired_genes = ['TP53', 'ACTB', 'GAPDH', ...]
adata = adata[:, desired_genes]
```
## Data Transformations
### Normalize
```python
# Total count normalization (CPM/TPM-like)
total_counts = adata.X.sum(axis=1)
adata.layers['normalized'] = adata.X / total_counts[:, np.newaxis] * 1e6
# Log transformation
adata.layers['log1p'] = np.log1p(adata.X)
# Z-score normalization
mean = adata.X.mean(axis=0)
std = adata.X.std(axis=0)
adata.layers['scaled'] = (adata.X - mean) / std
```
### Filter
```python
# Filter cells by total counts
total_counts = np.array(adata.X.sum(axis=1)).flatten()
adata.obs['total_counts'] = total_counts
adata = adata[adata.obs['total_counts'] > 1000, :]
# Filter genes by detection rate
detection_rate = (adata.X > 0).sum(axis=0) / adata.n_obs
adata.var['detection_rate'] = np.array(detection_rate).flatten()
adata = adata[:, adata.var['detection_rate'] > 0.01]
```
## Working with Views
Views are lightweight references to subsets of data that don't copy the underlying matrix:
```python
# Create view
view = adata[0:100, 0:500]
print(view.is_view) # True
# Views allow read access
data = view.X
# Modifying view data affects original
# (Be careful!)
# Convert view to independent copy
independent = view.copy()
# Force AnnData to be a copy, not a view
adata = adata.copy()
```
## Merging Metadata
```python
# Merge external metadata
external_metadata = pd.read_csv('additional_metadata.csv', index_col=0)
# Join metadata (inner join on index)
adata.obs = adata.obs.join(external_metadata)
# Left join (keep all adata observations)
adata.obs = adata.obs.merge(
external_metadata,
left_index=True,
right_index=True,
how='left'
)
```
## Common Manipulation Patterns
### Quality control filtering
```python
# Calculate QC metrics
adata.obs['n_genes'] = (adata.X > 0).sum(axis=1)
adata.obs['total_counts'] = adata.X.sum(axis=1)
adata.var['n_cells'] = (adata.X > 0).sum(axis=0)
# Filter low-quality cells
adata = adata[adata.obs['n_genes'] > 200, :]
adata = adata[adata.obs['total_counts'] < 50000, :]
# Filter rarely detected genes
adata = adata[:, adata.var['n_cells'] >= 3]
```
### Select highly variable genes
```python
# Mark highly variable genes
gene_variance = np.var(adata.X, axis=0)
adata.var['variance'] = np.array(gene_variance).flatten()
adata.var['highly_variable'] = adata.var['variance'] > np.percentile(gene_variance, 90)
# Subset to highly variable genes
adata_hvg = adata[:, adata.var['highly_variable']].copy()
```
### Downsample
```python
# Random sampling of observations
np.random.seed(42)
n_sample = 500
sample_indices = np.random.choice(adata.n_obs, n_sample, replace=False)
adata_downsampled = adata[sample_indices, :].copy()
# Stratified sampling by cell type
from sklearn.model_selection import train_test_split
train_idx, test_idx = train_test_split(
range(adata.n_obs),
test_size=0.2,
stratify=adata.obs['cell_type']
)
adata_train = adata[train_idx, :].copy()
adata_test = adata[test_idx, :].copy()
```
### Split train/test
```python
# Random train/test split
np.random.seed(42)
n_obs = adata.n_obs
train_size = int(0.8 * n_obs)
indices = np.random.permutation(n_obs)
train_indices = indices[:train_size]
test_indices = indices[train_size:]
adata_train = adata[train_indices, :].copy()
adata_test = adata[test_indices, :].copy()
```
skills/arbor/SKILL.md
+150
View File
@@ -0,0 +1,150 @@
---
name: arbor
description: Autonomously improve a real artifact (code, training recipe, agent harness, data pipeline, prompt) against an objective and an evaluator, using Hypothesis Tree Refinement (HTR) from the Arbor paper. Use this whenever someone wants to iteratively optimize something over many experiments without overfitting — e.g. "get my model's eval score up", "improve this agent/harness", "tune this pipeline", "beat the baseline on this benchmark", "run a search over approaches and keep the best", "do an MLE-bench / Kaggle-style optimization", or any long-horizon "make this artifact better and don't just memorize the dev set" task. Trigger it even when the user doesn't say "Arbor" or "hypothesis tree" but describes repeated experiment-and-evaluate loops, branching exploration of competing ideas, or worries about a dev/test gap. Runs Claude itself as the coordinator with subagent executors in isolated git worktrees; for the standalone `arbor` CLI tool see references/arbor-upstream.md.
allowed-tools: Read Write Edit Bash Agent
license: MIT license
metadata: {"version": "1.0", "skill-author": "K-Dense Inc."}
---
# Arbor — Autonomous Optimization via Hypothesis Tree Refinement
## Overview
This skill runs an **Autonomous Optimization (AO)** loop: starting from an existing artifact and a measurable objective, improve it through many rounds of experiment and evaluation — without step-by-step human supervision and without overfitting to the feedback signal. It's the right tool when the bottleneck isn't writing one good change, but *organizing dozens of trials* so that lessons accumulate instead of evaporating.
It implements **Hypothesis Tree Refinement (HTR)** from *Arbor* (Jin et al., 2026). The key idea: keep the research state in a persistent **hypothesis tree** rather than in conversation history. Each node binds a hypothesis, the distilled insight it produced, and a pointer to the artifact version that realizes it. You play the long-lived **coordinator** that owns this tree and decides where to search; short-lived **executor** subagents test one hypothesis each in isolated git worktrees and report back. A **held-out merge gate** admits a change only when it improves on a *test* evaluator the search never optimized against. This is what turns trial-and-error into cumulative, auditable research.
Use the `scripts/tree.py` state manager for all the bookkeeping (creating nodes, writing evidence, propagating insights, pruning, the merge gate, the Observe projection). It keeps the state consistent and frees you to spend judgment on what the evidence *means*.
## When to use this skill
Reach for Arbor when the task is **iterative improvement of a concrete artifact under an evaluator**:
- Model training: optimizer/architecture/recipe changes to lower loss or hit a target in fewer steps.
- Harness/agent engineering: raising pass rate or accuracy of an agent loop, search harness, or tool-use scaffold.
- Data synthesis: improving a generation/filtering pipeline judged by downstream model behavior.
- Benchmark optimization: MLE-bench / Kaggle-style "improve the submission" tasks.
- Prompt/system optimization where you can score outputs automatically.
The distinguishing signals: there's an **artifact you can modify**, an **objective**, a way to **score** candidates, and you expect to run **many experiments**. If the user only wants a single fix or a one-shot answer, this is overkill — just do the work directly. If they want open-ended ideation with no evaluator, use `hypothesis-generation` or `scientific-brainstorming` instead.
## The AO setup — pin this down first
Before any experiments, establish the task tuple `(M_0, O, E_dev, E_test)`. Getting this right matters more than any later decision, so confirm it explicitly:
- **M_0 — initial material**: the artifact to improve (a repo, a script, a config, a prompt). Make sure it's under git and currently runs.
- **O — objective**: the natural-language goal and the metric *direction* (maximize accuracy? minimize loss/steps?).
- **E_dev — development evaluator**: a command you can run freely during search to score a candidate. Fast, repeatable.
- **E_test — held-out test evaluator**: a *separate* evaluator (different seeds, different split, or a larger run) used only at the merge gate. It must not be used as a search oracle — that's the whole point.
If the user hasn't given you a clean dev/test split, **construct one and say so**. The dev/test separation is the mechanism that catches overfitting: a candidate that wins on dev but not on test isn't a success, it's a warning that you're exploiting the feedback signal. Without it, autonomous search reliably overfits.
Initialize the run:
```bash
python scripts/tree.py init \
--objective "Improve BrowseComp answer accuracy on the search harness" \
--dev-eval "python eval.py --split dev --n 50" \
--test-eval "python eval.py --split test --n 300" \
--material "." --metric-direction max --branching 3 --max-depth 2 --budget 12
```
`--branching` is how many sibling hypotheses you propose per parent; `--max-depth 2` keeps directions at depth 1 and concrete interventions at depth 2 (the paper's default); `--budget` is the number of coordinator cycles. Start small (10–20 cycles) — structured search beats brute force, and you can extend if progress is still being made.
## The coordinator loop
You run repeated cycles of six steps. This is the heart of HTR; do not collapse it into ad-hoc editing. Run `python scripts/tree.py cycle` once per cycle to track the budget.
### 1. Observe
Begin every cycle by re-grounding in the tree, not in your memory of the conversation:
```bash
python scripts/tree.py observe
```
This prints the objective, global insights, the active frontier (selectable hypotheses), executed nodes with their evidence, pruned lessons (negative constraints), and the current best artifact. Treating the tree as the source of truth is what keeps you coherent over a long run, after context compression has thrown away the details.
### 2. Ideate
Pick a promising parent and propose a few child hypotheses under it. **Condition on the tree's evidence** — this is the difference between Arbor and random search:
- Validated insights are assumptions you can build on.
- Pruned nodes are dead ends to avoid.
- A "half-right" result is a *starting point for a sharper hypothesis*, not a reason to abandon the direction.
Each hypothesis should be a **falsifiable claim about how changing the artifact will move the metric**, not a vague intention. Depth-1 nodes are broad directions ("the search harness loses correct answers it already retrieved"); depth-2 nodes are concrete, executable interventions ("run K=5 independent rollouts and aggregate by evidence dossier instead of majority vote").
```bash
python scripts/tree.py add-node --parent n0 --hypothesis "Verification, not retrieval, is the bottleneck: candidates are found but discarded"
python scripts/tree.py add-node --parent n4 --hypothesis "Decompose the question into atomic constraints and verify each independently"
```
### 3. Select
Choose which pending leaves to run next. **Selection is not pure score-maximization** — pick a hypothesis because it has strong prior evidence, because it would resolve an ambiguity its siblings exposed, or because its failure would clarify an important assumption. Frontier control under delayed feedback rewards informative experiments, not just promising ones.
### 4. Dispatch
Run each selected hypothesis as an **executor subagent in an isolated worktree** (use the Agent tool with `isolation: "worktree"`, or have the executor create one with `git worktree add`). Isolation matters: parallel experiments must not clobber each other or the current best, and exploratory changes stay quarantined until they pass the merge gate.
Dispatch siblings **in parallel** (multiple Agent calls in one message) when they're independent — comparative evidence within one direction is exactly what makes later pruning and abstraction possible.
Give each executor a tight, **hypothesis-bound** brief. See `references/executor-brief.md` for the full template. The contract that makes HTR work: **the executor may not change the hypothesis when the metric stalls.** It repairs its own code and reruns, but `h_n` is fixed — otherwise the returned score is no longer evidence about the assigned node and the tree's semantics break. The executor returns exactly four things:
- **dev_score** — the dev evaluator result (for selection);
- **result** — a factual summary of what happened;
- **insight** — the distilled, reusable lesson (*why* the result supports, weakens, or bounds the hypothesis);
- **branch_ref** — the git branch/commit/worktree path holding the artifact.
Mark a node `running` before dispatch (`tree.py set-status --node n5 --status running`) so the Observe projection stays accurate.
### 5. Backpropagate
When an executor returns, write its report into the node, then **abstract the lesson upward**:
```bash
python scripts/tree.py set-evidence --node n5 --dev-score 70.0 \
--result "K=5 dossier aggregation recovers answers in minority rollouts" \
--insight "Correct answers often appear in a minority of rollouts; aggregation beats majority vote" \
--branch-ref "wt/n5"
python scripts/tree.py propagate --node n5 \
--insight "Candidate coverage, not verification, limits this direction" --to-root
```
This is the step that makes the tree more than a log. A leaf-level observation ("data-interface mismatch") should become a direction-level constraint and, if it generalizes, a global prior that shapes future ideation. **Insight propagation is the component that drives most of HTR's gains** — in the paper's MLE-Bench Lite ablation, a tree *without* insight feedback scored even lower than a flat experiment queue with no tree at all (54.5% vs. 63.6% any-medal, against 81.8% for the full system). Hierarchy alone isn't enough: the semantic memory is what matters. So spend real thought on the abstraction; don't just copy the leaf insight upward verbatim.
### 6. Decide
Decide what to do with the new evidence: keep expanding a direction, prune a falsified subtree, or attempt to merge a candidate.
- **Prune** dead ends, recording *why* — the reason becomes a negative constraint:
```bash
python scripts/tree.py prune --node n7 --reason "search-augmented judge overfits dev questions; no test transfer"
```
- **Merge gate** — promote a candidate to the new best **only if it improves on `E_test`**. Run the test evaluator in a *fresh* worktree (not the dev worktree, to avoid leakage), then:
```bash
python scripts/tree.py merge --node n5 --test-score 67.67 --branch-ref "wt/n5"
```
If the gate rejects it, that's informative: a high-dev / low-test candidate is evidence the direction may be exploiting the dev signal rather than producing a transferable improvement. Record that lesson; don't quietly promote it anyway.
Repeat until the budget is spent, the frontier is exhausted, or progress has clearly stalled.
## Finishing the run
When you stop, produce a short report (see `references/report-template.md`) covering:
- the final best artifact, its test score, and its delta over `M_0`;
- the tree (`python scripts/tree.py status`) as the audit trail of what was tried;
- the main hypothesis shifts — how task understanding deepened across the run (early nodes test broad mechanisms; later nodes find their limits; ancestor insights compress these into the constraints behind the final design);
- merged vs. explored: many nodes improve dev, far fewer pass the test gate — report that gap honestly rather than overstating dev wins.
Always leave `M_best` as a real, runnable artifact on a named branch, and tell the user how to check it out.
## Principles that make this work (not rote rules)
These come from the paper's analysis; understanding *why* matters more than following them mechanically.
- **The tree is the memory; conversation is not.** Over a long horizon your context gets compressed. Re-Observe each cycle so decisions rest on durable evidence, not a lossy summary.
- **Structured search, not more sampling.** Arbor's gains come from how the budget is *organized* — maintaining competing hypotheses, comparing siblings, carrying lessons forward — not from spending more tokens. Don't fan out aimlessly; each experiment should be conditioned on what the tree already knows.
- **Dev guides, test admits.** Use dev feedback freely to steer exploration, but never let a dev win into the final artifact without test confirmation. The dev/test disagreement is itself a signal worth reading.
- **Executors are hypothesis-bound.** Local engineering flexibility (edit, debug, rerun) is fine; silently changing the hypothesis to chase a better number is not — it destroys the meaning of the evidence.
- **Failures are constraints, not noise.** A falsified hypothesis tells you what the solution must avoid. Pruned-with-a-reason is more valuable than pruned-and-forgotten.
## Reference files
- `references/htr-methodology.md` — deeper explanation of HTR, the node structure, the six steps, and the paper's empirical lessons (ablations, transfer, cost). Read when you want the rationale behind a design choice.
- `references/executor-brief.md` — the template for the brief you hand each executor subagent.
- `references/report-template.md` — the final-report structure.
- `references/arbor-upstream.md` — how to install and run the standalone `arbor` CLI from RUC-NLPIR/Arbor instead of orchestrating it natively, and when to prefer each.
skills/arbor/references/arbor-upstream.md
+91
View File
@@ -0,0 +1,91 @@
# Running the standalone Arbor CLI (upstream tool)
This skill normally runs HTR **natively** — Claude is the coordinator and
subagents are executors. That's the recommended path: no extra install, no
separate API keys, and you stay in the loop to read evidence between cycles.
But the paper's authors also ship a full implementation as a CLI. Use it instead
when the user explicitly wants to run *the published system* (e.g. to reproduce
paper results), wants Arbor to run fully unattended for many hours via its own
live dashboard, or wants its built-in report/web-UI tooling.
Source: https://github.com/RUC-NLPIR/Arbor
## Install
Requires Python ≥ 3.10 and Git.
```bash
git clone https://github.com/RUC-NLPIR/Arbor.git
cd Arbor
python -m venv .venv && source .venv/bin/activate
pip install -e .
arbor doctor # verify install, PATH, git, API keys
```
## Configure provider/model/keys
```bash
arbor setup # writes ~/.arbor/config.yaml (provider, model, base URL, keys)
```
Supported backends: Anthropic, OpenAI / OpenAI-compatible Responses API, and
LiteLLM (DeepSeek, Gemini, Qwen, vLLM, Ollama, local gateways). Keys can also be
set via environment variables.
## Run
1. Prepare a benchmark directory: an initial artifact under a **clean git repo**
plus an evaluation script (your `E_dev` / `E_test`).
2. Author a project `research_config.yaml` (task description, coordinator
settings — max cycles, depth, merge thresholds — executor max turns, UI mode).
See `examples/research_config.example.yaml` in the repo.
3. Start the interactive session:
```bash
arbor
```
Arbor runs an intake conversation, forms a Research Contract, then a live
dashboard takes over. Each experiment runs in an isolated git worktree;
verified improvements merge into a per-run trunk.
4. Outputs land in `.arbor/sessions/` with `REPORT.md`, the event log, and
results. Re-render a past session's report with `arbor report <session>`.
## Key CLI commands
| Command | Purpose |
|---|---|
| `arbor` | Start an interactive research session |
| `arbor setup` | Configure provider / model / keys |
| `arbor doctor` | Diagnose install, PATH, git, API keys |
| `arbor report <session>` | Re-render reports for a past session |
| `arbor version` | Print installed version |
## Codebase map (for the curious / for debugging)
The implementation lives under `src/` (a src-layout; the CLI installs as
`arbor`). The package directories are:
- `core/` — ReAct loop, tools, LLM providers, context management
- `coordinator/` — coordinator agent, the tree, orchestrator, coordinator tools
- `executor/` — executor agent and CLI
- `cli/` — intake, live dashboard, setup, doctor, config
- `events/` — typed event bus and payloads
- `report/`, `webui/` — report generation and read-only run monitor
- `search_agent/` — the minimal ReAct search harness (the `M_0` for the
BrowseComp / search-agent tasks)
- `plugins/` — domain plugins (e.g. `mle_kaggle.yaml`)
- `skills/` — on-demand markdown playbooks
(top-level `src/` also has `dashboard.py`, `run.py`, `review.py`.)
**Naming note:** the paper and this skill call the persistent state the
**hypothesis tree**; the tool's code and dashboard call the same structure the
**Idea Tree**. They are the same thing. The depth convention also matches the
native skill: root/depth 0 = objective + global insights, depth 1 = research
directions, depth 2+ = concrete tested methods.
## Native vs. upstream — quick guide
- **Native (this skill)**: best default. Lower setup, transparent, you read and
steer between cycles, reuses your existing Claude Code session and worktrees.
- **Upstream CLI**: choose for paper reproduction, long unattended runs with the
official dashboard, or when the user specifically asks for the `arbor` tool.
skills/arbor/references/executor-brief.md
+68
View File
@@ -0,0 +1,68 @@
# Executor brief template
Each executor is a short-lived subagent that tests **one** hypothesis in an
isolated git worktree and returns structured evidence. Dispatch it with the
Agent tool (use `isolation: "worktree"` so it gets its own copy of the repo, or
instruct it to run `git worktree add` itself). Dispatch independent siblings in
parallel — multiple Agent calls in one message.
Fill in the bracketed parts. Keep the brief tight: the executor needs the
hypothesis, the context that lets it implement well, and a crisp contract for
what to return — nothing more.
---
```
You are an Arbor executor. Test ONE hypothesis in an isolated git worktree and
return structured evidence. Do not change the hypothesis — your job is to give
the coordinator clean evidence about THIS claim, even if it turns out false.
HYPOTHESIS (h_n):
[the falsifiable claim, e.g. "Aggregating K=5 independent rollouts by an
evidence dossier recovers correct answers that majority vote discards."]
CURRENT BEST ARTIFACT (M_best):
[path or git ref of the current best, e.g. branch `arbor/best` — start from this]
RELEVANT INSIGHTS FROM THE TREE (assume these; build on them, don't re-litigate):
[ancestor + sibling insights, e.g. "Verification is not the bottleneck;
candidate coverage is. Search-augmented judging overfits dev questions."]
OBJECTIVE & METRIC:
[O and direction, e.g. "Maximize BrowseComp answer accuracy."]
DEVELOPMENT EVALUATOR (E_dev) — run this to score your candidate:
[exact command, e.g. `python eval.py --split dev --n 50`]
WHAT TO DO:
1. Create/confirm an isolated worktree from M_best so you don't touch other
experiments or the current best.
2. Implement the MINIMAL change that realizes the hypothesis. You may edit,
debug, and rerun freely to get a working implementation — but keep the
change bound to this hypothesis. If the metric stalls, fix YOUR code; do
not pivot to a different idea.
3. Run E_dev and record the score. Run it more than once if it's noisy.
4. Commit the artifact on a clearly named branch.
RETURN EXACTLY THIS (your final message IS the data the coordinator reads):
- dev_score: <number from E_dev>
- result: <1-3 sentences of factual outcome — what the change did>
- insight: <the reusable lesson: WHY this result supports / weakens /
bounds the hypothesis. This is the most valuable output —
make it a constraint future experiments can use, not a restatement
of the score.>
- branch_ref: <git branch/commit/worktree path holding the artifact>
Do NOT run the held-out test evaluator — that is the coordinator's merge gate.
```
---
After the executor returns, the coordinator records it with:
```bash
python scripts/tree.py set-evidence --node <id> \
--dev-score <n> --result "..." --insight "..." --branch-ref "<ref>"
```
then abstracts the lesson upward with `tree.py propagate`.
skills/arbor/references/htr-methodology.md
+177
View File
@@ -0,0 +1,177 @@
# Hypothesis Tree Refinement (HTR) — methodology and evidence
Background reference for the `arbor` skill. Source: *Toward Generalist
Autonomous Research via Hypothesis-Tree Refinement* (Jin et al., 2026,
arXiv:2606.11926; code: github.com/RUC-NLPIR/Arbor). Read this when you want
the reasoning behind a design choice in the main loop.
## The problem: Autonomous Optimization (AO)
AO is the operational core of autonomous research. An agent starts from an
initial artifact and a research objective, then improves the artifact through
experimental feedback **without step-level human supervision**. Formally a task
is a tuple `P = (M_0, O, E_dev, E_test)`:
- `M_0` — mutable initial material (usually a codebase + its data).
- `O` — objective: what "better" means, as a metric direction over the
artifact's output.
- `E_dev` — development evaluator the agent may use freely during search.
- `E_test` — held-out test evaluator. Same objective, different evidence.
The goal is to return `M* = argmax over candidates of S_test(M')`, subject to
the constraint that hypotheses and implementation decisions are made **without
using `E_test` as an exploration oracle**. A candidate that exploits dev-split
idiosyncrasies may raise `S_dev` but is not a successful AO solution unless the
gain also transfers to `S_test`.
Why this is hard: feedback is delayed, experiments are expensive, and failed
attempts contain information that should guide later search. If an agent treats
each trial as an independent local attempt, it loses the structure of the
research process — what was tried, what evidence came back, how each result
reshapes the space of future hypotheses.
## The three design requirements
HTR is built to satisfy three requirements that ordinary agentic tool use does
not:
1. **Branching with coherence.** Multiple competing hypotheses can be plausible
at once, so exploration must branch — but unrestricted branching degenerates
into an unstructured log. The frontier must keep competing directions
organized, comparable, and actionable.
2. **Global strategy with local execution.** Strategic decisions depend on
evidence across the whole run; implementing one hypothesis is short-horizon
code editing. Separate the two so low-level traces don't obscure the global
state, and outcomes stay attributable to the hypotheses that produced them.
3. **Exploration with held-out admission.** Dev feedback guides search;
artifact-level progress is admitted only when it transfers beyond that
feedback. The system must distinguish exploratory dev improvement from
verified test improvement.
## The hypothesis tree as research state
A rooted tree `T = (V, E)`. Each node is a research unit `n = <h_n, iota_n, mu_n>`:
- **Hypothesis `h_n`** — a verifiable/falsifiable claim about how changing the
material improves the objective. Granularity tracks depth: nodes near the root
are broad directions; deeper nodes are concrete interventions an executor can
implement and evaluate. This organizes exploration as progressive refinement
rather than a flat sequence of independent trials.
- **Insight `iota_n`** — the reusable interpretation of evidence. For an
executed leaf: what was tried, what happened, and *why* the result supports,
weakens, or constrains the hypothesis. For an internal node: an abstraction
over its children's insights — the current understanding of that direction.
It is **not** an execution transcript; it is compact semantic memory for later
ideation and selection.
- **Metadata `mu_n`** — connects the semantic hypothesis to executable evidence:
node status, dev score, factual result, implementation reference (git branch
or commit), optional background. The material itself is **not** duplicated in
the tree — only references to external artifact states produced in isolated
worktrees. This keeps the state compact while every hypothesis stays grounded
in a verifiable implementation.
Internal nodes hold abstract directions and accumulated lessons; leaves hold
candidate interventions to dispatch. After a leaf executes, its score, result,
artifact ref, and insight are written back, and the insight is propagated upward
along the path to the root. Through this abstraction, local outcomes become
direction-level lessons and eventually a compact global understanding.
The tree therefore plays three roles at once: a **search frontier** (which
directions are active/validated/pruned), a **long-term memory** (reusable
evidence from successes *and* failures), and an **auditable record** (each
artifact change linked to the hypothesis and evidence that motivated it).
## The coordinator–executor split
- A persistent **coordinator** owns the shared tree and decides where to expand,
which evidence to trust, what to prune, and when to merge. It sees the whole
frontier but does not perform every low-level implementation step.
- Short-lived **executors** are invoked to test one hypothesis each. An executor
gets `h_n`, relevant ancestor insights, and the current best artifact; it
creates an isolated git worktree, implements the minimal change `h_n` requires,
evaluates on `E_dev`, repairs its own broken/inactive code, and returns
structured evidence.
The boundary is the point: exploratory code changes stay isolated until they
pass the merge gate, and the tree records only decision-relevant evidence
(scores, factual outcomes, artifact refs, distilled insights) rather than a raw
log of tool calls. This is how transient execution traces become persistent
research state.
### Executors are hypothesis-bound (and why)
An executor's local loop may involve many edits and reruns, but it stays bound
to the assigned hypothesis: `h_n` is fixed. If an executor were allowed to
change the hypothesis when the metric stalls, the returned score would no longer
be evidence about the assigned node, and ancestor insights built from it would
become impossible to interpret. Keeping executors hypothesis-bound preserves the
semantic meaning of every tree update while still allowing local engineering
flexibility.
## The six-step cycle (Algorithm 1, HTR)
Each coordinator cycle is a controlled mutation of the tree through a narrow
interface:
1. **Observe** — re-ground in a structured projection of the tree (frontier,
root/global insights, ancestor insights, current best). Makes the tree the
authoritative state after context compression, instead of relying on lossy
conversation history.
2. **Ideate** — under a chosen parent, propose `k` child hypotheses, each a
refinement/alternative/correction. Ideation is conditioned on tree evidence:
validated insights are assumptions to build on, pruned nodes are negative
constraints, recent reports suggest what's feasible or under-tested.
3. **Select** — choose pending nodes to execute. Balance expected utility
against the evidence already accumulated around ancestors and siblings. A
node may be selected because it has strong prior evidence, because its
siblings exposed an unresolved ambiguity, or because its failure would
clarify an important assumption. Selection is frontier control under partial,
delayed feedback — not raw score maximization.
4. **Dispatch** — selected hypotheses go to independent executors in fresh
worktrees. Parallel sibling execution yields comparative evidence within one
direction, which feeds later pruning and abstraction.
5. **Backpropagate** — write each executor's evidence into its leaf, then update
insights along the path to the root. The propagated signal is not just a
scalar: it includes causal attributions, applicability conditions, and
reusable lessons. A leaf-level data-interface mismatch can become a
direction-level constraint and then a global prior.
6. **Decide** — continue expanding a direction, prune a falsified subtree, or
attempt a merge. Promotion is guarded by the **held-out merge gate**: the
candidate is evaluated on `E_test` in a fresh worktree and merged into
`M_best` only if it improves under `O`. This separates exploratory success on
`E_dev` from verified artifact-level progress.
## Empirical lessons (use these to prioritize effort)
From the paper's experiments across six AO tasks (model training, harness
engineering, data synthesis) plus MLE-Bench Lite:
- **Insight feedback is the dominant component.** Ablating insight propagation
while *keeping* the tree caused a larger drop than removing the tree entirely
(on MLE-Bench Lite: full 81.82% any-medal vs. 54.54% w/o insight feedback vs.
63.64% w/o tree). Hierarchy alone is not enough — a tree without propagated
lessons organizes experiments syntactically but provides no semantic memory.
**Invest your judgment in the abstraction at Backpropagate**, not just in
generating more hypotheses.
- **Structured search, not a bigger budget.** Arbor used a comparable token
budget to single-trajectory baselines (~20–43M tokens) yet got larger held-out
gains. The win is in how the budget is *organized*: maintaining competing
hypotheses, isolated execution, comparison, and an updated frontier.
- **The dev/test split exposes overfitting.** Across tasks, many nodes improved
dev but only a subset passed the test gate. On Terminal-Bench, the highest-dev
candidate was *not* the best on test. Always report the merged-vs-explored gap
honestly; a high-dev/low-test result is evidence of feedback exploitation.
- **Refinement deepens task understanding.** Early nodes test whether a broad
mechanism holds; later nodes localize where it stops working; ancestor
insights compress these into the constraints the final design must satisfy.
Successful proposals are usually *evidence-conditioned* responses to earlier
failures, not fresh guesses.
- **Lessons transfer.** A harness optimized only on one task's dev feedback
improved unrelated held-out tasks, indicating HTR discovers generally useful
design changes rather than fitting the source benchmark — when, and only when,
the merge gate is enforced.
- **What HTR does *not* fix.** Arbor is strongest at a sequence of concrete
refinements once a runnable solution exists. It is weaker when progress
requires a genuinely new high-level formulation only weakly connected to the
current tree — that still leans on good human task design (the choice of
`M_0`, evaluator, metric, and interface).
skills/arbor/references/report-template.md
+39
View File
@@ -0,0 +1,39 @@
# Final report template
Produce this when the run ends (budget spent, frontier exhausted, or progress
stalled). The point is an honest, auditable account — not a victory lap. Keep it
concise and grounded in the tree.
```markdown
# Arbor run report: [objective]
## Result
- **Best artifact**: [git branch/ref of M_best, e.g. `arbor/best`]
- **Test score**: [S_test of M_best] vs. initial [S_test of M_0] → delta [Δ]
- **How to check it out**: `git checkout [ref]`
- One-line summary of the change that won.
## What was tried (audit trail)
[Paste `python scripts/tree.py status` — the tree shows every direction,
which were pruned, which merged, with dev/test scores.]
## How understanding evolved
2-4 bullets tracing the main hypothesis shifts: which early nodes tested broad
mechanisms, what they confirmed or ruled out, and how that reshaped later
hypotheses. The story should explain *why* the final design looks the way it
does — i.e. the constraints the run discovered.
## Dev vs. test (overfitting check)
- Nodes that improved **dev**: [count]
- Nodes that passed the **test merge gate**: [count]
- Comment on the gap: were there high-dev / low-test candidates? What did
rejecting them tell you? An honest gap here is more trustworthy than a clean
"everything worked".
## Open directions
What you'd explore with more budget, and any direction that seemed to need a
new high-level formulation rather than further refinement (HTR's known weak
spot — flag it for the human).
```
Always leave `M_best` as a real, runnable artifact on a named git branch.
skills/arbor/scripts/tree.py
+564
View File
@@ -0,0 +1,564 @@
#!/usr/bin/env python3
"""
tree.py — persistent hypothesis-tree state manager for Arbor-style
Hypothesis Tree Refinement (HTR).
The hypothesis tree is the durable research state for an Autonomous
Optimization (AO) run. This script owns the *mechanical* parts of that state
— creating nodes, writing back evidence, propagating insights up the tree,
pruning falsified branches, recording the held-out merge gate, and rendering
an "Observe" projection — so the coordinator (you, the model) can spend its
judgment on what the evidence *means* rather than on bookkeeping.
Division of labor:
- This script keeps the state consistent and auditable. It never decides
which hypothesis to try or whether one is good.
- The coordinator reads the projection (`observe`), forms hypotheses,
interprets executor reports, and calls the mutating commands to record
those decisions.
State lives in `.arbor/` under the run directory (default: current dir):
.arbor/tree.json — the hypothesis tree (nodes, edges, evidence, insights)
.arbor/run.json — run-level config: objective, evaluators, budget, M_best
Node fields mirror the paper's research unit n = <h, iota, mu>:
hypothesis (h) — the falsifiable claim this node tests
insight (iota) — distilled, reusable lesson (filled after execution)
metadata (mu) — status, dev_score, test_score, result, branch_ref, depth
Run `python tree.py --help` or `python tree.py <command> --help`.
"""
import argparse
import json
import os
import sys
import time
from pathlib import Path
# ----------------------------------------------------------------------------
# Storage helpers
# ----------------------------------------------------------------------------
VALID_STATUS = {"pending", "running", "executed", "merged", "pruned", "root"}
def _dir(run_dir):
return Path(run_dir) / ".arbor"
def _tree_path(run_dir):
return _dir(run_dir) / "tree.json"
def _run_path(run_dir):
return _dir(run_dir) / "run.json"
def _load(path, what):
if not path.exists():
sys.exit(
f"error: no {what} found at {path}. Run `tree.py init` first "
f"(from the run directory, or pass --run-dir)."
)
with open(path) as f:
return json.load(f)
def _save(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
with open(tmp, "w") as f:
json.dump(data, f, indent=2)
tmp.replace(path)
def _load_tree(run_dir):
return _load(_tree_path(run_dir), "hypothesis tree")
def _load_run(run_dir):
return _load(_run_path(run_dir), "run config")
def _next_id(tree):
n = tree.get("_counter", 0) + 1
tree["_counter"] = n
return f"n{n}"
def _node(tree, node_id):
node = tree["nodes"].get(node_id)
if node is None:
sys.exit(f"error: no node with id '{node_id}'. Run `tree.py status` to list nodes.")
return node
def _children(tree, node_id):
return [nid for nid, n in tree["nodes"].items() if n.get("parent") == node_id]
def _ancestors(tree, node_id):
"""Path from node up to (and including) root, nearest first."""
path = []
cur = tree["nodes"][node_id].get("parent")
while cur is not None:
path.append(cur)
cur = tree["nodes"][cur].get("parent")
return path
def _depth(tree, node_id):
return len(_ancestors(tree, node_id))
def _stamp(node):
node["updated_at"] = int(time.time())
# ----------------------------------------------------------------------------
# Commands
# ----------------------------------------------------------------------------
def cmd_init(args):
d = _dir(args.run_dir)
if _tree_path(args.run_dir).exists() and not args.force:
sys.exit(
f"error: a tree already exists at {_tree_path(args.run_dir)}. "
f"Use --force to overwrite (this erases the current run)."
)
root = {
"id": "n0",
"parent": None,
"depth": 0,
"status": "root",
"hypothesis": args.objective,
"insight": "", # global insights accumulate here via backpropagation
"metadata": {"dev_score": None, "test_score": None, "result": "", "branch_ref": None},
"created_at": int(time.time()),
"updated_at": int(time.time()),
}
tree = {"_counter": 0, "root": "n0", "nodes": {"n0": root}}
run = {
"objective": args.objective,
"metric_direction": args.metric_direction,
"dev_eval": args.dev_eval,
"test_eval": args.test_eval,
"material": args.material,
"branching": args.branching,
"max_depth": args.max_depth,
"budget_cycles": args.budget,
"cycles_used": 0,
"best_node": None, # node id of current M_best
"best_test_score": None, # held-out score of M_best
"best_branch_ref": None, # git ref / path of M_best artifact
"created_at": int(time.time()),
}
_save(_tree_path(args.run_dir), tree)
_save(_run_path(args.run_dir), run)
print(f"Initialized Arbor run in {d}")
print(f" objective : {args.objective}")
print(f" metric direction: {args.metric_direction} (higher-is-better after orienting)")
print(f" dev evaluator : {args.dev_eval}")
print(f" test evaluator : {args.test_eval}")
print(f" budget : {args.budget} cycles, branching {args.branching}, max depth {args.max_depth}")
print("\nNext: `tree.py observe` to read the state, then add direction nodes under n0.")
def cmd_add_node(args):
tree = _load_tree(args.run_dir)
parent = _node(tree, args.parent)
run = _load_run(args.run_dir)
nid = _next_id(tree)
depth = _depth(tree, args.parent) + 1
if depth > run["max_depth"]:
print(
f"warning: node depth {depth} exceeds max_depth {run['max_depth']}. "
f"Deep nodes should be concrete, executable interventions.",
file=sys.stderr,
)
node = {
"id": nid,
"parent": args.parent,
"depth": depth,
"status": "pending",
"hypothesis": args.hypothesis,
"insight": "",
"metadata": {"dev_score": None, "test_score": None, "result": "", "branch_ref": None},
"created_at": int(time.time()),
"updated_at": int(time.time()),
}
tree["nodes"][nid] = node
_save(_tree_path(args.run_dir), tree)
kind = "direction" if depth == 1 else "intervention"
print(f"Added {kind} node {nid} (depth {depth}) under {args.parent}: {args.hypothesis}")
def cmd_set_status(args):
tree = _load_tree(args.run_dir)
node = _node(tree, args.node)
if args.status not in VALID_STATUS:
sys.exit(f"error: status must be one of {sorted(VALID_STATUS)}")
node["status"] = args.status
_stamp(node)
_save(_tree_path(args.run_dir), tree)
print(f"{args.node} -> status={args.status}")
def cmd_set_evidence(args):
"""Write an executor's report back into its node (the Backpropagate step,
leaf part). Insight propagation upward is a separate, deliberate call."""
tree = _load_tree(args.run_dir)
node = _node(tree, args.node)
meta = node["metadata"]
if args.dev_score is not None:
meta["dev_score"] = args.dev_score
if args.result is not None:
meta["result"] = args.result
if args.branch_ref is not None:
meta["branch_ref"] = args.branch_ref
if args.insight is not None:
node["insight"] = args.insight
node["status"] = args.status or "executed"
_stamp(node)
_save(_tree_path(args.run_dir), tree)
print(f"Wrote evidence to {args.node}: dev_score={meta['dev_score']} status={node['status']}")
if node["insight"]:
print(f" insight: {node['insight']}")
anc = _ancestors(tree, args.node)
if anc:
print(
"\nReminder: abstract this leaf insight upward. Decide what direction-level "
f"lesson it implies for ancestors {anc} and record it with "
f"`tree.py propagate --node {args.node} --insight \"...\"` (and update n0 "
"global insights if it generalizes)."
)
def cmd_propagate(args):
"""Backpropagate a distilled, direction-level lesson up the ancestor path.
The coordinator decides the abstracted wording; this appends it to the
chosen ancestor(s) so later ideation is conditioned on it. By default it
updates the immediate parent; --to-root also updates global insights."""
tree = _load_tree(args.run_dir)
_node(tree, args.node) # validate
targets = _ancestors(tree, args.node)
if not targets:
sys.exit("error: node has no ancestors (is it the root?).")
if not args.to_root:
targets = targets[:1] # immediate parent only
for tid in targets:
anc = tree["nodes"][tid]
existing = anc.get("insight", "")
line = f"[from {args.node}] {args.insight}"
anc["insight"] = (existing + "\n" + line).strip() if existing else line
_stamp(anc)
_save(_tree_path(args.run_dir), tree)
print(f"Propagated insight from {args.node} to ancestors {targets}")
def cmd_prune(args):
"""Mark a node (and its subtree) pruned. Pruned hypotheses become negative
constraints — record *why* so future ideation avoids the dead end."""
tree = _load_tree(args.run_dir)
_node(tree, args.node)
stack = [args.node]
pruned = []
while stack:
cur = stack.pop()
node = tree["nodes"][cur]
if node["status"] in ("merged", "root"):
continue
node["status"] = "pruned"
if cur == args.node and args.reason:
node["metadata"]["prune_reason"] = args.reason
_stamp(node)
pruned.append(cur)
stack.extend(_children(tree, cur))
_save(_tree_path(args.run_dir), tree)
print(f"Pruned {pruned}" + (f" — reason: {args.reason}" if args.reason else ""))
def cmd_merge(args):
"""Record a held-out merge gate decision. Only call this AFTER evaluating
the candidate on the TEST evaluator in a fresh worktree. Admitting a
candidate that only improved dev defeats the purpose of the split."""
tree = _load_tree(args.run_dir)
run = _load_run(args.run_dir)
node = _node(tree, args.node)
direction = run["metric_direction"]
prev = run["best_test_score"]
def better(new, old):
if old is None:
return True
return new > old if direction == "max" else new < old
improves = better(args.test_score, prev)
node["metadata"]["test_score"] = args.test_score
if improves:
node["status"] = "merged"
run["best_node"] = args.node
run["best_test_score"] = args.test_score
run["best_branch_ref"] = args.branch_ref or node["metadata"].get("branch_ref")
_stamp(node)
_save(_tree_path(args.run_dir), tree)
_save(_run_path(args.run_dir), run)
print(
f"MERGE GATE PASSED: {args.node} test={args.test_score} "
f"beats previous best={prev}. M_best is now {args.node} "
f"(ref: {run['best_branch_ref']})."
)
else:
_stamp(node)
_save(_tree_path(args.run_dir), tree)
print(
f"MERGE GATE REJECTED: {args.node} test={args.test_score} does not beat "
f"best={prev} (direction={direction}). This is informative, not a failure: "
"a high-dev / low-test gap means the candidate may be exploiting the dev "
"signal. Record that lesson and keep M_best unchanged."
)
def cmd_cycle(args):
"""Increment the cycle counter (call once per Observe->Decide pass)."""
run = _load_run(args.run_dir)
run["cycles_used"] += 1
_save(_run_path(args.run_dir), run)
left = run["budget_cycles"] - run["cycles_used"]
print(f"Cycle {run['cycles_used']}/{run['budget_cycles']} ({left} remaining).")
if left <= 0:
print("Budget exhausted — finish the run: do a final merge-gate check and report.")
# ----------------------------------------------------------------------------
# Read-only projections
# ----------------------------------------------------------------------------
def _fmt_score(node, run):
s = node["metadata"].get("dev_score")
t = node["metadata"].get("test_score")
bits = []
if s is not None:
bits.append(f"dev={s}")
if t is not None:
bits.append(f"test={t}")
return (" [" + " ".join(bits) + "]") if bits else ""
def cmd_observe(args):
"""The Observe step: a compact projection the coordinator re-grounds on at
the start of each cycle, so decisions come from the tree rather than from a
lossy conversation history."""
tree = _load_tree(args.run_dir)
run = _load_run(args.run_dir)
nodes = tree["nodes"]
root = nodes[tree["root"]]
print("=" * 72)
print("OBSERVE — current research state")
print("=" * 72)
print(f"Objective : {run['objective']}")
print(f"Metric direction: {run['metric_direction']}")
print(f"Dev evaluator : {run['dev_eval']}")
print(f"Test evaluator : {run['test_eval']}")
print(
f"Budget : cycle {run['cycles_used']}/{run['budget_cycles']}, "
f"branching {run['branching']}, max depth {run['max_depth']}"
)
print(
f"Current best : "
+ (
f"{run['best_node']} (test={run['best_test_score']}, ref={run['best_branch_ref']})"
if run["best_node"]
else "none yet — M_best is the initial material"
)
)
print("\n-- Global insights (root) --")
print(root["insight"].strip() if root["insight"].strip() else " (none yet)")
# Active frontier = pending/running leaves
frontier = [
n
for n in nodes.values()
if n["status"] in ("pending", "running") and not _children(tree, n["id"])
]
print("\n-- Active frontier (selectable hypotheses) --")
if not frontier:
print(" (empty — ideate new children under a promising node)")
for n in sorted(frontier, key=lambda x: x["id"]):
anc = _ancestors(tree, n["id"])
anc_ins = " | ".join(
nodes[a]["insight"].replace("\n", " ")[:80] for a in anc if nodes[a]["insight"].strip()
)
print(f" {n['id']} (depth {n['depth']}, {n['status']}): {n['hypothesis']}")
if anc_ins:
print(f" ancestor insights: {anc_ins}")
# Validated / executed leaves with evidence
executed = [n for n in nodes.values() if n["status"] in ("executed", "merged")]
print("\n-- Executed / merged nodes (evidence) --")
if not executed:
print(" (none yet)")
for n in sorted(executed, key=lambda x: x["id"]):
print(f" {n['id']} [{n['status']}]{_fmt_score(n, run)}: {n['hypothesis']}")
if n["insight"].strip():
print(f" insight: {n['insight'].splitlines()[0][:120]}")
# Pruned lessons (negative constraints)
pruned = [n for n in nodes.values() if n["status"] == "pruned"]
print("\n-- Pruned lessons (negative constraints — avoid these) --")
if not pruned:
print(" (none yet)")
for n in sorted(pruned, key=lambda x: x["id"]):
reason = n["metadata"].get("prune_reason", "")
print(f" {n['id']}: {n['hypothesis']}" + (f" — {reason}" if reason else ""))
print("\n" + "=" * 72)
print(
"Next: Ideate children under a promising node, Select one or more frontier\n"
"leaves, Dispatch each to an executor subagent in an isolated worktree."
)
def cmd_status(args):
"""ASCII tree render — useful for reports and quick scans."""
tree = _load_tree(args.run_dir)
run = _load_run(args.run_dir)
nodes = tree["nodes"]
symbol = {
"root": "*",
"pending": "o",
"running": "~",
"executed": "=",
"merged": "V",
"pruned": "x",
}
def render(nid, prefix=""):
n = nodes[nid]
sym = symbol.get(n["status"], "?")
best = " <== M_best" if nid == run["best_node"] else ""
print(f"{prefix}[{sym}] {nid} {n['hypothesis'][:70]}{_fmt_score(n, run)}{best}")
kids = sorted(_children(tree, nid))
for i, c in enumerate(kids):
render(c, prefix + " ")
print(f"Run: {run['objective']}")
print(f"Cycle {run['cycles_used']}/{run['budget_cycles']} | legend: * root o pending ~ running = executed V merged x pruned\n")
render(tree["root"])
def cmd_validate(args):
"""Check invariants — catch a corrupted or inconsistent tree early."""
tree = _load_tree(args.run_dir)
run = _load_run(args.run_dir)
nodes = tree["nodes"]
problems = []
if tree["root"] not in nodes:
problems.append("root id not present in nodes")
for nid, n in nodes.items():
p = n.get("parent")
if p is not None and p not in nodes:
problems.append(f"{nid}: parent {p} missing")
if n["status"] not in VALID_STATUS:
problems.append(f"{nid}: invalid status {n['status']}")
if run["best_node"] and run["best_node"] not in nodes:
problems.append(f"best_node {run['best_node']} missing")
merged = [nid for nid, n in nodes.items() if n["status"] == "merged"]
if run["best_node"] and run["best_node"] not in merged and run["best_node"] != tree["root"]:
problems.append(f"best_node {run['best_node']} is not marked merged")
if problems:
print("INVALID:")
for p in problems:
print(f" - {p}")
sys.exit(1)
print(f"OK — {len(nodes)} nodes, root={tree['root']}, best={run['best_node']}")
# ----------------------------------------------------------------------------
# CLI
# ----------------------------------------------------------------------------
def build_parser():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--run-dir", default=".", help="Run directory holding .arbor/ (default: current dir)")
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("init", help="Initialize a new AO run / hypothesis tree")
s.add_argument("--objective", required=True, help="Natural-language research objective (root hypothesis)")
s.add_argument("--dev-eval", required=True, help="Command/description of the development evaluator")
s.add_argument("--test-eval", required=True, help="Command/description of the held-out test evaluator")
s.add_argument("--material", default="", help="Path/ref to the initial artifact M_0")
s.add_argument("--metric-direction", choices=["max", "min"], default="max", help="Is higher or lower the better score?")
s.add_argument("--branching", type=int, default=3, help="Max children proposed per parent (k)")
s.add_argument("--max-depth", type=int, default=2, help="Max tree depth (directions at 1, interventions at 2+)")
s.add_argument("--budget", type=int, default=20, help="Coordinator cycle budget B")
s.add_argument("--force", action="store_true", help="Overwrite an existing run")
s.set_defaults(func=cmd_init)
s = sub.add_parser("observe", help="Print the research-state projection (start of each cycle)")
s.set_defaults(func=cmd_observe)
s = sub.add_parser("add-node", help="Add a pending child hypothesis under a parent (Ideate)")
s.add_argument("--parent", required=True, help="Parent node id (n0 for a new research direction)")
s.add_argument("--hypothesis", required=True, help="Falsifiable claim this node tests")
s.set_defaults(func=cmd_add_node)
s = sub.add_parser("set-status", help="Set a node's status manually")
s.add_argument("--node", required=True)
s.add_argument("--status", required=True, help=f"One of {sorted(VALID_STATUS)}")
s.set_defaults(func=cmd_set_status)
s = sub.add_parser("set-evidence", help="Write an executor report into its node (Backpropagate, leaf)")
s.add_argument("--node", required=True)
s.add_argument("--dev-score", type=float, default=None, help="Dev evaluator score returned by the executor")
s.add_argument("--result", default=None, help="Factual result summary")
s.add_argument("--insight", default=None, help="Distilled, reusable lesson from this experiment")
s.add_argument("--branch-ref", default=None, help="Git branch/commit/worktree path of the artifact")
s.add_argument("--status", default=None, help="Override status (default: executed)")
s.set_defaults(func=cmd_set_evidence)
s = sub.add_parser("propagate", help="Abstract a leaf insight up to ancestors (Backpropagate, upward)")
s.add_argument("--node", required=True, help="The leaf the lesson came from")
s.add_argument("--insight", required=True, help="Direction-level abstraction of the lesson")
s.add_argument("--to-root", action="store_true", help="Also record as a global insight on the root")
s.set_defaults(func=cmd_propagate)
s = sub.add_parser("prune", help="Prune a falsified node and its subtree (Decide)")
s.add_argument("--node", required=True)
s.add_argument("--reason", default="", help="Why this direction is a dead end (becomes a negative constraint)")
s.set_defaults(func=cmd_prune)
s = sub.add_parser("merge", help="Record a held-out merge gate decision (Decide)")
s.add_argument("--node", required=True)
s.add_argument("--test-score", type=float, required=True, help="Score on the TEST evaluator in a fresh worktree")
s.add_argument("--branch-ref", default=None, help="Artifact ref to promote if it passes")
s.set_defaults(func=cmd_merge)
s = sub.add_parser("cycle", help="Increment the coordinator cycle counter")
s.set_defaults(func=cmd_cycle)
s = sub.add_parser("status", help="Render the tree as ASCII (for reports)")
s.set_defaults(func=cmd_status)
s = sub.add_parser("validate", help="Check tree invariants")
s.set_defaults(func=cmd_validate)
return p
def main():
args = build_parser().parse_args()
args.func(args)
if __name__ == "__main__":
main()
skills/arboreto/SKILL.md
+265
View File
@@ -0,0 +1,265 @@
---
name: arboreto
description: Infer gene regulatory networks (GRNs) from gene expression data using scalable algorithms (GRNBoost2, GENIE3). Use when analyzing transcriptomics data (bulk RNA-seq, single-cell RNA-seq) to identify transcription factor-target gene relationships and regulatory interactions. Supports distributed computation for large-scale datasets.
license: BSD-3-Clause license
metadata: {"version": "1.0", "skill-author": "K-Dense Inc."}
---
# Arboreto
## Overview
Arboreto is a Python library from [Aerts Lab](https://github.com/aertslab/arboreto) for inferring gene regulatory networks (GRNs) from gene expression data. It parallelizes tree-based ensemble regression (GRNBoost2, GENIE3) with [Dask](https://distributed.dask.org/) across local cores or remote clusters.
**Core capability**: Identify which transcription factors (TFs) regulate which target genes based on expression patterns across observations (cells, samples, conditions).
**Upstream**: PyPI **0.1.6** (2021-02-09, latest). Docs: [arboreto.readthedocs.io](https://arboreto.readthedocs.io/en/latest/). Primary downstream consumer: [pySCENIC](https://github.com/aertslab/pySCENIC).
## Quick Start
Install arboreto:
```bash
uv pip install arboreto
```
Basic GRN inference:
```python
import pandas as pd
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Load expression data (genes as columns)
expression_matrix = pd.read_csv('expression_data.tsv', sep='\t')
# Infer regulatory network
network = grnboost2(expression_data=expression_matrix)
# Save results (TF, target, importance)
network.to_csv('network.tsv', sep='\t', index=False, header=False)
```
**Critical**: Always use `if __name__ == '__main__':` guard because Dask spawns new processes.
## Core Capabilities
### 1. Basic GRN Inference
For standard GRN inference workflows including:
- Input data preparation (Pandas DataFrame or NumPy array)
- Running inference with GRNBoost2 or GENIE3
- Filtering by transcription factors
- Output format and interpretation
**See**: `references/basic_inference.md`
**Use the ready-to-run script**: `scripts/basic_grn_inference.py` for standard inference tasks:
```bash
python scripts/basic_grn_inference.py expression_data.tsv output_network.tsv --tf-file tfs.txt --seed 777 --limit 5000
```
### 2. Algorithm Selection
Arboreto provides two algorithms:
**GRNBoost2 (Recommended)**:
- Fast gradient boosting-based inference
- Optimized for large datasets (10k+ observations)
- Default choice for most analyses
**GENIE3**:
- Random Forest-based inference
- Original multiple regression approach
- Use for comparison or validation
Quick comparison:
```python
from arboreto.algo import grnboost2, genie3
# Fast, recommended
network_grnboost = grnboost2(expression_data=matrix)
# Classic algorithm
network_genie3 = genie3(expression_data=matrix)
```
**For detailed algorithm comparison, parameters, and selection guidance**: `references/algorithms.md`
### 3. Distributed Computing
Scale inference from local multi-core to cluster environments:
**Local (default)** - Uses all available cores automatically:
```python
network = grnboost2(expression_data=matrix)
```
**Custom local client** - Control resources:
```python
from distributed import LocalCluster, Client
local_cluster = LocalCluster(n_workers=10, memory_limit='8GB')
client = Client(local_cluster)
network = grnboost2(expression_data=matrix, client_or_address=client)
client.close()
local_cluster.close()
```
**Cluster computing** - Connect to remote Dask scheduler:
```python
from distributed import Client
client = Client('tcp://scheduler:8786')
network = grnboost2(expression_data=matrix, client_or_address=client)
```
**For cluster setup, performance optimization, and large-scale workflows**: `references/distributed_computing.md`
## Installation
```bash
uv pip install arboreto
```
Conda (Bioconda):
```bash
conda install -c bioconda arboreto
```
**Dependencies** (from upstream `requirements.txt`): `dask[complete]`, `distributed`, `numpy`, `pandas`, `scikit-learn`, `scipy`
**Input formats**: pandas DataFrame, dense `numpy.ndarray`, or sparse `scipy.sparse.csc_matrix` (rows = observations, columns = genes). For array/matrix inputs, pass `gene_names` explicitly.
## Common Use Cases
### Single-Cell RNA-seq Analysis
```python
import pandas as pd
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Load single-cell expression matrix (cells x genes)
sc_data = pd.read_csv('scrna_counts.tsv', sep='\t')
# Infer cell-type-specific regulatory network
network = grnboost2(expression_data=sc_data, seed=42)
# Filter high-confidence links
high_confidence = network[network['importance'] > 0.5]
high_confidence.to_csv('grn_high_confidence.tsv', sep='\t', index=False)
```
### Bulk RNA-seq with TF Filtering
```python
from arboreto.utils import load_tf_names
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Load data
expression_data = pd.read_csv('rnaseq_tpm.tsv', sep='\t')
tf_names = load_tf_names('human_tfs.txt')
# Infer with TF restriction
network = grnboost2(
expression_data=expression_data,
tf_names=tf_names,
seed=123
)
network.to_csv('tf_target_network.tsv', sep='\t', index=False)
```
### Comparative Analysis (Multiple Conditions)
```python
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Infer networks for different conditions
conditions = ['control', 'treatment_24h', 'treatment_48h']
for condition in conditions:
data = pd.read_csv(f'{condition}_expression.tsv', sep='\t')
network = grnboost2(expression_data=data, seed=42)
network.to_csv(f'{condition}_network.tsv', sep='\t', index=False)
```
## Output Interpretation
Arboreto returns a DataFrame with regulatory links:
| Column | Description |
|--------|-------------|
| `TF` | Transcription factor (regulator) |
| `target` | Target gene |
| `importance` | Regulatory importance score (higher = stronger) |
**Filtering strategy**:
- `limit=N` at inference time (return top N links globally)
- Post-hoc importance threshold (e.g., > 0.5)
- Top links per target via `groupby('target')`
- Statistical significance testing (permutation tests, external tools)
## Integration with pySCENIC
Arboreto powers the GRN inference step in [pySCENIC](https://github.com/aertslab/pySCENIC). pySCENIC 0.11+ passes sparse expression matrices to `grnboost2` / `genie3`; pySCENIC 0.12+ defaults to `arboreto_with_multiprocessing.py` (no Dask) for compatibility — use standalone arboreto when you need Dask scaling.
```python
# Standalone: infer co-expression modules before pySCENIC cisTarget pruning
from arboreto.algo import grnboost2
network = grnboost2(expression_data=expression_df, tf_names=tf_list, limit=5000)
# Downstream: pySCENIC ctx pruning, regulon definition, AUCell (see pySCENIC docs)
```
Convert AnnData to a DataFrame for arboreto directly:
```python
expression_df = adata.to_df() # cells x genes
```
## Reproducibility
Always set a seed for reproducible results:
```python
network = grnboost2(expression_data=matrix, seed=777)
```
Run multiple seeds for robustness analysis:
```python
from distributed import LocalCluster, Client
if __name__ == '__main__':
client = Client(LocalCluster())
seeds = [42, 123, 777]
networks = []
for seed in seeds:
net = grnboost2(expression_data=matrix, client_or_address=client, seed=seed)
networks.append(net)
# Consensus: links recurring across runs (example: mean importance per TF-target pair)
import pandas as pd
combined = pd.concat(networks)
consensus = (
combined.groupby(['TF', 'target'], as_index=False)['importance']
.mean()
.query('importance > 0.5')
)
```
## Troubleshooting
**Memory errors**: Reduce dataset size by filtering low-variance genes or use distributed computing
**Slow performance**: Use GRNBoost2 instead of GENIE3, enable distributed client, filter TF list
**Dask errors**: Ensure `if __name__ == '__main__':` guard is present in scripts (required on Windows/macOS with spawn-based multiprocessing)
**Empty results**: Check data format (genes as columns), verify TF names match column names in the expression matrix
**Sparse data**: Use `scipy.sparse.csc_matrix` and pass matching `gene_names`; supported since arboreto 0.1.6 / pySCENIC 0.11
skills/arboreto/references/algorithms.md
+152
View File
@@ -0,0 +1,152 @@
# GRN Inference Algorithms
Arboreto provides two high-level algorithms for gene regulatory network (GRN) inference, both based on the multiple regression approach.
## Algorithm Overview
Both algorithms follow the same inference strategy:
1. For each target gene in the dataset, train a regression model
2. Identify the most important features (potential regulators) from the model
3. Emit these features as candidate regulators with importance scores
The key difference is **computational efficiency** and the underlying regression method.
## GRNBoost2 (Recommended)
**Purpose**: Fast GRN inference for large-scale datasets using gradient boosting.
### When to Use
- **Large datasets**: Tens of thousands of observations (e.g., single-cell RNA-seq)
- **Time-constrained analysis**: Need faster results than GENIE3
- **Default choice**: GRNBoost2 is the flagship algorithm and recommended for most use cases
### Technical Details
- **Method**: Stochastic gradient boosting with early-stopping regularization
- **Performance**: Significantly faster than GENIE3 on large datasets
- **Output**: Same format as GENIE3 (TF-target-importance triplets)
### Usage
```python
from arboreto.algo import grnboost2
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
seed=42,
limit=5000,
)
```
### Parameters (`grnboost2`)
```python
grnboost2(
expression_data, # DataFrame, ndarray, or scipy.sparse.csc_matrix
gene_names=None, # Required for ndarray/sparse inputs
tf_names='all', # TF list, None/'all' → all genes as regulators
client_or_address='local', # 'local', scheduler address, or Dask Client
early_stop_window_length=25, # Early-stopping window (GRNBoost2 only)
limit=None, # Return top N links globally
seed=None, # Random seed; None = non-deterministic
verbose=False,
)
```
## GENIE3
**Purpose**: Classic Random Forest-based GRN inference, serving as the conceptual blueprint.
### When to Use
- **Smaller datasets**: When dataset size allows for longer computation
- **Comparison studies**: When comparing with published GENIE3 results
- **Validation**: To validate GRNBoost2 results
### Technical Details
- **Method**: Random Forest regression (ExtraTrees available via `diy`)
- **Foundation**: Original multiple regression GRN inference strategy
- **Trade-off**: More computationally expensive but well-established
### Usage
```python
from arboreto.algo import genie3
network = genie3(
expression_data=expression_matrix,
tf_names=tf_names,
seed=42,
)
```
### Parameters (`genie3`)
```python
genie3(
expression_data,
gene_names=None,
tf_names='all',
client_or_address='local',
limit=None,
seed=None,
verbose=False,
)
```
## Algorithm Comparison
| Feature | GRNBoost2 | GENIE3 |
|---------|-----------|--------|
| **Speed** | Fast (optimized for large data) | Slower |
| **Method** | Gradient boosting (GBM) | Random Forest |
| **Best for** | Large-scale data (10k+ observations) | Small-medium datasets |
| **Output format** | Same | Same |
| **Inference strategy** | Multiple regression | Multiple regression |
| **Recommended** | Yes (default choice) | For comparison/validation |
| **Early stopping** | Yes (`early_stop_window_length`) | No |
## Advanced: Custom Regressors with `diy`
For custom scikit-learn regressor settings, use `diy()` (not `grnboost2`/`genie3` kwargs):
```python
from arboreto.algo import diy
from arboreto.core import SGBM_KWARGS, RF_KWARGS
# Custom GRNBoost2-style run
custom_gbm = diy(
expression_data=expression_matrix,
regressor_type='GBM', # 'RF', 'GBM', or 'ET'
regressor_kwargs={
**SGBM_KWARGS,
'n_estimators': 100,
'max_depth': 5,
'learning_rate': 0.1,
},
tf_names=tf_names,
seed=42,
)
# Custom GENIE3-style run
custom_rf = diy(
expression_data=expression_matrix,
regressor_type='RF',
regressor_kwargs={
**RF_KWARGS,
'n_estimators': 1000,
'max_features': 'sqrt',
},
tf_names=tf_names,
)
```
Import default kwargs from `arboreto.core` and override only the keys you need.
## Choosing the Right Algorithm
**Decision guide**:
1. **Start with GRNBoost2** — faster and better suited to large single-cell datasets
2. **Use GENIE3 if**:
- Comparing with existing GENIE3 publications
- Dataset is small-medium sized
- Validating GRNBoost2 results
3. **Use `diy()` if** you need non-default regressor hyperparameters
Both algorithms produce comparable regulatory networks with the same output format.
skills/arboreto/references/basic_inference.md
+181
View File
@@ -0,0 +1,181 @@
# Basic GRN Inference with Arboreto
## Input Data Requirements
Arboreto requires gene expression data in one of two formats:
### Pandas DataFrame (Recommended)
- **Rows**: Observations (cells, samples, conditions)
- **Columns**: Genes (with gene names as column headers)
- **Format**: Numeric expression values
Example:
```python
import pandas as pd
# Load expression matrix with genes as columns
expression_matrix = pd.read_csv('expression_data.tsv', sep='\t')
# Columns: ['gene1', 'gene2', 'gene3', ...]
# Rows: observation data
```
### NumPy Array
- **Shape**: (observations, genes)
- **Requirement**: Separately provide gene names list matching column order
Example:
```python
import numpy as np
expression_matrix = np.genfromtxt('expression_data.tsv', delimiter='\t', skip_header=1)
with open('expression_data.tsv') as f:
gene_names = [gene.strip() for gene in f.readline().split('\t')]
assert expression_matrix.shape[1] == len(gene_names)
```
### Sparse CSC Matrix (arboreto 0.1.6+)
- **Format**: `scipy.sparse.csc_matrix` with shape (observations, genes)
- **Requirement**: Provide `gene_names` matching column order (same as NumPy)
- **Use case**: Large single-cell matrices; also used by pySCENIC 0.11+ when `--sparse` is enabled
Example:
```python
import scipy.sparse as sp
from arboreto.algo import grnboost2
# expression_sparse: csc_matrix, cells x genes
network = grnboost2(
expression_data=expression_sparse,
gene_names=gene_names,
tf_names=tf_names,
)
```
## Transcription Factors (TFs)
Optionally provide a list of transcription factor names to restrict regulatory inference:
```python
from arboreto.utils import load_tf_names
# Load from file (one TF per line)
tf_names = load_tf_names('transcription_factors.txt')
# Or define directly
tf_names = ['TF1', 'TF2', 'TF3']
```
If `tf_names` is `None` or `'all'`, all `gene_names` are treated as potential regulators.
## Basic Inference Workflow
### Using Pandas DataFrame
```python
import pandas as pd
from arboreto.utils import load_tf_names
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Load expression data
expression_matrix = pd.read_csv('expression_data.tsv', sep='\t')
# Load transcription factors (optional)
tf_names = load_tf_names('tf_list.txt')
# Run GRN inference
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names # Optional
)
# Save results
network.to_csv('network_output.tsv', sep='\t', index=False, header=False)
```
**Critical**: The `if __name__ == '__main__':` guard is required because Dask spawns new processes internally.
### Using NumPy Array
```python
import numpy as np
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Load expression matrix
expression_matrix = np.genfromtxt('expression_data.tsv', delimiter='\t', skip_header=1)
# Extract gene names from header
with open('expression_data.tsv') as f:
gene_names = [gene.strip() for gene in f.readline().split('\t')]
# Verify dimensions match
assert expression_matrix.shape[1] == len(gene_names)
# Run inference with explicit gene names
network = grnboost2(
expression_data=expression_matrix,
gene_names=gene_names,
tf_names=tf_names
)
network.to_csv('network_output.tsv', sep='\t', index=False, header=False)
```
## Output Format
Arboreto returns a Pandas DataFrame with three columns:
| Column | Description |
|--------|-------------|
| `TF` | Transcription factor (regulator) gene name |
| `target` | Target gene name |
| `importance` | Regulatory importance score (higher = stronger regulation) |
Example output:
```
TF1 gene5 0.856
TF2 gene12 0.743
TF1 gene8 0.621
```
## Setting Random Seed
For reproducible results, pass an explicit `seed` (`None` uses random seeds per regressor):
```python
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
seed=777
)
```
## Limiting Output Size
Return only the top N regulatory links globally:
```python
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
limit=5000,
)
```
## Algorithm Selection
Use `grnboost2()` for most cases (faster, handles large datasets):
```python
from arboreto.algo import grnboost2
network = grnboost2(expression_data=expression_matrix)
```
Use `genie3()` for comparison or specific requirements:
```python
from arboreto.algo import genie3
network = genie3(expression_data=expression_matrix)
```
See `references/algorithms.md` for detailed algorithm comparison.
skills/arboreto/references/distributed_computing.md
+242
View File
@@ -0,0 +1,242 @@
# Distributed Computing with Arboreto
Arboreto leverages Dask for parallelized computation, enabling efficient GRN inference from single-machine multi-core processing to multi-node cluster environments.
## Computation Architecture
GRN inference is inherently parallelizable:
- Each target gene's regression model can be trained independently
- Arboreto represents computation as a Dask task graph
- Tasks are distributed across available computational resources
## Local Multi-Core Processing (Default)
By default, arboreto uses all available CPU cores on the local machine:
```python
from arboreto.algo import grnboost2
# Automatically uses all local cores
network = grnboost2(expression_data=expression_matrix, tf_names=tf_names)
```
This is sufficient for most use cases and requires no additional configuration.
## Custom Local Dask Client
For fine-grained control over local resources, create a custom Dask client:
```python
from distributed import LocalCluster, Client
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Configure local cluster
local_cluster = LocalCluster(
n_workers=10, # Number of worker processes
threads_per_worker=1, # Threads per worker
memory_limit='8GB' # Memory limit per worker
)
# Create client
custom_client = Client(local_cluster)
# Run inference with custom client
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
client_or_address=custom_client
)
# Clean up
custom_client.close()
local_cluster.close()
```
### Benefits of Custom Client
- **Resource control**: Limit CPU and memory usage
- **Multiple runs**: Reuse same client for different parameter sets
- **Monitoring**: Access Dask dashboard for performance insights
## Multiple Inference Runs with Same Client
Reuse a single Dask client for multiple inference runs with different parameters:
```python
from distributed import LocalCluster, Client
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Initialize client once
local_cluster = LocalCluster(n_workers=8, threads_per_worker=1)
client = Client(local_cluster)
# Run multiple inferences
network_seed1 = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
client_or_address=client,
seed=666
)
network_seed2 = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
client_or_address=client,
seed=777
)
# Different algorithms with same client
from arboreto.algo import genie3
network_genie3 = genie3(
expression_data=expression_matrix,
tf_names=tf_names,
client_or_address=client
)
# Clean up once
client.close()
local_cluster.close()
```
## Distributed Cluster Computing
For very large datasets, connect to a remote Dask distributed scheduler running on a cluster:
### Step 1: Set Up Dask Scheduler (on cluster head node)
```bash
dask-scheduler
# Output: Scheduler at tcp://10.118.224.134:8786
```
### Step 2: Start Dask Workers (on cluster compute nodes)
```bash
dask-worker tcp://10.118.224.134:8786
```
### Step 3: Connect from Client
```python
from distributed import Client
from arboreto.algo import grnboost2
if __name__ == '__main__':
# Connect to remote scheduler
scheduler_address = 'tcp://10.118.224.134:8786'
cluster_client = Client(scheduler_address)
# Run inference on cluster
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
client_or_address=cluster_client
)
cluster_client.close()
```
### Cluster Configuration Best Practices
**Worker configuration**:
```bash
dask-worker tcp://scheduler:8786 \
--nprocs 4 \ # Number of processes per node
--nthreads 1 \ # Threads per process
--memory-limit 16GB # Memory per process
```
**For large-scale inference**:
- Use more workers with moderate memory rather than fewer workers with large memory
- Set `threads_per_worker=1` to avoid GIL contention in scikit-learn
- Monitor memory usage to prevent workers from being killed
## Monitoring and Debugging
### Dask Dashboard
Access the Dask dashboard for real-time monitoring:
```python
from distributed import Client
client = Client() # Prints dashboard URL
# Dashboard available at: http://localhost:8787/status
```
The dashboard shows:
- **Task progress**: Number of tasks completed/pending
- **Resource usage**: CPU, memory per worker
- **Task stream**: Real-time visualization of computation
- **Performance**: Bottleneck identification
### Verbose Output
Enable verbose logging to track inference progress:
```python
network = grnboost2(
expression_data=expression_matrix,
tf_names=tf_names,
verbose=True
)
```
## Performance Optimization Tips
### 1. Data Format
- **Use Pandas DataFrame when possible**: More efficient than NumPy for Dask operations
- **Reduce data size**: Filter low-variance genes before inference
### 2. Worker Configuration
- **CPU-bound tasks**: Set `threads_per_worker=1`, increase `n_workers`
- **Memory-bound tasks**: Increase `memory_limit` per worker
### 3. Cluster Setup
- **Network**: Ensure high-bandwidth, low-latency network between nodes
- **Storage**: Use shared filesystem or object storage for large datasets
- **Scheduling**: Allocate dedicated nodes to avoid resource contention
### 4. Transcription Factor Filtering
- **Limit TF list**: Providing specific TF names reduces computation
```python
# Full search (slow)
network = grnboost2(expression_data=matrix)
# Filtered search (faster)
network = grnboost2(expression_data=matrix, tf_names=known_tfs)
```
## Example: Large-Scale Single-Cell Analysis
Complete workflow for processing single-cell RNA-seq data on a cluster:
```python
from distributed import Client
from arboreto.algo import grnboost2
import pandas as pd
if __name__ == '__main__':
# Connect to cluster
client = Client('tcp://cluster-scheduler:8786')
# Load large single-cell dataset (50,000 cells x 20,000 genes)
expression_data = pd.read_csv('scrnaseq_data.tsv', sep='\t')
# Load cell-type-specific TFs
tf_names = pd.read_csv('tf_list.txt', header=None)[0].tolist()
# Run distributed inference
network = grnboost2(
expression_data=expression_data,
tf_names=tf_names,
client_or_address=client,
verbose=True,
seed=42
)
# Save results
network.to_csv('grn_results.tsv', sep='\t', index=False)
client.close()
```
This approach enables analysis of datasets that would be impractical on a single machine.
skills/arboreto/scripts/basic_grn_inference.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""
Basic GRN inference example using Arboreto.
This script demonstrates the standard workflow for inferring gene regulatory
networks from expression data using GRNBoost2.
Usage:
python basic_grn_inference.py <expression_file> <output_file> [--tf-file TF_FILE] [--seed SEED] [--limit LIMIT]
Arguments:
expression_file: Path to expression matrix (TSV format, genes as columns)
output_file: Path for output network (TSV format)
--tf-file: Optional path to transcription factors file (one per line)
--seed: Random seed for reproducibility (default: 777)
--limit: Return only the top N regulatory links (optional)
"""
import argparse
import pandas as pd
from arboreto.algo import grnboost2
from arboreto.utils import load_tf_names
def run_grn_inference(expression_file, output_file, tf_file=None, seed=777, limit=None):
"""
Run GRN inference using GRNBoost2.
Args:
expression_file: Path to expression matrix TSV file
output_file: Path for output network file
tf_file: Optional path to TF names file
seed: Random seed for reproducibility
limit: Optional cap on number of regulatory links returned
"""
print(f"Loading expression data from {expression_file}...")
expression_data = pd.read_csv(expression_file, sep='\t')
print(f"Expression matrix shape: {expression_data.shape}")
print(f"Number of genes: {expression_data.shape[1]}")
print(f"Number of observations: {expression_data.shape[0]}")
# Load TF names if provided
tf_names = 'all'
if tf_file:
print(f"Loading transcription factors from {tf_file}...")
tf_names = load_tf_names(tf_file)
print(f"Number of TFs: {len(tf_names)}")
# Run GRN inference
print(f"Running GRNBoost2 with seed={seed}...")
network = grnboost2(
expression_data=expression_data,
tf_names=tf_names,
seed=seed,
limit=limit,
verbose=True
)
# Save results
print(f"Saving network to {output_file}...")
network.to_csv(output_file, sep='\t', index=False, header=False)
print(f"Done! Network contains {len(network)} regulatory links.")
print(f"\nTop 10 regulatory links:")
print(network.head(10).to_string(index=False))
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Infer gene regulatory network using GRNBoost2'
)
parser.add_argument(
'expression_file',
help='Path to expression matrix (TSV format, genes as columns)'
)
parser.add_argument(
'output_file',
help='Path for output network (TSV format)'
)
parser.add_argument(
'--tf-file',
help='Path to transcription factors file (one per line)',
default=None
)
parser.add_argument(
'--seed',
help='Random seed for reproducibility (default: 777)',
type=int,
default=777
)
parser.add_argument(
'--limit',
help='Return only the top N regulatory links',
type=int,
default=None
)
args = parser.parse_args()
run_grn_inference(
expression_file=args.expression_file,
output_file=args.output_file,
tf_file=args.tf_file,
seed=args.seed,
limit=args.limit
)
skills/astropy/SKILL.md
+351
View File
@@ -0,0 +1,351 @@
---
name: astropy
description: Core Python library for astronomy and astrophysics workflows that need Astropy APIs, including units/quantities, coordinates, FITS I/O, tables, time systems, WCS, and cosmology. Use when implementing or debugging astronomical data analysis code with Astropy.
license: BSD-3-Clause license
compatibility: Requires Python 3.11+ with astropy installed (uv for package installation). Some features (object name resolution, site lookups, remote FITS reads, IERS updates) need network access.
metadata: {"version": "1.2", "skill-author": "K-Dense Inc."}
---
# Astropy
## Overview
Astropy is the core Python package for astronomy, providing essential functionality for astronomical research and data analysis. Use astropy for coordinate transformations, unit and quantity calculations, FITS file operations, cosmological calculations, precise time handling, tabular data manipulation, and astronomical image processing.
## When to Use This Skill
Use astropy when tasks involve:
- Converting between celestial coordinate systems (ICRS, Galactic, FK5, AltAz, etc.)
- Working with physical units and quantities (converting Jy to mJy, parsecs to km, etc.)
- Reading, writing, or manipulating FITS files (images or tables)
- Cosmological calculations (luminosity distance, lookback time, Hubble parameter)
- Precise time handling with different time scales (UTC, TAI, TT, TDB) and formats (JD, MJD, ISO)
- Table operations (reading catalogs, cross-matching, filtering, joining)
- WCS transformations between pixel and world coordinates
- Astronomical constants and calculations
## Quick Start
```python
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.time import Time
from astropy.io import fits
from astropy.table import Table
from astropy.cosmology import Planck18
# Units and quantities
distance = 100 * u.pc
distance_km = distance.to(u.km)
# Coordinates
coord = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree, frame='icrs')
coord_galactic = coord.galactic
# Time
t = Time('2023-01-15 12:30:00')
jd = t.jd # Julian Date
# FITS files
data = fits.getdata('image.fits')
header = fits.getheader('image.fits')
# Tables
table = Table.read('catalog.fits')
# Cosmology
d_L = Planck18.luminosity_distance(z=1.0)
```
## Core Capabilities
### 1. Units and Quantities (`astropy.units`)
Handle physical quantities with units, perform unit conversions, and ensure dimensional consistency in calculations.
**Key operations:**
- Create quantities by multiplying values with units
- Convert between units using `.to()` method
- Perform arithmetic with automatic unit handling
- Use equivalencies for domain-specific conversions (spectral, doppler, parallax)
- Work with logarithmic units (magnitudes, decibels)
**See:** `references/units.md` for comprehensive documentation, unit systems, equivalencies, performance optimization, and unit arithmetic.
### 2. Coordinate Systems (`astropy.coordinates`)
Represent celestial positions and transform between different coordinate frames.
**Key operations:**
- Create coordinates with `SkyCoord` in any frame (ICRS, Galactic, FK5, AltAz, etc.)
- Transform between coordinate systems
- Calculate angular separations and position angles
- Match coordinates to catalogs
- Include distance for 3D coordinate operations
- Handle proper motions and radial velocities
- Query named objects from online databases
**See:** `references/coordinates.md` for detailed coordinate frame descriptions, transformations, observer-dependent frames (AltAz), catalog matching, and performance tips.
### 3. Cosmological Calculations (`astropy.cosmology`)
Perform cosmological calculations using standard cosmological models.
**Key operations:**
- Use built-in cosmologies (Planck18, WMAP9, etc.)
- Create custom cosmological models
- Calculate distances (luminosity, comoving, angular diameter)
- Compute ages and lookback times
- Determine Hubble parameter at any redshift
- Calculate density parameters and volumes
- Perform inverse calculations (find z for given distance)
**See:** `references/cosmology.md` for available models, distance calculations, time calculations, density parameters, and neutrino effects.
### 4. FITS File Handling (`astropy.io.fits`)
Read, write, and manipulate FITS (Flexible Image Transport System) files.
**Key operations:**
- Open FITS files with context managers
- Access HDUs (Header Data Units) by index or name
- Read and modify headers (keywords, comments, history)
- Work with image data (NumPy arrays)
- Handle table data (binary and ASCII tables)
- Create new FITS files (single or multi-extension)
- Use memory mapping for large files
- Access remote FITS files (S3, HTTP)
**See:** `references/fits.md` for comprehensive file operations, header manipulation, image and table handling, multi-extension files, and performance considerations.
### 5. Table Operations (`astropy.table`)
Work with tabular data with support for units, metadata, and various file formats.
**Key operations:**
- Create tables from arrays, lists, or dictionaries
- Read/write tables in multiple formats (FITS, CSV, HDF5, VOTable)
- Access and modify columns and rows
- Sort, filter, and index tables
- Perform database-style operations (join, group, aggregate)
- Stack and concatenate tables
- Work with unit-aware columns (QTable)
- Handle missing data with masking
**See:** `references/tables.md` for table creation, I/O operations, data manipulation, sorting, filtering, joins, grouping, and performance tips.
### 6. Time Handling (`astropy.time`)
Precise time representation and conversion between time scales and formats.
**Key operations:**
- Create Time objects in various formats (ISO, JD, MJD, Unix, etc.)
- Convert between time scales (UTC, TAI, TT, TDB, etc.)
- Perform time arithmetic with TimeDelta
- Calculate sidereal time for observers
- Compute light travel time corrections (barycentric, heliocentric)
- Work with time arrays efficiently
- Handle masked (missing) times
**See:** `references/time.md` for time formats, time scales, conversions, arithmetic, observing features, and precision handling.
### 7. World Coordinate System (`astropy.wcs`)
Transform between pixel coordinates in images and world coordinates.
**Key operations:**
- Read WCS from FITS headers
- Convert pixel coordinates to world coordinates (and vice versa)
- Calculate image footprints
- Access WCS parameters (reference pixel, projection, scale)
- Create custom WCS objects
**See:** `references/wcs_and_other_modules.md` for WCS operations and transformations.
## Additional Capabilities
The `references/wcs_and_other_modules.md` file also covers:
### NDData and CCDData
Containers for n-dimensional datasets with metadata, uncertainty, masking, and WCS information.
### Modeling
Framework for creating and fitting mathematical models to astronomical data.
### Visualization
Tools for astronomical image display with appropriate stretching and scaling.
### Constants
Physical and astronomical constants with proper units (speed of light, solar mass, Planck constant, etc.).
### Convolution
Image processing kernels for smoothing and filtering.
### Statistics
Robust statistical functions including sigma clipping and outlier rejection.
## Installation
```bash
# Reproducible install against the current stable release
uv pip install "astropy==7.2.0"
# Recommended optional dependencies for plotting and common workflows
uv pip install "astropy[recommended]==7.2.0"
# Full optional dependency set for broad astronomy workflows
uv pip install "astropy[all]==7.2.0"
```
Astropy 7.2.0 requires Python 3.11+ and depends on NumPy, PyERFA, PyYAML, and packaging. Use an isolated virtual environment; do not install Astropy with elevated privileges.
Note that the `[recommended]` and `[all]` extras pull in transitive dependencies (matplotlib, scipy, etc.) at unpinned versions. For reproducible production environments, pin the full dependency tree with a lockfile (`uv lock` in a project, or `uv pip compile` for requirements files) and review the resolved versions before deploying.
## Common Workflows
### Converting Coordinates Between Systems
```python
from astropy.coordinates import SkyCoord
import astropy.units as u
# Create coordinate
c = SkyCoord(ra='05h23m34.5s', dec='-69d45m22s', frame='icrs')
# Transform to galactic
c_gal = c.galactic
print(f"l={c_gal.l.deg}, b={c_gal.b.deg}")
# Transform to alt-az (requires time and location)
from astropy.time import Time
from astropy.coordinates import EarthLocation, AltAz
observing_time = Time('2023-06-15 23:00:00')
observing_location = EarthLocation(lat=40*u.deg, lon=-120*u.deg)
aa_frame = AltAz(obstime=observing_time, location=observing_location)
c_altaz = c.transform_to(aa_frame)
print(f"Alt={c_altaz.alt.deg}, Az={c_altaz.az.deg}")
```
### Reading and Analyzing FITS Files
```python
from astropy.io import fits
import numpy as np
# Open FITS file
with fits.open('observation.fits') as hdul:
# Display structure
hdul.info()
# Get image data and header
data = hdul[1].data
header = hdul[1].header
# Access header values
exptime = header['EXPTIME']
filter_name = header['FILTER']
# Analyze data
mean = np.mean(data)
median = np.median(data)
print(f"Mean: {mean}, Median: {median}")
```
### Cosmological Distance Calculations
```python
from astropy.cosmology import Planck18
import astropy.units as u
import numpy as np
# Calculate distances at z=1.5
z = 1.5
d_L = Planck18.luminosity_distance(z)
d_A = Planck18.angular_diameter_distance(z)
print(f"Luminosity distance: {d_L}")
print(f"Angular diameter distance: {d_A}")
# Age of universe at that redshift
age = Planck18.age(z)
print(f"Age at z={z}: {age.to(u.Gyr)}")
# Lookback time
t_lookback = Planck18.lookback_time(z)
print(f"Lookback time: {t_lookback.to(u.Gyr)}")
```
### Cross-Matching Catalogs
```python
from astropy.table import Table
from astropy.coordinates import SkyCoord, match_coordinates_sky
import astropy.units as u
# Read catalogs
cat1 = Table.read('catalog1.fits')
cat2 = Table.read('catalog2.fits')
# Create coordinate objects
coords1 = SkyCoord(ra=cat1['RA']*u.degree, dec=cat1['DEC']*u.degree)
coords2 = SkyCoord(ra=cat2['RA']*u.degree, dec=cat2['DEC']*u.degree)
# Find matches
idx, sep, _ = coords1.match_to_catalog_sky(coords2)
# Filter by separation threshold
max_sep = 1 * u.arcsec
matches = sep < max_sep
# Create matched catalogs
cat1_matched = cat1[matches]
cat2_matched = cat2[idx[matches]]
print(f"Found {len(cat1_matched)} matches")
```
## Best Practices
1. **Always use units**: Attach units to quantities to avoid errors and ensure dimensional consistency
2. **Use context managers for FITS files**: Ensures proper file closing
3. **Prefer arrays over loops**: Process multiple coordinates/times as arrays for better performance
4. **Check coordinate frames**: Verify the frame before transformations
5. **Use appropriate cosmology**: Choose the right cosmological model for your analysis
6. **Handle missing data**: Use masked columns for tables with missing values
7. **Specify time scales**: Be explicit about time scales (UTC, TT, TDB) for precise timing
8. **Use QTable for unit-aware tables**: When table columns have units
9. **Check WCS validity**: Verify WCS before using transformations
10. **Cache frequently used values**: Expensive calculations (e.g., cosmological distances) can be cached
11. **Be explicit about network access**: `SkyCoord.from_name()`, `EarthLocation.of_site(refresh_cache=True)`, `EarthLocation.of_address()`, `download_file()`, remote FITS reads, and some IERS time/coordinate transforms can contact external services or update local caches. Avoid sending sensitive target names, addresses, URLs, or proprietary file locations to third-party services. When working with potentially sensitive targets or data locations, confirm with the user before making these network calls.
12. **Pin for reproducibility**: Use pinned versions such as `astropy==7.2.0` for shared environments; update pins intentionally after reviewing release notes.
## Current-Version Notes
- Current stable release researched: Astropy 7.2.0 (released 2025-11-25; verified current as of 2026-06-10)
- Python requirement: 3.11+
- **Astropy 8.0 is at release-candidate stage** (8.0.0rc1, 2026-05-26). Key changes to anticipate:
- The deprecated `astropy.cosmology` submodule shims (`astropy.cosmology.flrw`, `.core`, `.funcs`, `.connect`, `.parameter`) are removed — import everything directly from `astropy.cosmology` (e.g., `from astropy.cosmology import FlatLambdaCDM, z_at_value`)
- `astropy.constants` defaults change from CODATA 2018 to CODATA 2022; pin a constants version via the `astropyconst` science states if reproducibility matters
- NumPy 2.0 becomes the minimum supported version; the 7.2.x LTS branch retains NumPy 1.x support for six months after the 8.0 release
- The built-in test runner (`astropy.test()`, `TestRunner`) is formally deprecated — invoke `pytest` directly
- Recent 7.x deprecations to avoid in new code: passing a table index identifier as the first `.loc` element (`t.loc["b", 2]`) — use `t.loc.with_index("b")[2]` instead (removal planned for 9.0); `astropy.utils.isiterable()` — use `numpy.iterable()`
- Recent 7.0 removals: older deprecated FITS APIs such as `(Bin)Table.update`, `_ExtensionHDU`, `_NonstandardExtHDU`, and the `tile_size` argument for `CompImageHDU`; `CompImageHeader` is deprecated. Avoid those legacy patterns in new examples.
- The recommended optional extras are `recommended` for common plotting/scientific dependencies and `all` only when a broad optional feature set is needed.
## Documentation and Resources
- Official Astropy Documentation: https://docs.astropy.org/en/stable/
- Tutorials: https://learn.astropy.org/
- GitHub: https://github.com/astropy/astropy
## Reference Files
For detailed information on specific modules:
- `references/units.md` - Units, quantities, conversions, and equivalencies
- `references/coordinates.md` - Coordinate systems, transformations, and catalog matching
- `references/cosmology.md` - Cosmological models and calculations
- `references/fits.md` - FITS file operations and manipulation
- `references/tables.md` - Table creation, I/O, and operations
- `references/time.md` - Time formats, scales, and calculations
- `references/wcs_and_other_modules.md` - WCS, NDData, modeling, visualization, constants, and utilities
skills/astropy/references/coordinates.md
+280
View File
@@ -0,0 +1,280 @@
# Astronomical Coordinates (astropy.coordinates)
The `astropy.coordinates` package provides tools for representing celestial coordinates and transforming between different coordinate systems.
## Creating Coordinates with SkyCoord
The high-level `SkyCoord` class is the recommended interface:
```python
from astropy import units as u
from astropy.coordinates import SkyCoord
# Decimal degrees
c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs')
# Sexagesimal strings
c = SkyCoord(ra='00h42m30s', dec='+41d12m00s', frame='icrs')
# Mixed formats
c = SkyCoord('00h42.5m +41d12m', unit=(u.hourangle, u.deg))
# Galactic coordinates
c = SkyCoord(l=120.5*u.degree, b=-23.4*u.degree, frame='galactic')
```
## Array Coordinates
Process multiple coordinates efficiently using arrays:
```python
# Create array of coordinates
coords = SkyCoord(ra=[10, 11, 12]*u.degree,
dec=[41, -5, 42]*u.degree)
# Access individual elements
coords[0]
coords[1:3]
# Array operations
coords.shape
len(coords)
```
## Accessing Components
```python
c = SkyCoord(ra=10.68*u.degree, dec=41.27*u.degree, frame='icrs')
# Access coordinates
c.ra # <Longitude 10.68 deg>
c.dec # <Latitude 41.27 deg>
c.ra.hour # Convert to hours
c.ra.hms # Hours, minutes, seconds tuple
c.dec.dms # Degrees, arcminutes, arcseconds tuple
```
## String Formatting
```python
c.to_string('decimal') # '10.68 41.27'
c.to_string('dms') # '10d40m48s 41d16m12s'
c.to_string('hmsdms') # '00h42m43.2s +41d16m12s'
# Custom formatting
c.ra.to_string(unit=u.hour, sep=':', precision=2)
```
## Coordinate Transformations
Transform between reference frames:
```python
c_icrs = SkyCoord(ra=10.68*u.degree, dec=41.27*u.degree, frame='icrs')
# Simple transformations (as attributes)
c_galactic = c_icrs.galactic
c_fk5 = c_icrs.fk5
c_fk4 = c_icrs.fk4
# Explicit transformations
c_icrs.transform_to('galactic')
c_icrs.transform_to(FK5(equinox='J1975')) # Custom frame parameters
```
## Common Coordinate Frames
### Celestial Frames
- **ICRS**: International Celestial Reference System (default, most common)
- **FK5**: Fifth Fundamental Catalogue (equinox J2000.0 by default)
- **FK4**: Fourth Fundamental Catalogue (older, requires equinox specification)
- **GCRS**: Geocentric Celestial Reference System
- **CIRS**: Celestial Intermediate Reference System
### Galactic Frames
- **Galactic**: IAU 1958 galactic coordinates
- **Supergalactic**: De Vaucouleurs supergalactic coordinates
- **Galactocentric**: Galactic center-based 3D coordinates
### Horizontal Frames
- **AltAz**: Altitude-azimuth (observer-dependent)
- **HADec**: Hour angle-declination
### Ecliptic Frames
- **GeocentricMeanEcliptic**: Geocentric mean ecliptic
- **BarycentricMeanEcliptic**: Barycentric mean ecliptic
- **HeliocentricMeanEcliptic**: Heliocentric mean ecliptic
## Observer-Dependent Transformations
For altitude-azimuth coordinates, specify observation time and location:
```python
from astropy.time import Time
from astropy.coordinates import EarthLocation, AltAz
# Define observer location
observing_location = EarthLocation(lat=40.8*u.deg, lon=-121.5*u.deg, height=1060*u.m)
# Or use named observatory
observing_location = EarthLocation.of_site('Apache Point Observatory')
# Define observation time
observing_time = Time('2023-01-15 23:00:00')
# Transform to alt-az
aa_frame = AltAz(obstime=observing_time, location=observing_location)
aa = c_icrs.transform_to(aa_frame)
print(f"Altitude: {aa.alt}")
print(f"Azimuth: {aa.az}")
```
## Working with Distances
Add distance information for 3D coordinates:
```python
# With distance
c = SkyCoord(ra=10*u.degree, dec=9*u.degree, distance=770*u.kpc, frame='icrs')
# Access 3D Cartesian coordinates
c.cartesian.x
c.cartesian.y
c.cartesian.z
# Distance from origin
c.distance
# 3D separation
c1 = SkyCoord(ra=10*u.degree, dec=9*u.degree, distance=10*u.pc)
c2 = SkyCoord(ra=11*u.degree, dec=10*u.degree, distance=11.5*u.pc)
sep_3d = c1.separation_3d(c2) # 3D distance
```
## Angular Separation
Calculate on-sky separations:
```python
c1 = SkyCoord(ra=10*u.degree, dec=9*u.degree, frame='icrs')
c2 = SkyCoord(ra=11*u.degree, dec=10*u.degree, frame='fk5')
# Angular separation (handles frame conversion automatically)
sep = c1.separation(c2)
print(f"Separation: {sep.arcsec} arcsec")
# Position angle
pa = c1.position_angle(c2)
```
## Catalog Matching
Match coordinates to catalog sources:
```python
# Single target matching
catalog = SkyCoord(ra=ra_array*u.degree, dec=dec_array*u.degree)
target = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree)
# Find closest match
idx, sep2d, dist3d = target.match_to_catalog_sky(catalog)
matched_coord = catalog[idx]
# Match with maximum separation constraint
matches = target.separation(catalog) < 1*u.arcsec
```
## Named Objects
Retrieve coordinates from online catalogs:
**Network note:** `SkyCoord.from_name()` sends the object name to remote name-resolution services such as Sesame/SIMBAD/NED. Do not use it with confidential target names or proprietary survey identifiers; use explicit coordinates when privacy or reproducibility matters.
```python
# Query by name (requires internet)
m31 = SkyCoord.from_name("M31")
crab = SkyCoord.from_name("Crab Nebula")
psr = SkyCoord.from_name("PSR J1012+5307")
```
## Earth Locations
Define observer locations:
**Network note:** `EarthLocation.of_site()` normally uses the bundled site registry, but `refresh_cache=True` downloads an updated registry. `EarthLocation.of_address()` sends the address to a geocoding service, so prefer explicit latitude/longitude/height for sensitive sites.
```python
# By coordinates
location = EarthLocation(lat=40*u.deg, lon=-120*u.deg, height=1000*u.m)
# By named observatory
keck = EarthLocation.of_site('Keck Observatory')
vlt = EarthLocation.of_site('Paranal Observatory')
# Force a fresh observatory registry only when network access is acceptable
keck = EarthLocation.of_site('Keck Observatory', refresh_cache=True)
# By address (requires internet)
location = EarthLocation.of_address('1 Observatory Road, Example City')
# List available observatories
EarthLocation.get_site_names()
```
## Velocity Information
Include proper motion and radial velocity:
```python
# Proper motion
c = SkyCoord(ra=10*u.degree, dec=41*u.degree,
pm_ra_cosdec=15*u.mas/u.yr,
pm_dec=5*u.mas/u.yr,
distance=150*u.pc)
# Radial velocity
c = SkyCoord(ra=10*u.degree, dec=41*u.degree,
radial_velocity=20*u.km/u.s)
# Both
c = SkyCoord(ra=10*u.degree, dec=41*u.degree, distance=150*u.pc,
pm_ra_cosdec=15*u.mas/u.yr, pm_dec=5*u.mas/u.yr,
radial_velocity=20*u.km/u.s)
```
## Representation Types
Switch between coordinate representations:
```python
# Cartesian representation
c = SkyCoord(x=1*u.kpc, y=2*u.kpc, z=3*u.kpc,
representation_type='cartesian', frame='icrs')
# Change representation
c.representation_type = 'cylindrical'
c.rho # Cylindrical radius
c.phi # Azimuthal angle
c.z # Height
# Spherical (default for most frames)
c.representation_type = 'spherical'
```
## Performance Tips
1. **Use arrays, not loops**: Process multiple coordinates as single array
2. **Pre-compute frames**: Reuse frame objects for multiple transformations
3. **Use broadcasting**: Efficiently transform many positions across many times
4. **Enable interpolation**: For dense time sampling, use ErfaAstromInterpolator
```python
# Fast approach
coords = SkyCoord(ra=ra_array*u.degree, dec=dec_array*u.degree)
coords_transformed = coords.transform_to('galactic')
# Slow approach (avoid)
for ra, dec in zip(ra_array, dec_array):
c = SkyCoord(ra=ra*u.degree, dec=dec*u.degree)
c_transformed = c.transform_to('galactic')
```
skills/astropy/references/cosmology.md
+309
View File
@@ -0,0 +1,309 @@
# Cosmological Calculations (astropy.cosmology)
The `astropy.cosmology` subpackage provides tools for cosmological calculations based on various cosmological models.
> **Import paths:** Always import classes and functions directly from `astropy.cosmology` (as shown in all examples below). The old submodule import paths (`astropy.cosmology.flrw`, `.core`, `.funcs`, `.connect`, `.parameter`) were deprecated in v7.1 and removed in Astropy 8.0.
## Using Built-in Cosmologies
Preloaded cosmologies based on WMAP and Planck observations:
```python
from astropy.cosmology import Planck18, Planck15, Planck13
from astropy.cosmology import WMAP9, WMAP7, WMAP5
from astropy import units as u
# Use Planck 2018 cosmology
cosmo = Planck18
# Calculate distance to z=4
d = cosmo.luminosity_distance(4)
print(f"Luminosity distance at z=4: {d}")
# Age of universe at z=0
age = cosmo.age(0)
print(f"Current age of universe: {age.to(u.Gyr)}")
```
## Creating Custom Cosmologies
### FlatLambdaCDM (Most Common)
Flat universe with cosmological constant:
```python
from astropy.cosmology import FlatLambdaCDM
# Define cosmology
cosmo = FlatLambdaCDM(
H0=70 * u.km / u.s / u.Mpc, # Hubble constant at z=0
Om0=0.3, # Matter density parameter at z=0
Tcmb0=2.725 * u.K # CMB temperature (optional)
)
```
### LambdaCDM (Non-Flat)
Non-flat universe with cosmological constant:
```python
from astropy.cosmology import LambdaCDM
cosmo = LambdaCDM(
H0=70 * u.km / u.s / u.Mpc,
Om0=0.3,
Ode0=0.7 # Dark energy density parameter
)
```
### wCDM and w0wzCDM
Dark energy with equation of state parameter:
```python
from astropy.cosmology import FlatwCDM, w0wzCDM
# Constant w
cosmo_w = FlatwCDM(H0=70 * u.km/u.s/u.Mpc, Om0=0.3, w0=-0.9)
# Evolving w(z) = w0 + wz * z
cosmo_wz = w0wzCDM(H0=70 * u.km/u.s/u.Mpc, Om0=0.3, Ode0=0.7,
w0=-1.0, wz=0.1)
```
## Distance Calculations
### Comoving Distance
Line-of-sight comoving distance:
```python
d_c = cosmo.comoving_distance(z)
```
### Luminosity Distance
Distance for calculating luminosity from observed flux:
```python
d_L = cosmo.luminosity_distance(z)
# Calculate absolute magnitude from apparent magnitude
M = m - 5*np.log10(d_L.to(u.pc).value) + 5
```
### Angular Diameter Distance
Distance for calculating physical size from angular size:
```python
d_A = cosmo.angular_diameter_distance(z)
# Calculate physical size from angular size
theta = 10 * u.arcsec # Angular size
physical_size = d_A * theta.to(u.radian).value
```
### Comoving Transverse Distance
Transverse comoving distance (equals comoving distance in flat universe):
```python
d_M = cosmo.comoving_transverse_distance(z)
```
### Distance Modulus
```python
dm = cosmo.distmod(z)
# Relates apparent and absolute magnitudes: m - M = dm
```
## Scale Calculations
### kpc per Arcminute
Physical scale at a given redshift:
```python
scale = cosmo.kpc_proper_per_arcmin(z)
# e.g., "50 kpc per arcminute at z=1"
```
### Comoving Volume
Volume element for survey volume calculations:
```python
vol = cosmo.comoving_volume(z) # Total volume to redshift z
vol_element = cosmo.differential_comoving_volume(z) # dV/dz
```
## Time Calculations
### Age of Universe
Age at a given redshift:
```python
age = cosmo.age(z)
age_now = cosmo.age(0) # Current age
age_at_z1 = cosmo.age(1) # Age at z=1
```
### Lookback Time
Time since photons were emitted:
```python
t_lookback = cosmo.lookback_time(z)
# Time between z and z=0
```
## Hubble Parameter
Hubble parameter as function of redshift:
```python
H_z = cosmo.H(z) # H(z) in km/s/Mpc
E_z = cosmo.efunc(z) # E(z) = H(z)/H0
```
## Density Parameters
Evolution of density parameters with redshift:
```python
Om_z = cosmo.Om(z) # Matter density at z
Ode_z = cosmo.Ode(z) # Dark energy density at z
Ok_z = cosmo.Ok(z) # Curvature density at z
Ogamma_z = cosmo.Ogamma(z) # Photon density at z
Onu_z = cosmo.Onu(z) # Neutrino density at z
```
## Critical and Characteristic Densities
```python
rho_c = cosmo.critical_density(z) # Critical density at z
rho_m = cosmo.critical_density(z) * cosmo.Om(z) # Matter density
```
## Inverse Calculations
Find redshift corresponding to a specific value:
```python
from astropy.cosmology import z_at_value
# Find z at specific lookback time
z = z_at_value(cosmo.lookback_time, 10*u.Gyr)
# Find z at specific luminosity distance
z = z_at_value(cosmo.luminosity_distance, 1000*u.Mpc)
# Find z at specific age
z = z_at_value(cosmo.age, 1*u.Gyr)
```
## Array Operations
All methods accept array inputs:
```python
import numpy as np
z_array = np.linspace(0, 5, 100)
d_L_array = cosmo.luminosity_distance(z_array)
H_array = cosmo.H(z_array)
age_array = cosmo.age(z_array)
```
## Neutrino Effects
Include massive neutrinos:
```python
from astropy.cosmology import FlatLambdaCDM
# With massive neutrinos
cosmo = FlatLambdaCDM(
H0=70 * u.km/u.s/u.Mpc,
Om0=0.3,
Tcmb0=2.725 * u.K,
Neff=3.04, # Effective number of neutrino species
m_nu=[0., 0., 0.06] * u.eV # Neutrino masses
)
```
Note: Massive neutrinos reduce performance by 3-4x but provide more accurate results.
## Cloning and Modifying Cosmologies
Cosmology objects are immutable. Create modified copies:
```python
# Clone with different H0
cosmo_new = cosmo.clone(H0=72 * u.km/u.s/u.Mpc)
# Clone with modified name
cosmo_named = cosmo.clone(name="My Custom Cosmology")
```
## Common Use Cases
### Calculating Absolute Magnitude
```python
# From apparent magnitude and redshift
z = 1.5
m_app = 24.5 # Apparent magnitude
d_L = cosmo.luminosity_distance(z)
M_abs = m_app - cosmo.distmod(z).value
```
### Survey Volume Calculations
```python
# Volume between two redshifts
z_min, z_max = 0.5, 1.5
volume = cosmo.comoving_volume(z_max) - cosmo.comoving_volume(z_min)
# Convert to Gpc^3
volume_gpc3 = volume.to(u.Gpc**3)
```
### Physical Size from Angular Size
```python
theta = 1 * u.arcsec # Angular size
z = 2.0
d_A = cosmo.angular_diameter_distance(z)
size_kpc = (d_A * theta.to(u.radian)).to(u.kpc)
```
### Time Since Big Bang
```python
# Age at specific redshift
z_formation = 6
age_at_formation = cosmo.age(z_formation)
time_since_formation = cosmo.age(0) - age_at_formation
```
## Comparison of Cosmologies
```python
# Compare different models
from astropy.cosmology import Planck18, WMAP9
z = 1.0
print(f"Planck18 d_L: {Planck18.luminosity_distance(z)}")
print(f"WMAP9 d_L: {WMAP9.luminosity_distance(z)}")
```
## Performance Considerations
- Calculations are fast for most purposes
- Massive neutrinos reduce speed significantly
- Array operations are vectorized and efficient
- Results valid for z < 5000-6000 (depends on model)
skills/astropy/references/fits.md
+398
View File
@@ -0,0 +1,398 @@
# FITS File Handling (astropy.io.fits)
The `astropy.io.fits` module provides comprehensive tools for reading, writing, and manipulating FITS (Flexible Image Transport System) files.
## Opening FITS Files
### Basic File Opening
```python
from astropy.io import fits
# Open file (returns HDUList - list of HDUs)
hdul = fits.open('filename.fits')
# Always close when done
hdul.close()
# Better: use context manager (automatically closes)
with fits.open('filename.fits') as hdul:
hdul.info() # Display file structure
data = hdul[0].data
```
### File Opening Modes
```python
fits.open('file.fits', mode='readonly') # Read-only (default)
fits.open('file.fits', mode='update') # Read and write
fits.open('file.fits', mode='append') # Add HDUs to file
```
### Memory Mapping
For large files, use memory mapping (default behavior):
```python
hdul = fits.open('large_file.fits', memmap=True)
# Only loads data chunks as needed
```
### Remote Files
Access cloud-hosted FITS files:
**Network note:** Remote FITS reads disclose the target URI to the storage provider and may use credentials from the configured filesystem backend. Use anonymous reads only for public data, and prefer local files for proprietary observations.
```python
uri = "s3://bucket-name/image.fits"
with fits.open(uri, use_fsspec=True, fsspec_kwargs={"anon": True}) as hdul:
# Use .section to get cutouts without downloading entire file
cutout = hdul[1].section[100:200, 100:200]
```
## HDU Structure
FITS files contain Header Data Units (HDUs):
- **Primary HDU** (`hdul[0]`): First HDU, always present
- **Extension HDUs** (`hdul[1:]`): Image or table extensions
```python
hdul.info() # Display all HDUs
# Output:
# No. Name Ver Type Cards Dimensions Format
# 0 PRIMARY 1 PrimaryHDU 220 ()
# 1 SCI 1 ImageHDU 140 (1014, 1014) float32
# 2 ERR 1 ImageHDU 51 (1014, 1014) float32
```
## Accessing HDUs
```python
# By index
primary = hdul[0]
extension1 = hdul[1]
# By name
sci = hdul['SCI']
# By name and version number
sci2 = hdul['SCI', 2] # Second SCI extension
```
## Working with Headers
### Reading Header Values
```python
hdu = hdul[0]
header = hdu.header
# Get keyword value (case-insensitive)
observer = header['OBSERVER']
exptime = header['EXPTIME']
# Get with default if missing
filter_name = header.get('FILTER', 'Unknown')
# Access by index
value = header[7] # 8th card's value
```
### Modifying Headers
```python
# Update existing keyword
header['OBSERVER'] = 'Edwin Hubble'
# Add/update with comment
header['OBSERVER'] = ('Edwin Hubble', 'Name of observer')
# Add keyword at specific position
header.insert(5, ('NEWKEY', 'value', 'comment'))
# Add HISTORY and COMMENT
header['HISTORY'] = 'File processed on 2025-01-15'
header['COMMENT'] = 'Note about the data'
# Delete keyword
del header['OLDKEY']
```
### Header Cards
Each keyword is stored as a "card" (80-character record):
```python
# Access full card
card = header.cards[0]
print(f"{card.keyword} = {card.value} / {card.comment}")
# Iterate over all cards
for card in header.cards:
print(f"{card.keyword}: {card.value}")
```
## Working with Image Data
### Reading Image Data
```python
# Get data from HDU
data = hdul[1].data # Returns NumPy array
# Data properties
print(data.shape) # e.g., (1024, 1024)
print(data.dtype) # e.g., float32
print(data.min(), data.max())
# Access specific pixels
pixel_value = data[100, 200]
region = data[100:200, 300:400]
```
### Data Operations
Data is a NumPy array, so use standard NumPy operations:
```python
import numpy as np
# Statistics
mean = np.mean(data)
median = np.median(data)
std = np.std(data)
# Modify data
data[data < 0] = 0 # Clip negative values
data = data * gain + bias # Calibration
# Mathematical operations
log_data = np.log10(data)
smoothed = scipy.ndimage.gaussian_filter(data, sigma=2)
```
### Cutouts and Sections
Extract regions without loading entire array:
```python
# Section notation [y_start:y_end, x_start:x_end]
cutout = hdul[1].section[500:600, 700:800]
```
## Creating New FITS Files
### Simple Image File
```python
# Create data
data = np.random.random((100, 100))
# Create HDU
hdu = fits.PrimaryHDU(data=data)
# Add header keywords
hdu.header['OBJECT'] = 'Test Image'
hdu.header['EXPTIME'] = 300.0
# Write to file
hdu.writeto('new_image.fits')
# Overwrite if exists
hdu.writeto('new_image.fits', overwrite=True)
```
### Multi-Extension File
```python
# Create primary HDU (can have no data)
primary = fits.PrimaryHDU()
primary.header['TELESCOP'] = 'HST'
# Create image extensions
sci_data = np.ones((100, 100))
sci = fits.ImageHDU(data=sci_data, name='SCI')
err_data = np.ones((100, 100)) * 0.1
err = fits.ImageHDU(data=err_data, name='ERR')
# Combine into HDUList
hdul = fits.HDUList([primary, sci, err])
# Write to file
hdul.writeto('multi_extension.fits')
```
## Working with Table Data
### Reading Tables
```python
# Open table
with fits.open('table.fits') as hdul:
table = hdul[1].data # BinTableHDU or TableHDU
# Access columns
ra = table['RA']
dec = table['DEC']
mag = table['MAG']
# Access rows
first_row = table[0]
subset = table[10:20]
# Column info
cols = hdul[1].columns
print(cols.names)
cols.info()
```
### Creating Tables
```python
# Define columns
col1 = fits.Column(name='ID', format='K', array=[1, 2, 3, 4])
col2 = fits.Column(name='RA', format='D', array=[10.5, 11.2, 12.3, 13.1])
col3 = fits.Column(name='DEC', format='D', array=[41.2, 42.1, 43.5, 44.2])
col4 = fits.Column(name='Name', format='20A',
array=['Star1', 'Star2', 'Star3', 'Star4'])
# Create table HDU
table_hdu = fits.BinTableHDU.from_columns([col1, col2, col3, col4])
table_hdu.name = 'CATALOG'
# Write to file
table_hdu.writeto('catalog.fits', overwrite=True)
```
### Column Formats
Common FITS table column formats:
- `'A'`: Character string (e.g., '20A' for 20 characters)
- `'L'`: Logical (boolean)
- `'B'`: Unsigned byte
- `'I'`: 16-bit integer
- `'J'`: 32-bit integer
- `'K'`: 64-bit integer
- `'E'`: 32-bit floating point
- `'D'`: 64-bit floating point
## Modifying Existing Files
### Update Mode
```python
with fits.open('file.fits', mode='update') as hdul:
# Modify header
hdul[0].header['NEWKEY'] = 'value'
# Modify data
hdul[1].data[100, 100] = 999
# Changes automatically saved when context exits
```
### Append Mode
```python
# Add new extension to existing file
new_data = np.random.random((50, 50))
new_hdu = fits.ImageHDU(data=new_data, name='NEW_EXT')
with fits.open('file.fits', mode='append') as hdul:
hdul.append(new_hdu)
```
## Convenience Functions
For quick operations without managing HDU lists:
```python
# Get data only
data = fits.getdata('file.fits', ext=1)
# Get header only
header = fits.getheader('file.fits', ext=0)
# Get both
data, header = fits.getdata('file.fits', ext=1, header=True)
# Get single keyword value
exptime = fits.getval('file.fits', 'EXPTIME', ext=0)
# Set keyword value
fits.setval('file.fits', 'NEWKEY', value='newvalue', ext=0)
# Write simple file
fits.writeto('output.fits', data, header, overwrite=True)
# Append to file
fits.append('file.fits', data, header)
# Display file info
fits.info('file.fits')
```
## Comparing FITS Files
```python
# Print differences between two files
fits.printdiff('file1.fits', 'file2.fits')
# Compare programmatically
diff = fits.FITSDiff('file1.fits', 'file2.fits')
print(diff.report())
```
## Converting Between Formats
### FITS to/from Astropy Table
```python
from astropy.table import Table
# FITS to Table
table = Table.read('catalog.fits')
# Table to FITS
table.write('output.fits', format='fits', overwrite=True)
```
## Best Practices
1. **Always use context managers** (`with` statements) for safe file handling
2. **Avoid modifying structural keywords** (SIMPLE, BITPIX, NAXIS, etc.)
3. **Use memory mapping** for large files to conserve RAM
4. **Use .section** for remote files to avoid full downloads
5. **Check HDU structure** with `.info()` before accessing data
6. **Verify data types** before operations to avoid unexpected behavior
7. **Use convenience functions** for simple one-off operations
## Common Issues
### Handling Non-Standard FITS
Some files violate FITS standards:
```python
# Ignore verification warnings
hdul = fits.open('bad_file.fits', ignore_missing_end=True)
# Fix non-standard files
hdul = fits.open('bad_file.fits')
hdul.verify('fix') # Try to fix issues
hdul.writeto('fixed_file.fits')
```
### Large File Performance
```python
# Use memory mapping (default)
hdul = fits.open('huge_file.fits', memmap=True)
# For write operations with large arrays, use Dask
import dask.array as da
large_array = da.random.random((10000, 10000))
fits.writeto('output.fits', large_array)
```
skills/astropy/references/tables.md
+495
View File
@@ -0,0 +1,495 @@
# Table Operations (astropy.table)
The `astropy.table` module provides flexible tools for working with tabular data, with support for units, masked values, and various file formats.
## Creating Tables
### Basic Table Creation
```python
from astropy.table import Table, QTable
import astropy.units as u
import numpy as np
# From column arrays
a = [1, 4, 5]
b = [2.0, 5.0, 8.2]
c = ['x', 'y', 'z']
t = Table([a, b, c], names=('id', 'flux', 'name'))
# With units (use QTable)
flux = [1.2, 2.3, 3.4] * u.Jy
wavelength = [500, 600, 700] * u.nm
t = QTable([flux, wavelength], names=('flux', 'wavelength'))
```
### From Lists of Rows
```python
# List of tuples
rows = [(1, 10.5, 'A'), (2, 11.2, 'B'), (3, 12.3, 'C')]
t = Table(rows=rows, names=('id', 'value', 'name'))
# List of dictionaries
rows = [{'id': 1, 'value': 10.5}, {'id': 2, 'value': 11.2}]
t = Table(rows)
```
### From NumPy Arrays
```python
# Structured array
arr = np.array([(1, 2.0, 'x'), (4, 5.0, 'y')],
dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'U10')])
t = Table(arr)
# 2D array with column names
data = np.random.random((100, 3))
t = Table(data, names=['col1', 'col2', 'col3'])
```
### From Pandas DataFrame
```python
import pandas as pd
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
t = Table.from_pandas(df)
```
## Accessing Table Data
### Basic Access
```python
# Column access
ra_col = t['ra'] # Returns Column object
dec_col = t['dec']
# Row access
first_row = t[0] # Returns Row object
row_slice = t[10:20] # Returns new Table
# Cell access
value = t['ra'][5] # 6th value in 'ra' column
value = t[5]['ra'] # Same thing
# Multiple columns
subset = t['ra', 'dec', 'mag']
```
### Table Properties
```python
len(t) # Number of rows
t.colnames # List of column names
t.dtype # Column data types
t.info # Detailed information
t.meta # Metadata dictionary
```
### Iteration
```python
# Iterate over rows
for row in t:
print(row['ra'], row['dec'])
# Iterate over columns
for colname in t.colnames:
print(t[colname])
```
## Modifying Tables
### Adding Columns
```python
# Add new column
t['new_col'] = [1, 2, 3, 4, 5]
t['calc'] = t['a'] + t['b'] # Calculated column
# Add column with units
t['velocity'] = [10, 20, 30] * u.km / u.s
# Add empty column
from astropy.table import Column
t['empty'] = Column(length=len(t), dtype=float)
# Insert at specific position
t.add_column([7, 8, 9], name='inserted', index=2)
```
### Removing Columns
```python
# Remove single column
t.remove_column('old_col')
# Remove multiple columns
t.remove_columns(['col1', 'col2'])
# Delete syntax
del t['col_name']
# Keep only specific columns
t.keep_columns(['ra', 'dec', 'mag'])
```
### Renaming Columns
```python
t.rename_column('old_name', 'new_name')
# Rename multiple
t.rename_columns(['old1', 'old2'], ['new1', 'new2'])
```
### Adding Rows
```python
# Add single row
t.add_row([1, 2.5, 'new'])
# Add row as dict
t.add_row({'ra': 10.5, 'dec': 41.2, 'mag': 18.5})
# Note: Adding rows one at a time is slow!
# Better to collect rows and create table at once
```
### Modifying Data
```python
# Modify column values
t['flux'] = t['flux'] * gain
t['mag'][t['mag'] < 0] = np.nan
# Modify single cell
t['ra'][5] = 10.5
# Modify entire row
t[0] = [new_id, new_ra, new_dec]
```
## Sorting and Filtering
### Sorting
```python
# Sort by single column
t.sort('mag')
# Sort descending
t.sort('mag', reverse=True)
# Sort by multiple columns
t.sort(['priority', 'mag'])
# Get sorted indices without modifying table
indices = t.argsort('mag')
sorted_table = t[indices]
```
### Filtering
```python
# Boolean indexing
bright = t[t['mag'] < 18]
nearby = t[t['distance'] < 100*u.pc]
# Multiple conditions
selected = t[(t['mag'] < 18) & (t['dec'] > 0)]
# Using numpy functions
high_snr = t[np.abs(t['flux'] / t['error']) > 5]
```
## Reading and Writing Files
### Supported Formats
FITS, HDF5, ASCII (CSV, ECSV, IPAC, etc.), VOTable, Parquet, ASDF
### Reading Files
```python
# Automatic format detection
t = Table.read('catalog.fits')
t = Table.read('data.csv')
t = Table.read('table.vot')
# Specify format explicitly
t = Table.read('data.txt', format='ascii')
t = Table.read('catalog.hdf5', path='/dataset/table')
# Read specific HDU from FITS
t = Table.read('file.fits', hdu=2)
```
### Writing Files
```python
# Automatic format from extension
t.write('output.fits')
t.write('output.csv')
# Specify format
t.write('output.txt', format='ascii.csv')
t.write('output.hdf5', path='/data/table', serialize_meta=True)
# Overwrite existing file
t.write('output.fits', overwrite=True)
```
### ASCII Format Options
```python
# CSV with custom delimiter
t.write('output.csv', format='ascii.csv', delimiter='|')
# Fixed-width format
t.write('output.txt', format='ascii.fixed_width')
# IPAC format
t.write('output.tbl', format='ascii.ipac')
# LaTeX table
t.write('table.tex', format='ascii.latex')
```
## Table Operations
### Stacking Tables (Vertical)
```python
from astropy.table import vstack
# Concatenate tables vertically
t1 = Table([[1, 2], [3, 4]], names=('a', 'b'))
t2 = Table([[5, 6], [7, 8]], names=('a', 'b'))
t_combined = vstack([t1, t2])
```
### Joining Tables (Horizontal)
```python
from astropy.table import hstack
# Concatenate tables horizontally
t1 = Table([[1, 2]], names=['a'])
t2 = Table([[3, 4]], names=['b'])
t_combined = hstack([t1, t2])
```
### Database-Style Joins
```python
from astropy.table import join
# Inner join on common column
t1 = Table([[1, 2, 3], ['a', 'b', 'c']], names=('id', 'data1'))
t2 = Table([[1, 2, 4], ['x', 'y', 'z']], names=('id', 'data2'))
t_joined = join(t1, t2, keys='id')
# Left/right/outer joins
t_joined = join(t1, t2, join_type='left')
t_joined = join(t1, t2, join_type='outer')
```
### Grouping and Aggregating
```python
# Group by column
g = t.group_by('filter')
# Aggregate groups
means = g.groups.aggregate(np.mean)
# Iterate over groups
for group in g.groups:
print(f"Filter: {group['filter'][0]}")
print(f"Mean mag: {np.mean(group['mag'])}")
```
### Unique Rows
```python
# Get unique rows
t_unique = t.unique('id')
# Multiple columns
t_unique = t.unique(['ra', 'dec'])
```
## Units and Quantities
Use QTable for unit-aware operations:
```python
from astropy.table import QTable
# Create table with units
t = QTable()
t['flux'] = [1.2, 2.3, 3.4] * u.Jy
t['wavelength'] = [500, 600, 700] * u.nm
# Unit conversions
t['flux'].to(u.mJy)
t['wavelength'].to(u.angstrom)
# Calculations preserve units
t['freq'] = t['wavelength'].to(u.Hz, equivalencies=u.spectral())
```
## Masking Missing Data
```python
from astropy.table import MaskedColumn
import numpy as np
# Create masked column
flux = MaskedColumn([1.2, np.nan, 3.4], mask=[False, True, False])
t = Table([flux], names=['flux'])
# Operations automatically handle masks
mean_flux = np.ma.mean(t['flux'])
# Fill masked values
t['flux'].filled(0) # Replace masked with 0
```
## Indexing for Fast Lookup
Create indices for fast row retrieval:
```python
# Add index on column
t.add_index('id')
# Fast lookup by index
row = t.loc[12345] # Find row where id=12345
# Range queries
subset = t.loc[100:200]
# With multiple indices, select which index to use explicitly.
# t.loc["b", 2] (index name as first element) is deprecated since 7.2
# and slated for removal in 9.0.
t.add_index('name')
row = t.loc.with_index('name')['M31']
```
## Table Metadata
```python
# Set table-level metadata
t.meta['TELESCOPE'] = 'HST'
t.meta['FILTER'] = 'F814W'
t.meta['EXPTIME'] = 300.0
# Set column-level metadata
t['ra'].meta['unit'] = 'deg'
t['ra'].meta['description'] = 'Right Ascension'
t['ra'].description = 'Right Ascension' # Shortcut
```
## Performance Tips
### Fast Table Construction
```python
# SLOW: Adding rows one at a time
t = Table(names=['a', 'b'])
for i in range(1000):
t.add_row([i, i**2])
# FAST: Build from lists
rows = [(i, i**2) for i in range(1000)]
t = Table(rows=rows, names=['a', 'b'])
```
### Memory-Mapped FITS Tables
```python
# Don't load entire table into memory
t = Table.read('huge_catalog.fits', memmap=True)
# Only loads data when accessed
subset = t[10000:10100] # Efficient
```
### Copy vs. View
```python
# Create view (shares data, fast)
t_view = t['ra', 'dec']
# Create copy (independent data)
t_copy = t['ra', 'dec'].copy()
```
## Displaying Tables
```python
# Print to console
print(t)
# Show in interactive browser
t.show_in_browser()
t.show_in_browser(jsviewer=True) # Interactive sorting/filtering
# Paginated viewing
t.more()
# Custom formatting
t['flux'].format = '%.3f'
t['ra'].format = '{:.6f}'
```
## Converting to Other Formats
```python
# To NumPy array
arr = np.array(t)
# To Pandas DataFrame
df = t.to_pandas()
# To dictionary
d = {name: t[name] for name in t.colnames}
```
## Common Use Cases
### Cross-Matching Catalogs
```python
from astropy.coordinates import SkyCoord, match_coordinates_sky
# Create coordinate objects from table columns
coords1 = SkyCoord(t1['ra'], t1['dec'], unit='deg')
coords2 = SkyCoord(t2['ra'], t2['dec'], unit='deg')
# Find matches
idx, sep, _ = coords1.match_to_catalog_sky(coords2)
# Filter by separation
max_sep = 1 * u.arcsec
matches = sep < max_sep
t1_matched = t1[matches]
t2_matched = t2[idx[matches]]
```
### Binning Data
```python
from astropy.table import Table
import numpy as np
# Bin by magnitude
mag_bins = np.arange(10, 20, 0.5)
binned = t.group_by(np.digitize(t['mag'], mag_bins))
counts = binned.groups.aggregate(len)
```
skills/astropy/references/time.md
+412
View File
@@ -0,0 +1,412 @@
# Time Handling (astropy.time)
The `astropy.time` module provides robust tools for manipulating times and dates with support for various time scales and formats.
## Creating Time Objects
### Basic Creation
```python
from astropy.time import Time
import astropy.units as u
# ISO format (automatically detected)
t = Time('2023-01-15 12:30:45')
t = Time('2023-01-15T12:30:45')
# Specify format explicitly
t = Time('2023-01-15 12:30:45', format='iso', scale='utc')
# Julian Date
t = Time(2460000.0, format='jd')
# Modified Julian Date
t = Time(59945.0, format='mjd')
# Unix time (seconds since 1970-01-01)
t = Time(1673785845.0, format='unix')
```
### Array of Times
```python
# Multiple times
times = Time(['2023-01-01', '2023-06-01', '2023-12-31'])
# From arrays
import numpy as np
jd_array = np.linspace(2460000, 2460100, 100)
times = Time(jd_array, format='jd')
```
## Time Formats
### Supported Formats
```python
# ISO 8601
t = Time('2023-01-15 12:30:45', format='iso')
t = Time('2023-01-15T12:30:45.123', format='isot')
# Julian dates
t = Time(2460000.0, format='jd') # Julian Date
t = Time(59945.0, format='mjd') # Modified Julian Date
# Decimal year
t = Time(2023.5, format='decimalyear')
t = Time(2023.5, format='jyear') # Julian year
t = Time(2023.5, format='byear') # Besselian year
# Year and day-of-year
t = Time('2023:046', format='yday') # 46th day of 2023
# FITS format
t = Time('2023-01-15T12:30:45', format='fits')
# GPS seconds
t = Time(1000000000.0, format='gps')
# Unix time
t = Time(1673785845.0, format='unix')
# Matplotlib dates
t = Time(738521.0, format='plot_date')
# datetime objects
from datetime import datetime
dt = datetime(2023, 1, 15, 12, 30, 45)
t = Time(dt)
```
## Time Scales
### Available Time Scales
```python
# UTC - Coordinated Universal Time (default)
t = Time('2023-01-15 12:00:00', scale='utc')
# TAI - International Atomic Time
t = Time('2023-01-15 12:00:00', scale='tai')
# TT - Terrestrial Time
t = Time('2023-01-15 12:00:00', scale='tt')
# TDB - Barycentric Dynamical Time
t = Time('2023-01-15 12:00:00', scale='tdb')
# TCG - Geocentric Coordinate Time
t = Time('2023-01-15 12:00:00', scale='tcg')
# TCB - Barycentric Coordinate Time
t = Time('2023-01-15 12:00:00', scale='tcb')
# UT1 - Universal Time
t = Time('2023-01-15 12:00:00', scale='ut1')
```
### Converting Time Scales
```python
t = Time('2023-01-15 12:00:00', scale='utc')
# Convert to different scales
t_tai = t.tai
t_tt = t.tt
t_tdb = t.tdb
t_ut1 = t.ut1
# Check offset
print(f"TAI - UTC = {(t.tai - t.utc).sec} seconds")
# TAI - UTC = 37 seconds (leap seconds)
```
## Format Conversions
### Change Output Format
```python
t = Time('2023-01-15 12:30:45')
# Access in different formats
print(t.jd) # Julian Date
print(t.mjd) # Modified Julian Date
print(t.iso) # ISO format
print(t.isot) # ISO with 'T'
print(t.unix) # Unix time
print(t.decimalyear) # Decimal year
# Change default format
t.format = 'mjd'
print(t) # Displays as MJD
```
### High-Precision Output
```python
# Use subfmt for precision control
t.to_value('mjd', subfmt='float') # Standard float
t.to_value('mjd', subfmt='long') # Extended precision
t.to_value('mjd', subfmt='decimal') # Decimal (highest precision)
t.to_value('mjd', subfmt='str') # String representation
```
## Time Arithmetic
### TimeDelta Objects
```python
from astropy.time import TimeDelta
# Create time difference
dt = TimeDelta(1.0, format='jd') # 1 day
dt = TimeDelta(3600.0, format='sec') # 1 hour
# Subtract times
t1 = Time('2023-01-15')
t2 = Time('2023-02-15')
dt = t2 - t1
print(dt.jd) # 31 days
print(dt.sec) # 2678400 seconds
```
### Adding/Subtracting Time
```python
t = Time('2023-01-15 12:00:00')
# Add TimeDelta
t_future = t + TimeDelta(7, format='jd') # Add 7 days
# Add Quantity
t_future = t + 1*u.hour
t_future = t + 30*u.day
t_future = t + 1*u.year
# Subtract
t_past = t - 1*u.week
```
### Time Ranges
```python
# Create range of times
start = Time('2023-01-01')
end = Time('2023-12-31')
times = start + np.linspace(0, 365, 100) * u.day
# Or using TimeDelta
times = start + TimeDelta(np.linspace(0, 365, 100), format='jd')
```
## Observing-Related Features
Some high-precision time and coordinate operations use IERS Earth-rotation data. Astropy uses `astropy-iers-data` by default, but may auto-download fresher IERS-A data when required for UT1 or polar-motion values. Disable auto-download for offline or privacy-sensitive runs:
```python
from astropy.utils import iers
iers.conf.auto_download = False
```
### Sidereal Time
```python
from astropy.coordinates import EarthLocation
# Define observer location
location = EarthLocation(lat=40*u.deg, lon=-120*u.deg, height=1000*u.m)
# Create time with location
t = Time('2023-06-15 23:00:00', location=location)
# Calculate sidereal time
lst_apparent = t.sidereal_time('apparent')
lst_mean = t.sidereal_time('mean')
print(f"Local Sidereal Time: {lst_apparent}")
```
### Light Travel Time Corrections
```python
from astropy.coordinates import SkyCoord, EarthLocation
# Define target and observer
target = SkyCoord(ra=10*u.deg, dec=20*u.deg)
location = EarthLocation.of_site('Keck Observatory')
# Observation times
times = Time(['2023-01-01', '2023-06-01', '2023-12-31'],
location=location)
# Calculate light travel time to solar system barycenter
ltt_bary = times.light_travel_time(target, kind='barycentric')
ltt_helio = times.light_travel_time(target, kind='heliocentric')
# Apply correction
times_barycentric = times.tdb + ltt_bary
```
### Earth Rotation Angle
```python
# Earth rotation angle (for celestial to terrestrial transformations)
era = t.earth_rotation_angle()
```
## Handling Missing or Invalid Times
### Masked Times
```python
import numpy as np
# Create times with missing values
times = Time(['2023-01-01', '2023-06-01', '2023-12-31'])
times[1] = np.ma.masked # Mark as missing
# Check for masks
print(times.mask) # [False True False]
# Get unmasked version
times_clean = times.unmasked
# Fill masked values
times_filled = times.filled(Time('2000-01-01'))
```
## Time Precision and Representation
### Internal Representation
Time objects use two 64-bit floats (jd1, jd2) for high precision:
```python
t = Time('2023-01-15 12:30:45.123456789', format='iso', scale='utc')
# Access internal representation
print(t.jd1, t.jd2) # Integer and fractional parts
# This allows sub-nanosecond precision over astronomical timescales
```
### Precision
```python
# High precision for long time intervals
t1 = Time('1900-01-01')
t2 = Time('2100-01-01')
dt = t2 - t1
print(f"Time span: {dt.sec / (365.25 * 86400)} years")
# Maintains precision throughout
```
## Time Formatting
### Custom String Format
```python
t = Time('2023-01-15 12:30:45')
# Strftime-style formatting
t.strftime('%Y-%m-%d %H:%M:%S') # '2023-01-15 12:30:45'
t.strftime('%B %d, %Y') # 'January 15, 2023'
# ISO format subformats
t.iso # '2023-01-15 12:30:45.000'
t.isot # '2023-01-15T12:30:45.000'
t.to_value('iso', subfmt='date_hms') # '2023-01-15 12:30:45.000'
```
## Common Use Cases
### Converting Between Formats
```python
# MJD to ISO
t_mjd = Time(59945.0, format='mjd')
iso_string = t_mjd.iso
# ISO to JD
t_iso = Time('2023-01-15 12:00:00')
jd_value = t_iso.jd
# Unix to ISO
t_unix = Time(1673785845.0, format='unix')
iso_string = t_unix.iso
```
### Time Differences in Various Units
```python
t1 = Time('2023-01-01')
t2 = Time('2023-12-31')
dt = t2 - t1
print(f"Days: {dt.to(u.day)}")
print(f"Hours: {dt.to(u.hour)}")
print(f"Seconds: {dt.sec}")
print(f"Years: {dt.to(u.year)}")
```
### Creating Regular Time Series
```python
# Daily observations for a year
start = Time('2023-01-01')
times = start + np.arange(365) * u.day
# Hourly observations for a day
start = Time('2023-01-15 00:00:00')
times = start + np.arange(24) * u.hour
# Observations every 30 seconds
start = Time('2023-01-15 12:00:00')
times = start + np.arange(1000) * 30 * u.second
```
### Time Zone Handling
```python
# UTC to local time (requires datetime)
t = Time('2023-01-15 12:00:00', scale='utc')
dt_utc = t.to_datetime()
# Convert to specific timezone using pytz
import pytz
eastern = pytz.timezone('US/Eastern')
dt_eastern = dt_utc.replace(tzinfo=pytz.utc).astimezone(eastern)
```
### Barycentric Correction Example
```python
from astropy.coordinates import SkyCoord, EarthLocation
# Target coordinates
target = SkyCoord(ra='23h23m08.55s', dec='+18d24m59.3s')
# Observatory location
location = EarthLocation.of_site('Keck Observatory')
# Observation times (must include location)
times = Time(['2023-01-15 08:30:00', '2023-01-16 08:30:00'],
location=location)
# Calculate barycentric correction
ltt_bary = times.light_travel_time(target, kind='barycentric')
# Apply correction to get barycentric times
times_bary = times.tdb + ltt_bary
# For radial velocity work, use the coordinate helper instead
rv_correction = target.radial_velocity_correction(obstime=times, location=location)
```
## Performance Considerations
1. **Array operations are fast**: Process multiple times as arrays
2. **Format conversions are cached**: Repeated access is efficient
3. **Scale conversions may require IERS data**: Update `astropy-iers-data` before offline runs, or set `iers.conf.auto_download = False` to prevent network access
4. **High precision maintained**: Sub-nanosecond accuracy across astronomical timescales
skills/astropy/references/units.md
+178
View File
@@ -0,0 +1,178 @@
# Units and Quantities (astropy.units)
The `astropy.units` module handles defining, converting between, and performing arithmetic with physical quantities.
## Creating Quantities
Multiply or divide numeric values by built-in units to create Quantity objects:
```python
from astropy import units as u
import numpy as np
# Scalar quantities
distance = 42.0 * u.meter
velocity = 100 * u.km / u.s
# Array quantities
distances = np.array([1., 2., 3.]) * u.m
wavelengths = [500, 600, 700] * u.nm
```
Access components via `.value` and `.unit` attributes:
```python
distance.value # 42.0
distance.unit # Unit("m")
```
## Unit Conversions
Use `.to()` method for conversions:
```python
distance = 1.0 * u.parsec
distance.to(u.km) # <Quantity 30856775814671.914 km>
wavelength = 500 * u.nm
wavelength.to(u.angstrom) # <Quantity 5000. Angstrom>
```
## Arithmetic Operations
Quantities support standard arithmetic with automatic unit management:
```python
# Basic operations
speed = 15.1 * u.meter / (32.0 * u.second) # <Quantity 0.471875 m / s>
area = (5 * u.m) * (3 * u.m) # <Quantity 15. m2>
# Units cancel when appropriate
ratio = (10 * u.m) / (5 * u.m) # <Quantity 2. (dimensionless)>
# Decompose complex units
time = (3.0 * u.kilometer / (130.51 * u.meter / u.second))
time.decompose() # <Quantity 22.986744310780782 s>
```
## Unit Systems
Convert between major unit systems:
```python
# SI to CGS
pressure = 1.0 * u.Pa
pressure.cgs # <Quantity 10. Ba>
# Find equivalent representations
(u.s ** -1).compose() # [Unit("Bq"), Unit("Hz"), ...]
```
## Equivalencies
Domain-specific conversions require equivalencies:
```python
# Spectral equivalency (wavelength ↔ frequency)
wavelength = 1000 * u.nm
wavelength.to(u.Hz, equivalencies=u.spectral())
# <Quantity 2.99792458e+14 Hz>
# Doppler equivalencies
velocity = 1000 * u.km / u.s
velocity.to(u.Hz, equivalencies=u.doppler_optical(500*u.nm))
# Other equivalencies
u.brightness_temperature(500*u.GHz)
u.doppler_radio(1.4*u.GHz)
u.mass_energy()
u.parallax()
```
## Logarithmic Units
Special units for magnitudes, decibels, and dex:
```python
# Magnitudes
flux = -2.5 * u.mag(u.ct / u.s)
# Decibels
power_ratio = 3 * u.dB(u.W)
# Dex (base-10 logarithm)
abundance = 8.5 * u.dex(u.cm**-3)
```
## Common Units
### Length
`u.m, u.km, u.cm, u.mm, u.micron, u.angstrom, u.au, u.pc, u.kpc, u.Mpc, u.lyr`
### Time
`u.s, u.min, u.hour, u.day, u.year, u.Myr, u.Gyr`
### Mass
`u.kg, u.g, u.M_sun, u.M_earth, u.M_jup`
### Temperature
`u.K, u.deg_C`
### Angle
`u.deg, u.arcmin, u.arcsec, u.rad, u.hourangle, u.mas`
### Energy/Power
`u.J, u.erg, u.eV, u.keV, u.MeV, u.GeV, u.W, u.L_sun`
### Frequency
`u.Hz, u.kHz, u.MHz, u.GHz`
### Flux
`u.Jy, u.mJy, u.erg / u.s / u.cm**2`
## Performance Optimization
Pre-compute composite units for array operations:
```python
# Slow (creates intermediate quantities)
result = array * u.m / u.s / u.kg / u.sr
# Fast (pre-computed composite unit)
UNIT_COMPOSITE = u.m / u.s / u.kg / u.sr
result = array * UNIT_COMPOSITE
# Fastest (avoid copying with <<)
result = array << UNIT_COMPOSITE # 10000x faster
```
## String Formatting
Format quantities with standard Python syntax:
```python
velocity = 15.1 * u.meter / (32.0 * u.second)
f"{velocity:0.03f}" # '0.472 m / s'
f"{velocity:.2e}" # '4.72e-01 m / s'
f"{velocity.unit:FITS}" # 'm s-1'
```
## Defining Custom Units
```python
# Create new unit
bakers_fortnight = u.def_unit('bakers_fortnight', 13 * u.day)
# Enable in string parsing
u.add_enabled_units([bakers_fortnight])
```
## Constants
Access physical constants with units:
```python
from astropy.constants import c, G, M_sun, h, k_B
speed_of_light = c.to(u.km/u.s)
gravitational_constant = G.to(u.m**3 / u.kg / u.s**2)
```
skills/astropy/references/wcs_and_other_modules.md
+377
View File
@@ -0,0 +1,377 @@
# WCS and Other Astropy Modules
## World Coordinate System (astropy.wcs)
The WCS module manages transformations between pixel coordinates in images and world coordinates (e.g., celestial coordinates).
### Reading WCS from FITS
```python
from astropy.wcs import WCS
from astropy.io import fits
# Read WCS from FITS header
with fits.open('image.fits') as hdul:
wcs = WCS(hdul[0].header)
```
### Pixel to World Transformations
```python
# Single pixel to world coordinates
world = wcs.pixel_to_world(100, 200) # Returns SkyCoord
print(f"RA: {world.ra}, Dec: {world.dec}")
# Arrays of pixels
import numpy as np
x_pixels = np.array([100, 200, 300])
y_pixels = np.array([150, 250, 350])
world_coords = wcs.pixel_to_world(x_pixels, y_pixels)
```
### World to Pixel Transformations
```python
from astropy.coordinates import SkyCoord
import astropy.units as u
# Single coordinate
coord = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree)
x, y = wcs.world_to_pixel(coord)
# Array of coordinates
coords = SkyCoord(ra=[10, 11, 12]*u.degree, dec=[41, 42, 43]*u.degree)
x_pixels, y_pixels = wcs.world_to_pixel(coords)
```
### WCS Information
```python
# Print WCS details
print(wcs)
# Access key properties
print(wcs.wcs.crpix) # Reference pixel
print(wcs.wcs.crval) # Reference value (world coords)
print(wcs.wcs.cd) # CD matrix
print(wcs.wcs.ctype) # Coordinate types
# Pixel scale
pixel_scale = wcs.proj_plane_pixel_scales() # Returns Quantity array
```
### Creating WCS
```python
from astropy.wcs import WCS
# Create new WCS
wcs = WCS(naxis=2)
wcs.wcs.crpix = [512.0, 512.0] # Reference pixel
wcs.wcs.crval = [10.5, 41.2] # RA, Dec at reference pixel
wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN'] # Projection type
wcs.wcs.cdelt = [-0.0001, 0.0001] # Pixel scale (degrees/pixel)
wcs.wcs.cunit = ['deg', 'deg']
```
### Footprint and Coverage
```python
# Calculate image footprint (corner coordinates)
footprint = wcs.calc_footprint()
# Returns array of [RA, Dec] for each corner
```
## NDData (astropy.nddata)
Container for n-dimensional datasets with metadata, uncertainty, and masking.
### Creating NDData
```python
from astropy.nddata import NDData
import numpy as np
import astropy.units as u
# Basic NDData
data = np.random.random((100, 100))
ndd = NDData(data)
# With units
ndd = NDData(data, unit=u.electron/u.s)
# With uncertainty
from astropy.nddata import StdDevUncertainty
uncertainty = StdDevUncertainty(np.sqrt(data))
ndd = NDData(data, uncertainty=uncertainty, unit=u.electron/u.s)
# With mask
mask = data < 0.1 # Mask low values
ndd = NDData(data, mask=mask)
# With WCS
from astropy.wcs import WCS
ndd = NDData(data, wcs=wcs)
```
### CCDData for CCD Images
```python
from astropy.nddata import CCDData
# Create CCDData
ccd = CCDData(data, unit=u.adu, meta={'object': 'M31'})
# Read from FITS
ccd = CCDData.read('image.fits', unit=u.adu)
# Write to FITS
ccd.write('output.fits', overwrite=True)
```
## Modeling (astropy.modeling)
Framework for creating and fitting models to data.
### Common Models
```python
from astropy.modeling import models, fitting
import numpy as np
# 1D Gaussian
gauss = models.Gaussian1D(amplitude=10, mean=5, stddev=1)
x = np.linspace(0, 10, 100)
y = gauss(x)
# 2D Gaussian
gauss_2d = models.Gaussian2D(amplitude=10, x_mean=50, y_mean=50,
x_stddev=5, y_stddev=3)
# Polynomial
poly = models.Polynomial1D(degree=3)
# Power law
power_law = models.PowerLaw1D(amplitude=10, x_0=1, alpha=2)
```
### Fitting Models to Data
```python
# Generate noisy data
true_model = models.Gaussian1D(amplitude=10, mean=5, stddev=1)
x = np.linspace(0, 10, 100)
y_true = true_model(x)
y_noisy = y_true + np.random.normal(0, 0.5, x.shape)
# Fit model
fitter = fitting.LevMarLSQFitter()
initial_model = models.Gaussian1D(amplitude=8, mean=4, stddev=1.5)
fitted_model = fitter(initial_model, x, y_noisy)
print(f"Fitted amplitude: {fitted_model.amplitude.value}")
print(f"Fitted mean: {fitted_model.mean.value}")
print(f"Fitted stddev: {fitted_model.stddev.value}")
```
### Compound Models
```python
# Add models
double_gauss = models.Gaussian1D(amp=5, mean=3, stddev=1) + \
models.Gaussian1D(amp=8, mean=7, stddev=1.5)
# Compose models
composite = models.Gaussian1D(amp=10, mean=5, stddev=1) | \
models.Scale(factor=2) # Scale output
```
## Visualization (astropy.visualization)
Tools for visualizing astronomical images and data.
### Image Normalization
```python
from astropy.visualization import simple_norm
import matplotlib.pyplot as plt
# Load image
from astropy.io import fits
data = fits.getdata('image.fits')
# Normalize for display
norm = simple_norm(data, 'sqrt', percent=99)
# Display
plt.imshow(data, norm=norm, cmap='gray', origin='lower')
plt.colorbar()
plt.show()
```
### Stretching and Intervals
```python
from astropy.visualization import (MinMaxInterval, AsinhStretch,
ImageNormalize, ZScaleInterval)
# Z-scale interval
interval = ZScaleInterval()
vmin, vmax = interval.get_limits(data)
# Asinh stretch
stretch = AsinhStretch()
norm = ImageNormalize(data, interval=interval, stretch=stretch)
plt.imshow(data, norm=norm, cmap='gray', origin='lower')
```
### PercentileInterval
```python
from astropy.visualization import PercentileInterval
# Show data between 5th and 95th percentiles
interval = PercentileInterval(90) # 90% of data
vmin, vmax = interval.get_limits(data)
plt.imshow(data, vmin=vmin, vmax=vmax, cmap='gray', origin='lower')
```
## Constants (astropy.constants)
Physical and astronomical constants with units.
Note: through Astropy 7.x the default constants are CODATA 2018 + IAU 2015; Astropy 8.0 switches the default to CODATA 2022. For reproducible results across versions, select an explicit constants version via the science state (e.g., `astropy.physical_constants.set('codata2018')` before other astropy imports).
```python
from astropy import constants as const
# Speed of light
c = const.c
print(f"c = {c}")
print(f"c in km/s = {c.to(u.km/u.s)}")
# Gravitational constant
G = const.G
# Astronomical constants
M_sun = const.M_sun # Solar mass
R_sun = const.R_sun # Solar radius
L_sun = const.L_sun # Solar luminosity
au = const.au # Astronomical unit
pc = const.pc # Parsec
# Fundamental constants
h = const.h # Planck constant
hbar = const.hbar # Reduced Planck constant
k_B = const.k_B # Boltzmann constant
m_e = const.m_e # Electron mass
m_p = const.m_p # Proton mass
e = const.e # Elementary charge
N_A = const.N_A # Avogadro constant
```
### Using Constants in Calculations
```python
# Calculate Schwarzschild radius
M = 10 * const.M_sun
r_s = 2 * const.G * M / const.c**2
print(f"Schwarzschild radius: {r_s.to(u.km)}")
# Calculate escape velocity
M = const.M_earth
R = const.R_earth
v_esc = np.sqrt(2 * const.G * M / R)
print(f"Earth escape velocity: {v_esc.to(u.km/u.s)}")
```
## Convolution (astropy.convolution)
Convolution kernels for image processing.
```python
from astropy.convolution import Gaussian2DKernel, convolve
# Create Gaussian kernel
kernel = Gaussian2DKernel(x_stddev=2)
# Convolve image
smoothed_image = convolve(data, kernel)
# Handle NaNs
from astropy.convolution import convolve_fft
smoothed = convolve_fft(data, kernel, nan_treatment='interpolate')
```
## Stats (astropy.stats)
Statistical functions for astronomical data.
```python
from astropy.stats import sigma_clip, sigma_clipped_stats
# Sigma clipping
clipped_data = sigma_clip(data, sigma=3, maxiters=5)
# Get statistics with sigma clipping
mean, median, std = sigma_clipped_stats(data, sigma=3.0)
# Robust statistics
from astropy.stats import mad_std, biweight_location, biweight_scale
robust_std = mad_std(data)
robust_mean = biweight_location(data)
robust_scale = biweight_scale(data)
```
## Utils
### Data Downloads
`download_file()` fetches a remote URL and caches it locally. Treat the URL as data you are disclosing to the remote host; do not pass confidential signed URLs or internal file locations unless the workflow explicitly permits it. Use `cache=False` for one-off downloads that should not be retained in Astropy's cache.
```python
from astropy.utils.data import download_file
# Download file (caches locally)
url = 'https://example.com/data.fits'
local_file = download_file(url, cache=True)
```
### Progress Bars
```python
from astropy.utils.console import ProgressBar
with ProgressBar(len(data_list)) as bar:
for item in data_list:
# Process item
bar.update()
```
## SAMP (Simple Application Messaging Protocol)
Interoperability with other astronomy tools.
```python
from astropy.samp import SAMPIntegratedClient
# Connect to SAMP hub
client = SAMPIntegratedClient()
client.connect()
# Broadcast table to other applications
message = {
'samp.mtype': 'table.load.votable',
'samp.params': {
'url': 'file:///path/to/table.xml',
'table-id': 'my_table',
'name': 'My Catalog'
}
}
client.notify_all(message)
# Disconnect
client.disconnect()
```
skills/autoskill/.gitignore
+3
View File
@@ -0,0 +1,3 @@
__pycache__/
*.pyc
.pytest_cache/
skills/autoskill/SKILL.md
+218
View File
@@ -0,0 +1,218 @@
---
name: autoskill
description: Observe the user's screen via screenpipe, detect repeated research workflows, match them against existing scientific-agent-skills, and draft new skills (or composition recipes that chain existing ones) for the patterns not yet covered. Use when the user asks to analyze their recent work and propose skills based on what they actually do. Requires the screenpipe daemon (https://github.com/screenpipe/screenpipe) running locally on port 3030 — the skill has no other data source and will refuse to run if screenpipe is unreachable. All detection runs locally; only redacted cluster summaries reach the LLM.
allowed-tools: Read Write Edit Bash
license: MIT license
required_environment_variables: [{"name": "SCREENPIPE_TOKEN", "prompt": "Auth token for the local screenpipe daemon.", "required_for": "full functionality"}, {"name": "ANTHROPIC_API_KEY", "prompt": "For Claude API calls during skill drafting.", "required_for": "optional features"}, {"name": "FOUNDRY_API_KEY", "prompt": "Optional Foundry access for drafting.", "required_for": "optional features"}]
metadata: {"version": "1.1", "skill-author": "K-Dense Inc.", "openclaw": {"requires": {"bins": ["screenpipe"]}, "primaryEnv": "SCREENPIPE_TOKEN", "envVars": [{"name": "SCREENPIPE_TOKEN", "required": true, "description": "Auth token for the local screenpipe daemon."}, {"name": "ANTHROPIC_API_KEY", "required": false, "description": "For Claude API calls during skill drafting."}, {"name": "FOUNDRY_API_KEY", "required": false, "description": "Optional Foundry access for drafting."}]}}
---
# autoskill
> **Requires a running [screenpipe](https://github.com/screenpipe/screenpipe) daemon.** This skill has no alternate data source — it reads exclusively from the local screenpipe HTTP API (default `http://localhost:3030`). If the daemon isn't running, `run()` raises `ScreenpipeUnreachable` with install instructions.
> **Network access & environment variables.** This skill makes authenticated HTTP requests to (a) the user's local screenpipe daemon on loopback, and (b) the user-configured LLM backend — one of `http://localhost:1234/v1` (LM Studio, default), `https://api.anthropic.com` (opt-in Claude), or a user-supplied BYOK Foundry gateway. The skill reads three environment variables — `SCREENPIPE_TOKEN`, `ANTHROPIC_API_KEY`, `FOUNDRY_API_KEY` — and uses each only to authenticate to the single endpoint its name implies. No other network destinations, no telemetry, no data egress to any third party.
## Overview
Turn the user's own workflow history — captured passively by the local [screenpipe](https://github.com/screenpipe/screenpipe) daemon — into new skills. This skill is on-demand: the user invokes it with a time window, it queries screenpipe's local HTTP API, clusters repeated workflow patterns, compares each pattern against the existing skills in this repo, and produces a staged folder of proposals the user can review, edit, and promote.
## When to Use This Skill
Invoke this skill when the user asks to:
- "Analyze my last 4 hours / day / week and propose new skills."
- "Look at what I've been doing and tell me what's not covered yet."
- "Draft a skill from my recent workflow."
- "Find composition recipes for workflows I repeat."
Do **not** invoke it for one-off questions about screenpipe itself, for real-time screen queries, or without an explicit user request — the skill analyzes sensitive local content and must stay explicitly user-triggered.
## Privacy Posture
- **Screenpipe handles app/window filtering at capture time.** Install a starter deny-list by copying `references/screenpipe-config.yaml` into the user's screenpipe config. Sensitive apps (password managers, messaging, banking) are never OCR'd in the first place.
- **Raw OCR never leaves the machine.** `scripts/fetch_window.py` pulls data over localhost HTTP. `scripts/cluster.py` reduces the timeline to app/duration/title summaries. `scripts/redact.py` strips emails, API keys, bearer tokens, and phone numbers as defense-in-depth before any cluster summary reaches the LLM.
- **LLM backend defaults to `local`.** The recommended setup is [LM Studio](https://lmstudio.ai/) running `Gemma-4-31B-it` — strong reasoning at a size that fits on most workstation GPUs, and no data ever leaves your machine. Cloud backends (`claude`, `foundry`) are opt-in and documented in `config.yaml` for users who explicitly want them. Detection and embeddings always run locally regardless of backend choice.
- **Dry-run mode** (`--plan`) prints the exact timeline that will be analyzed before any LLM call.
- **TLS for localhost** (optional, for corporate policy): see `references/https-proxy.md` for the Caddy pattern.
## Prerequisites
### 1. Screenpipe daemon
Either install the official release or build from source. Either way the daemon binds HTTP on `localhost:3030` by default.
**From source** (recommended if you want the CLI daemon without the desktop GUI):
```bash
git clone --depth 1 https://github.com/mediar-ai/screenpipe.git
cd screenpipe
cargo build -p screenpipe-engine --release
# System deps (macOS): cmake + full Xcode.app (not just Command Line Tools).
# brew install cmake
# # if xcodebuild plug-ins error: sudo xcodebuild -runFirstLaunch
./target/release/screenpipe doctor # confirm permissions + ffmpeg
./target/release/screenpipe record --disable-audio --use-pii-removal
```
First run will prompt for macOS Screen Recording permission. Grant it and relaunch.
### 2. Screenpipe API token
The local API now requires bearer auth. Retrieve your token and export it:
```bash
export SCREENPIPE_TOKEN=$(screenpipe auth token)
```
(Or set `screenpipe.token` directly in `config.yaml` — env var is preferred since it keeps secrets out of version control.)
### 3. Python environment
Via `pipenv` from the repo root:
```bash
pipenv install httpx pyyaml sentence-transformers
```
The embedding model (`sentence-transformers/all-MiniLM-L6-v2`, ~80 MB) downloads on first run.
### 4. Local LLM (default path) — LM Studio
- Install [LM Studio](https://lmstudio.ai/).
- Download `Gemma-4-31B-it` (or another strong reasoning model; adjust `local.model` in `config.yaml`).
- Load it via the CLI for headless use (no GUI required):
```bash
lms load gemma-4-31b-it --context-length 131072 --gpu max -y
lms status # confirm server running on :1234
```
### 5. Cloud LLM backends (optional, opt-in)
Only if you explicitly opt out of local:
- `claude`: set `ANTHROPIC_API_KEY`, flip `backend: claude` in `config.yaml`.
- `foundry`: set `FOUNDRY_API_KEY`, flip `backend: foundry`, set `foundry.endpoint` to your corporate gateway URL.
## Architecture
```
screenpipe daemon (user-installed)
│ HTTP on localhost:3030
▼
scripts/fetch_window.py → normalized timeline events
scripts/redact.py → regex scrub (defense-in-depth)
scripts/cluster.py → sessions + clusters (local only)
scripts/match_skills.py → top-k vs existing 135 skills (local embeddings)
scripts/synthesize.py → LLM judge: reuse / compose / novel
│
▼
~/.autoskill/proposed/<timestamp>/ (default; override with --out)
├── report.md
├── composition-recipes/<name>/SKILL.md
└── new-skills/<name>/SKILL.md
scripts/promote.py → user-approved proposal → skills/<name>/
```
## Workflow
The skill ships a unified CLI at `scripts/autoskill.py` with three subcommands:
```bash
python scripts/autoskill.py doctor --config config.yaml --skills-dir ../
python scripts/autoskill.py run --start ... --end ... --config config.yaml
python scripts/autoskill.py promote --proposed ~/.autoskill/proposed/<ts> --skills-dir ../ --name <skill>
```
### 0. Preflight with `doctor`
Before a full run, verify every dependency in one shot:
```bash
python scripts/autoskill.py doctor \
--config skills/autoskill/config.yaml \
--skills-dir skills
```
The report covers `config` (backend choice valid), `skills_dir` (exists), `screenpipe` (reachable + authed), and `llm` (LM Studio serving or API key present). Non-zero exit on any failure, with the offending line marked `error`.
### 1. Run the pipeline
```bash
export SCREENPIPE_TOKEN=$(screenpipe auth token)
python scripts/autoskill.py run \
--start "2026-04-17T00:00:00Z" \
--end "2026-04-17T23:59:59Z" \
--config skills/autoskill/config.yaml \
--skills-dir skills
```
Proposals land in `~/.autoskill/proposed/<timestamp>/` by default, keeping experimental output out of the skills repo. Pass `--out PATH` to override.
Internally:
1. **Fetch** — `fetch_window` paginates screenpipe's `/search` endpoint, normalizes events to `{ts, app, window_title, text, content_type}`.
2. **Redact** — `redact` scrubs emails, API keys, bearer tokens, phones from OCR text and window titles as defense-in-depth over screenpipe's own PII removal.
3. **Cluster** — `segment_sessions` splits on idle gaps (default 10 min) and drops short sessions; `cluster_sessions` groups sessions by app-signature and keeps clusters of size `min_cluster_size` (default 2).
4. **Match** — `load_skill_descriptions` reads frontmatter from every `SKILL.md` in `skills/`; `top_k_matches` ranks each cluster against all skills using local `sentence-transformers` embeddings (cosine similarity).
5. **Synthesize** — `synthesize` prompts the configured LLM backend to classify each cluster as `reuse`, `compose`, or `novel` and emit a SKILL.md body where appropriate.
6. **Report** — writes `<out_dir>/<ts>/report.md`, plus `new-skills/<name>/SKILL.md` or `composition-recipes/<name>/SKILL.md` for each proposal.
Add `--dry-run` to stop after clustering; this skips the LLM (and the sentence-transformers load), writing only `plan.md` for inspection.
### 2. Review and promote
Open `~/.autoskill/proposed/<ts>/report.md`, edit drafts in place, delete anything you don't want. Then:
```bash
python scripts/autoskill.py promote \
--proposed ~/.autoskill/proposed/2026-04-17T14-30-00 \
--skills-dir skills \
--name zotero-pubmed-helper
```
`promote` moves the directory into `skills/<name>/`, refusing to overwrite an existing skill. Exits non-zero with a friendly error if the proposal isn't found or the target already exists.
## Configuration
See `config.yaml` for the full shape. Default values (local-first):
```yaml
backend: local
local:
endpoint: http://localhost:1234/v1 # LM Studio's Developer server
model: Gemma-4-31B-it
screenpipe:
url: http://localhost:3030 # or https://screenpipe.local via Caddy
cluster:
min_session_minutes: 5
idle_gap_minutes: 10
min_cluster_size: 2
```
To opt into a cloud backend:
```yaml
backend: claude # or foundry
claude:
model: claude-opus-4-7
```
## Composition recipes vs new skills
- **compose**: the LLM judged that chaining existing skills covers the workflow. The emitted SKILL.md is intentionally thin — frontmatter + a "Workflow" section that invokes existing skills in order. The same agent runtime that discovered the skill can then invoke it end-to-end.
- **novel**: no combination of existing skills covers it. A fuller SKILL.md is drafted, still following repo conventions (frontmatter, Overview, When to Use, Workflow). The user should always review new-skill drafts before promoting.
## Testing
The skill is covered by a small pytest suite at `tests/`. Each script is unit-tested in isolation with dependency injection (mock HTTP transport, stub backend, stub embedder):
```bash
cd skills/autoskill
python -m pytest tests/ -v
```
## Composition with other skills in this repo
The autoskill's embedding index covers all 135 sibling skills. Workflows that look like scientific writing will match `scientific-writing` / `literature-review` / `citation-management`; figure work will match `scientific-schematics` / `generate-image` / `infographics`; slide prep matches `scientific-slides` / `pptx`; etc. When a cluster scores high against two or three sibling skills the emitted composition recipe names them explicitly, so the user's future agent invocations use the optimized paths already documented in this repo.
skills/autoskill/config.yaml
+53
View File
@@ -0,0 +1,53 @@
# autoskill configuration
#
# LLM backend for skill synthesis. Detection/clustering always runs locally;
# only redacted cluster summaries are sent to the LLM.
#
# Local is the default — your screen content never leaves the machine.
# Cloud backends (claude, foundry) are available for users who explicitly
# opt in; see the backend sections below.
backend: local # local | claude | foundry
# Per-backend settings. Only the selected backend's block is used.
local:
# LM Studio exposes an OpenAI-compatible server. Start it from the
# "Developer" tab; the default port is 1234.
endpoint: http://localhost:1234/v1
# Gemma-4-31B-it is the recommended default: strong reasoning at a size
# most modern workstation GPUs can run. Swap for any LM Studio model ID.
model: Gemma-4-31B-it
claude:
model: claude-opus-4-7
# api_key read from ANTHROPIC_API_KEY env var
foundry:
endpoint: https://foundry.example.com/anthropic
model: claude-opus-4-7
# api_key read from FOUNDRY_API_KEY env var
# Screenpipe HTTP endpoint. For TLS, point this at your local Caddy proxy
# (see references/https-proxy.md).
screenpipe:
url: http://localhost:3030
# Screenpipe requires a bearer token for its local API. Either set `token`
# here, or export SCREENPIPE_TOKEN in your environment (preferred — keeps
# the token out of version control). Retrieve with: `screenpipe auth token`.
# token: your-token-here
# Embedding model for matching against existing scientific skills.
# Local only; no API calls.
embeddings:
model: sentence-transformers/all-MiniLM-L6-v2
# Clustering thresholds.
cluster:
min_session_minutes: 5 # skip sessions shorter than this
idle_gap_minutes: 10 # new session after this much inactivity
min_cluster_size: 2 # need this many similar sessions before proposing
# Content redaction regexes applied before any cluster summary leaves
# the local detection layer. Defense-in-depth on top of screenpipe's
# own app/window filtering.
redaction:
enabled: true
skills/autoskill/references/https-proxy.md
+62
View File
@@ -0,0 +1,62 @@
# Optional: TLS for localhost screenpipe access
Screenpipe's HTTP server (Axum, binding `localhost:3030`) speaks plain HTTP. For a Python script running as the same user on the same host, plain HTTP is adequate — loopback traffic never hits a network adapter, so TLS provides no additional confidentiality.
TLS on localhost is only useful when:
- A corporate security policy mandates "TLS everywhere" regardless of transport.
- The screenpipe endpoint is tunneled or exposed off-host.
- A browser client requires a "secure context" (Service Workers, WebCrypto).
If you need it, put a one-line Caddy reverse proxy in front. Caddy's `tls internal` generates and trusts a local CA automatically.
## Caddy
Install:
```bash
brew install caddy # macOS
# or see https://caddyserver.com/docs/install
```
Add to your `Caddyfile`:
```caddyfile
screenpipe.local {
tls internal
reverse_proxy localhost:3030
}
```
Ensure `screenpipe.local` resolves to loopback (add to `/etc/hosts`):
```
127.0.0.1 screenpipe.local
```
Start Caddy:
```bash
caddy run
```
Then update autoskill's `config.yaml`:
```yaml
screenpipe:
url: https://screenpipe.local
```
No code change is required on the autoskill side. `httpx` handles both HTTP and HTTPS transparently.
## mkcert (alternative)
If you prefer managing the cert yourself instead of Caddy's internal CA:
```bash
brew install mkcert
mkcert -install
mkcert localhost 127.0.0.1
```
Then terminate TLS with nginx, Caddy, or stunnel using the generated cert.
skills/autoskill/references/screenpipe-config.yaml
+61
View File
@@ -0,0 +1,61 @@
# Starter screenpipe configuration for autoskill users.
#
# Copy the relevant sections into your screenpipe config (or pass as CLI
# flags when starting the daemon). Screenpipe handles app/window filtering
# at capture time — sensitive apps on this list will never be OCR'd, so
# their content never reaches autoskill's pipeline.
#
# Reference: https://github.com/screenpipe/screenpipe
#
# Review and edit for your setup. This is a conservative baseline, not
# an exhaustive list. Add any app where you handle secrets.
ignored_apps:
# Password managers
- "1Password"
- "Bitwarden"
- "Dashlane"
- "Keeper"
- "LastPass"
- "KeePassXC"
# Private messaging
- "Signal"
- "Telegram"
- "WhatsApp"
- "iMessage"
- "Messages"
# Mail composition (inbound reading is fine; composition often contains secrets)
# Remove if you explicitly want mail workflows analyzed.
- "Mail"
- "Outlook"
# Banking / finance apps (add yours)
# - "Bank of America"
# - "Chase"
# Window title globs to ignore (matched across all apps).
# Useful for catching sensitive URLs in browsers that are otherwise OK to observe.
ignored_windows:
- "*Bitwarden*"
- "*1Password*"
- "*login*"
- "*Sign in*"
- "*Private Browsing*"
- "*Incognito*"
- "*online banking*"
- "*bank*"
- "*account settings*"
# Optionally restrict capture to specific hours (local time).
# Uncomment to apply.
# hours:
# start: "09:00"
# end: "19:00"
# Content types to capture. Drop "audio" if you don't want transcripts.
content_types:
- ocr
- ui
# - audio
skills/autoskill/scripts/autoskill.py
+35
View File
@@ -0,0 +1,35 @@
"""Unified CLI for the autoskill skill.
Subcommands:
run — detect workflows and draft proposed skills
doctor — verify screenpipe + LM Studio + config + skills dir
promote — move an approved proposal into skills/
"""
import argparse
import sys
def main(argv=None):
parser = argparse.ArgumentParser(prog="autoskill", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("command", choices=["run", "doctor", "promote"],
help="subcommand to run")
parser.add_argument("rest", nargs=argparse.REMAINDER,
help="arguments forwarded to the subcommand")
args = parser.parse_args(argv)
if args.command == "run":
import run as _run
return _run.main(args.rest)
if args.command == "doctor":
import doctor as _doctor
return _doctor.main(args.rest)
if args.command == "promote":
import promote as _promote
return _promote.main(args.rest)
raise AssertionError(f"unreachable: {args.command!r}")
if __name__ == "__main__":
sys.exit(main())
skills/autoskill/scripts/backends.py
+71
View File
@@ -0,0 +1,71 @@
import os
import httpx
class ClaudeBackend:
def __init__(self, api_key, model, client=None):
self.api_key = api_key
self.model = model
self.client = client or httpx.Client(base_url="https://api.anthropic.com", timeout=60.0)
def __call__(self, prompt):
response = self.client.post(
"/v1/messages",
headers={
"x-api-key": self.api_key,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
},
json={
"model": self.model,
"max_tokens": 4096,
"messages": [{"role": "user", "content": prompt}],
},
)
response.raise_for_status()
payload = response.json()
return payload["content"][0]["text"]
class LocalBackend:
def __init__(self, endpoint, model, client=None):
self.endpoint = endpoint
self.model = model
self.client = client or httpx.Client(base_url=endpoint, timeout=120.0)
def __call__(self, prompt):
response = self.client.post(
"/chat/completions",
json={
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
},
)
response.raise_for_status()
payload = response.json()
return payload["choices"][0]["message"]["content"]
def make_backend(config):
kind = config.get("backend")
if kind == "claude":
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
raise RuntimeError("ANTHROPIC_API_KEY environment variable not set")
model = config.get("claude", {}).get("model", "claude-opus-4-7")
return ClaudeBackend(api_key=api_key, model=model)
if kind == "foundry":
api_key = os.environ.get("FOUNDRY_API_KEY")
if not api_key:
raise RuntimeError("FOUNDRY_API_KEY environment variable not set")
f = config.get("foundry", {})
client = httpx.Client(base_url=f["endpoint"], timeout=60.0)
return ClaudeBackend(api_key=api_key, model=f.get("model", "claude-opus-4-7"), client=client)
if kind == "local":
l = config.get("local", {})
return LocalBackend(endpoint=l["endpoint"], model=l["model"])
raise ValueError(f"unknown backend: {kind!r}")
skills/autoskill/scripts/cluster.py
+54
View File
@@ -0,0 +1,54 @@
from collections import defaultdict
def segment_sessions(events, idle_gap_seconds, min_session_seconds):
if not events:
return []
events = sorted(events, key=lambda e: e["ts"])
groups = [[events[0]]]
for prev, curr in zip(events, events[1:]):
if curr["ts"] - prev["ts"] > idle_gap_seconds:
groups.append([curr])
else:
groups[-1].append(curr)
sessions = []
for group in groups:
duration = group[-1]["ts"] - group[0]["ts"]
if duration < min_session_seconds:
continue
apps, seen = [], set()
for evt in group:
if evt["app"] not in seen:
seen.add(evt["app"])
apps.append(evt["app"])
sessions.append({
"start_ts": group[0]["ts"],
"end_ts": group[-1]["ts"],
"duration_seconds": duration,
"apps": apps,
"window_titles": [e["window_title"] for e in group if e.get("window_title")],
})
return sessions
def cluster_sessions(sessions, min_cluster_size):
buckets = defaultdict(list)
for s in sessions:
buckets[tuple(s["apps"])].append(s)
clusters = []
for apps, members in buckets.items():
if len(members) < min_cluster_size:
continue
example_titles = []
for m in members:
if m["window_titles"]:
example_titles.append(m["window_titles"][0])
clusters.append({
"apps": list(apps),
"session_count": len(members),
"total_duration_seconds": sum(m["duration_seconds"] for m in members),
"example_titles": example_titles,
})
return clusters
skills/autoskill/scripts/doctor.py
+108
View File
@@ -0,0 +1,108 @@
import argparse
import os
import sys
from pathlib import Path
import httpx
_VALID_BACKENDS = {"local", "claude", "foundry"}
def default_screenpipe_probe(config):
sp = config.get("screenpipe", {})
url = sp.get("url", "http://localhost:3030")
token = sp.get("token") or os.environ.get("SCREENPIPE_TOKEN")
headers = {"Authorization": f"Bearer {token}"} if token else {}
try:
r = httpx.get(f"{url}/health", headers=headers, timeout=5.0)
if r.status_code == 200:
return ("ok", url)
return ("error", f"{url} returned HTTP {r.status_code}")
except httpx.HTTPError as e:
return ("error", f"{url}: {e}")
def default_llm_probe(config):
kind = config.get("backend")
if kind == "local":
endpoint = config.get("local", {}).get("endpoint", "http://localhost:1234/v1")
try:
r = httpx.get(f"{endpoint}/models", timeout=5.0)
if r.status_code == 200:
return ("ok", endpoint)
return ("error", f"{endpoint} returned HTTP {r.status_code}")
except httpx.HTTPError as e:
return ("error", f"{endpoint}: {e}")
if kind == "claude":
if not os.environ.get("ANTHROPIC_API_KEY"):
return ("error", "ANTHROPIC_API_KEY not set")
return ("ok", "ANTHROPIC_API_KEY present (not probed)")
if kind == "foundry":
if not os.environ.get("FOUNDRY_API_KEY"):
return ("error", "FOUNDRY_API_KEY not set")
return ("ok", "FOUNDRY_API_KEY present (not probed)")
return ("error", f"unknown backend: {kind!r}")
def check(config, *, skills_dir, screenpipe_probe, llm_probe):
result = {}
kind = config.get("backend")
if kind in _VALID_BACKENDS:
result["config"] = ("ok", f"backend={kind}")
else:
result["config"] = ("error", f"unknown backend: {kind!r}")
skills_dir = Path(skills_dir)
if skills_dir.is_dir():
result["skills_dir"] = ("ok", str(skills_dir))
else:
result["skills_dir"] = ("error", f"not a directory: {skills_dir}")
result["screenpipe"] = screenpipe_probe(config)
result["llm"] = llm_probe(config)
return result
def _format_report(result: dict) -> str:
lines = ["autoskill doctor", "================"]
for key in ("config", "skills_dir", "screenpipe", "llm"):
status, detail = result[key]
lines.append(f" {key:12s}: {status:5s} {detail}")
return "\n".join(lines)
def main(argv=None):
import yaml
parser = argparse.ArgumentParser(
prog="autoskill-doctor",
description="Check that screenpipe, LM Studio, config, and skills dir are ready.",
)
parser.add_argument("--config", required=True,
help="path to autoskill config.yaml")
parser.add_argument("--skills-dir", required=True,
help="path to skills/")
args = parser.parse_args(argv)
config = yaml.safe_load(Path(args.config).read_text())
result = check(
config,
skills_dir=args.skills_dir,
screenpipe_probe=default_screenpipe_probe,
llm_probe=default_llm_probe,
)
report = _format_report(result)
print(report)
any_error = any(status == "error" for status, _ in result.values())
if any_error:
print("\ndoctor: one or more checks failed", file=sys.stderr)
return 1
print("\ndoctor: all checks passed")
return 0
if __name__ == "__main__":
raise SystemExit(main())
skills/autoskill/scripts/fetch_window.py
+33
View File
@@ -0,0 +1,33 @@
_MAX_PAGES = 10_000 # bounded exit: hard ceiling so the loop cannot spin forever
def fetch_window(client, start_time, end_time, page_size=50, token=None):
events = []
offset = 0
headers = {"Authorization": f"Bearer {token}"} if token else {}
for _page in range(_MAX_PAGES):
response = client.get("/search", params={
"start_time": start_time,
"end_time": end_time,
"limit": page_size,
"offset": offset,
}, headers=headers)
response.raise_for_status()
payload = response.json()
data = payload.get("data", [])
total = payload.get("pagination", {}).get("total", 0)
for item in data:
content = item.get("content", {})
events.append({
"ts": content.get("timestamp"),
"app": content.get("app_name", ""),
"window_title": content.get("window_name", ""),
"text": content.get("text", ""),
"content_type": item.get("type", "").lower(),
})
offset += len(data)
if not data or offset >= total:
break
return events
skills/autoskill/scripts/match_skills.py
+46
View File
@@ -0,0 +1,46 @@
import math
from pathlib import Path
def _parse_frontmatter(content: str) -> dict:
if not content.startswith("---"):
return {}
_, _, rest = content.partition("---\n")
block, _, _ = rest.partition("\n---")
out = {}
for line in block.splitlines():
if ":" not in line:
continue
key, _, value = line.partition(":")
out[key.strip()] = value.strip()
return out
def load_skill_descriptions(skills_dir):
skills_dir = Path(skills_dir)
skills = []
for skill_md in sorted(skills_dir.glob("*/SKILL.md")):
fm = _parse_frontmatter(skill_md.read_text())
if "name" in fm and "description" in fm:
skills.append({"name": fm["name"], "description": fm["description"]})
return skills
def _cosine(a, b):
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
if na == 0 or nb == 0:
return 0.0
return dot / (na * nb)
def top_k_matches(query, skills, embedder, k):
q = embedder(query)
scored = [
{"name": s["name"], "description": s["description"],
"score": _cosine(q, embedder(s["description"]))}
for s in skills
]
scored.sort(key=lambda r: r["score"], reverse=True)
return scored[:k]
skills/autoskill/scripts/promote.py
+58
View File
@@ -0,0 +1,58 @@
import argparse
import shutil
import sys
from pathlib import Path
class PromoteError(Exception):
pass
_KINDS = ("new-skills", "composition-recipes")
def promote(proposed_path, skills_dir, name):
proposed_path = Path(proposed_path)
skills_dir = Path(skills_dir)
source = None
for kind in _KINDS:
candidate = proposed_path / kind / name
if candidate.is_dir():
source = candidate
break
if source is None:
raise PromoteError(f"proposed skill {name!r} not found under {proposed_path}")
target = skills_dir / name
if target.exists():
raise PromoteError(f"target {target} already exists")
shutil.move(str(source), str(target))
return target
def main(argv=None):
parser = argparse.ArgumentParser(
prog="autoskill-promote",
description="Move a proposed skill from _proposed/<ts>/ into skills/",
)
parser.add_argument("--proposed", required=True,
help="path to the _proposed/<ts>/ directory")
parser.add_argument("--skills-dir", required=True,
help="path to skills/")
parser.add_argument("--name", required=True, help="skill name to promote")
args = parser.parse_args(argv)
try:
target = promote(args.proposed, args.skills_dir, args.name)
except PromoteError as e:
print(f"promote failed: {e}", file=sys.stderr)
return 1
print(f"promoted: {args.name} -> {target}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
skills/autoskill/scripts/redact.py
+40
View File
@@ -0,0 +1,40 @@
import re
# Order matters: multi-line and prefixed patterns run before narrower ones.
_PATTERNS = [
(re.compile(r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+PRIVATE KEY-----"),
"[REDACTED:private_key]"),
# Known-env-var secret assignments: NAME=value (catches long values only)
(re.compile(
r"\b(?:AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID|GITHUB_TOKEN|HF_TOKEN"
r"|ANTHROPIC_API_KEY|OPENAI_API_KEY|FOUNDRY_API_KEY|SCREENPIPE_TOKEN"
r"|GOOGLE_API_KEY|SLACK_TOKEN|DEEPGRAM_API_KEY)"
r"\s*=\s*[^\s\"']+"
), "[REDACTED:kv_secret]"),
(re.compile(r"Bearer\s+[A-Za-z0-9_\-\.=]+"), "[REDACTED:bearer]"),
(re.compile(r"\beyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+"),
"[REDACTED:jwt]"),
(re.compile(r"\bxox[bpars]-[A-Za-z0-9\-]{10,}"), "[REDACTED:api_key]"),
(re.compile(r"\bhf_[A-Za-z0-9]{32,}"), "[REDACTED:api_key]"),
(re.compile(r"\bsk-[A-Za-z0-9_\-]{20,}"), "[REDACTED:api_key]"),
(re.compile(r"\b(?:sk|pk|rk)_live_[A-Za-z0-9]{24,}"), "[REDACTED:api_key]"),
(re.compile(r"\bghp_[A-Za-z0-9]{36}\b"), "[REDACTED:api_key]"),
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED:api_key]"),
(re.compile(r"\bAIza[A-Za-z0-9_\-]{35}\b"), "[REDACTED:api_key]"),
(re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}"),
"[REDACTED:email]"),
(re.compile(r"\(\d{3}\)\s*\d{3}-\d{4}"), "[REDACTED:phone]"),
(re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[REDACTED:ssn]"),
]
def redact(text: str) -> str:
for pattern, placeholder in _PATTERNS:
text = pattern.sub(placeholder, text)
return text
skills/autoskill/scripts/run.py
+194
View File
@@ -0,0 +1,194 @@
import datetime as _dt
from pathlib import Path
import httpx
from cluster import cluster_sessions, segment_sessions
from fetch_window import fetch_window
from match_skills import load_skill_descriptions, top_k_matches
from redact import redact
from synthesize import synthesize
class ScreenpipeUnreachable(RuntimeError):
"""Raised when the screenpipe daemon cannot be reached.
The autoskill skill cannot run without screenpipe. Install it from
https://github.com/screenpipe/screenpipe and start the daemon before
invoking this skill.
"""
def _default_now() -> str:
return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")
def _cluster_query(cluster: dict) -> str:
parts = ["apps: " + ", ".join(cluster["apps"])]
if cluster.get("example_titles"):
parts.append("titles: " + "; ".join(cluster["example_titles"]))
return " | ".join(parts)
def _write_plan(proposed_path: Path, clusters: list[dict]) -> None:
lines = ["# Dry-run plan", ""]
for i, c in enumerate(clusters, 1):
lines += [
f"## Cluster {i}",
f"- apps: {', '.join(c['apps'])}",
f"- sessions: {c['session_count']}",
f"- total_duration_seconds: {c['total_duration_seconds']}",
f"- example titles: {'; '.join(c.get('example_titles', []))}",
"",
]
(proposed_path / "plan.md").write_text("\n".join(lines))
def _write_report(proposed_path: Path, results: list[dict]) -> None:
lines = ["# autoskill report", ""]
if not results:
lines.append("No clusters met the minimum size threshold. Nothing to propose.")
for r in results:
c = r["cluster"]
lines += [
f"## {', '.join(c['apps'])} — {c['session_count']}× ({c['total_duration_seconds']}s)",
f"- verdict: **{r['verdict']}**",
]
if r["verdict"] == "reuse":
lines.append(f"- matched skill: `{r['target']}`")
else:
lines.append(f"- draft: `{r['draft_path']}`")
lines.append("- top matches:")
for s in r["top_k"]:
lines.append(f" - `{s['name']}` (score={s['score']:.2f})")
lines.append("")
(proposed_path / "report.md").write_text("\n".join(lines))
def run(config, *, start_time, end_time, out_dir,
screenpipe_client, backend, embedder, skills_dir,
screenpipe_token=None, now=None, dry_run=False):
now = now or _default_now
try:
events = fetch_window(screenpipe_client, start_time, end_time,
token=screenpipe_token)
except (httpx.ConnectError, httpx.ConnectTimeout) as e:
base = getattr(screenpipe_client, "base_url", "http://localhost:3030")
raise ScreenpipeUnreachable(
f"cannot reach screenpipe at {base}: {e}. "
"Install and start the daemon — see "
"https://github.com/screenpipe/screenpipe — "
"or point config.yaml's screenpipe.url at your instance."
) from e
for e in events:
e["text"] = redact(e.get("text", ""))
e["window_title"] = redact(e.get("window_title", ""))
cluster_cfg = config.get("cluster", {})
idle_gap = cluster_cfg.get("idle_gap_minutes", 10) * 60
min_session = cluster_cfg.get("min_session_minutes", 5) * 60
min_cluster = cluster_cfg.get("min_cluster_size", 2)
# fetch_window returns ts as ISO strings; convert to epoch for segmentation
for e in events:
if isinstance(e["ts"], str):
e["ts"] = int(_dt.datetime.fromisoformat(e["ts"].replace("Z", "+00:00")).timestamp())
sessions = segment_sessions(events, idle_gap_seconds=idle_gap, min_session_seconds=min_session)
clusters = cluster_sessions(sessions, min_cluster_size=min_cluster)
proposed_path = Path(out_dir) / now()
proposed_path.mkdir(parents=True, exist_ok=True)
if dry_run:
_write_plan(proposed_path, clusters)
return proposed_path
if not clusters:
_write_report(proposed_path, [])
return proposed_path
skills = load_skill_descriptions(Path(skills_dir))
results = []
for cluster in clusters:
query = _cluster_query(cluster)
top_k = top_k_matches(query, skills, embedder=embedder, k=5)
decision = synthesize(cluster, top_k, backend=backend)
entry = {"cluster": cluster, "top_k": top_k, "verdict": decision["verdict"]}
if decision["verdict"] == "reuse":
entry["target"] = decision.get("target")
else:
kind = "new-skills" if decision["verdict"] == "novel" else "composition-recipes"
name = decision["name"]
draft_dir = proposed_path / kind / name
draft_dir.mkdir(parents=True)
(draft_dir / "SKILL.md").write_text(decision["skill_body"])
entry["draft_path"] = str(draft_dir.relative_to(proposed_path))
results.append(entry)
_write_report(proposed_path, results)
return proposed_path
def main(argv=None):
import argparse
import sys
import httpx
import yaml
from backends import make_backend
parser = argparse.ArgumentParser(prog="autoskill")
parser.add_argument("--start", required=True, help="ISO start time, e.g. 2026-04-17T00:00:00Z")
parser.add_argument("--end", required=True, help="ISO end time")
parser.add_argument("--config", default=str(Path(__file__).resolve().parent.parent / "config.yaml"))
parser.add_argument("--out", default=None,
help="output directory for proposals (default: ~/.autoskill/proposed)")
parser.add_argument("--skills-dir", default=None,
help="path to skills/ (default: parent of this skill's dir)")
parser.add_argument("--dry-run", action="store_true",
help="stop after clustering; do not call the LLM backend")
args = parser.parse_args(argv)
config = yaml.safe_load(Path(args.config).read_text())
here = Path(__file__).resolve()
skills_dir = Path(args.skills_dir) if args.skills_dir else here.parent.parent.parent
out_dir = Path(args.out) if args.out else Path.home() / ".autoskill" / "proposed"
import os
screenpipe_cfg = config.get("screenpipe", {})
screenpipe_url = screenpipe_cfg.get("url", "http://localhost:3030")
screenpipe_token = (screenpipe_cfg.get("token")
or os.environ.get("SCREENPIPE_TOKEN"))
screenpipe_client = httpx.Client(base_url=screenpipe_url, timeout=60.0)
if args.dry_run:
backend = None
embedder = None
else:
backend = make_backend(config)
from sentence_transformers import SentenceTransformer
model = SentenceTransformer(
config.get("embeddings", {}).get("model", "sentence-transformers/all-MiniLM-L6-v2")
)
def embedder(text: str):
return list(map(float, model.encode(text)))
proposed = run(
config,
start_time=args.start, end_time=args.end, out_dir=out_dir,
screenpipe_client=screenpipe_client, backend=backend, embedder=embedder,
skills_dir=skills_dir, screenpipe_token=screenpipe_token,
dry_run=args.dry_run,
)
print(f"proposals written to: {proposed}", file=sys.stderr)
return 0
if __name__ == "__main__":
raise SystemExit(main())
skills/autoskill/scripts/synthesize.py
+72
View File
@@ -0,0 +1,72 @@
import json
import re
VALID_VERDICTS = {"reuse", "compose", "novel"}
class SynthesisError(Exception):
pass
def _build_prompt(cluster, top_k_skills):
apps = ", ".join(cluster["apps"])
titles = "; ".join(cluster.get("example_titles", []))
candidates = "\n".join(
f"- {s['name']} (score={s['score']:.2f}): {s['description']}"
for s in top_k_skills
)
return f"""You are classifying an observed user workflow against an existing skill library.
Cluster:
- apps: {apps}
- sessions: {cluster['session_count']}
- total_duration_seconds: {cluster['total_duration_seconds']}
- example titles: {titles}
Candidate existing skills (ranked by semantic similarity):
{candidates}
Decide one of:
- "reuse": an existing skill already covers this workflow.
- "compose": no single skill covers it, but chaining existing skills does. Draft a thin SKILL.md that invokes them in order.
- "novel": not covered; draft a full new SKILL.md.
Respond with a single JSON object:
- reuse: {{"verdict": "reuse", "target": "<skill-name>"}}
- compose: {{"verdict": "compose", "name": "<new-skill-name>", "skill_body": "<SKILL.md body>"}}
- novel: {{"verdict": "novel", "name": "<new-skill-name>", "skill_body": "<SKILL.md body>"}}
"""
def _extract_json(text: str) -> dict:
fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
if fence:
candidate = fence.group(1)
else:
start = text.find("{")
end = text.rfind("}")
if start == -1 or end == -1 or end <= start:
raise SynthesisError(f"no JSON object found in response: {text!r}")
candidate = text[start:end + 1]
try:
return json.loads(candidate)
except json.JSONDecodeError as e:
raise SynthesisError(f"invalid JSON in response: {e}") from e
def synthesize(cluster, top_k_skills, backend):
prompt = _build_prompt(cluster, top_k_skills)
response = backend(prompt)
payload = _extract_json(response)
verdict = payload.get("verdict")
if verdict not in VALID_VERDICTS:
raise SynthesisError(f"unknown verdict: {verdict!r}")
result = {"verdict": verdict, "skill_body": None}
if verdict == "reuse":
result["target"] = payload.get("target")
else:
result["name"] = payload.get("name")
result["skill_body"] = payload.get("skill_body")
return result
skills/autoskill/tests/conftest.py
+4
View File
@@ -0,0 +1,4 @@
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
skills/autoskill/tests/smoke_lmstudio.py
+60
View File
@@ -0,0 +1,60 @@
"""Real smoke test against a live LM Studio server.
Not part of the pytest suite — requires LM Studio running on localhost:1234
with Gemma-4-31B-it loaded. Run manually:
pipenv run python skills/autoskill/tests/smoke_lmstudio.py
"""
import json
import sys
from pathlib import Path
THIS_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(THIS_DIR.parent / "scripts"))
from backends import LocalBackend
from match_skills import load_skill_descriptions, top_k_matches
from synthesize import synthesize
def main() -> int:
backend = LocalBackend(endpoint="http://localhost:1234/v1", model="gemma-4-31b-it")
repo_skills_dir = THIS_DIR.parent.parent
all_skills = load_skill_descriptions(repo_skills_dir)
print(f"loaded {len(all_skills)} skills from {repo_skills_dir}")
# Real sentence-transformers embedder.
print("loading sentence-transformers/all-MiniLM-L6-v2 ...")
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
def embedder(text: str):
return list(map(float, model.encode(text)))
cluster = {
"apps": ["Chrome", "Zotero"],
"session_count": 4,
"total_duration_seconds": 7200,
"example_titles": ["PubMed search: tumor microenvironment", "bioRxiv preprint"],
}
query = ("apps: " + ", ".join(cluster["apps"]) +
" | titles: " + "; ".join(cluster["example_titles"]))
top_k = top_k_matches(query, all_skills, embedder=embedder, k=5)
print("real top-5 matches from embedding search:")
for s in top_k:
print(f" {s['score']:.3f} {s['name']}")
result = synthesize(cluster, top_k, backend=backend)
print("\n--- VERDICT ---")
print(json.dumps(result, indent=2)[:1000])
assert result["verdict"] in {"reuse", "compose", "novel"}, result
print("\nOK: real sentence-transformers top-k + real Gemma-4-31B-it produced a valid verdict.")
return 0
if __name__ == "__main__":
sys.exit(main())
skills/autoskill/tests/test_backends.py
+121
View File
@@ -0,0 +1,121 @@
import httpx
import pytest
from backends import ClaudeBackend, LocalBackend, make_backend
def _mock_client(handler, base_url=""):
return httpx.Client(transport=httpx.MockTransport(handler), base_url=base_url)
def test_claude_backend_posts_to_v1_messages_with_auth_headers():
calls = []
def handler(request):
calls.append(request)
return httpx.Response(200, json={"content": [{"type": "text", "text": "hello"}]})
backend = ClaudeBackend(api_key="sk-test", model="claude-opus-4-7",
client=_mock_client(handler, base_url="https://api.anthropic.com"))
result = backend("say hello")
assert result == "hello"
assert calls[0].url.path == "/v1/messages"
assert calls[0].headers["x-api-key"] == "sk-test"
assert calls[0].headers["anthropic-version"]
body = calls[0].read()
import json as _json
payload = _json.loads(body)
assert payload["model"] == "claude-opus-4-7"
assert payload["messages"][0]["role"] == "user"
assert payload["messages"][0]["content"] == "say hello"
def test_claude_backend_honors_custom_base_url_for_foundry():
calls = []
def handler(request):
calls.append(request)
return httpx.Response(200, json={"content": [{"type": "text", "text": "ok"}]})
backend = ClaudeBackend(api_key="k", model="claude-opus-4-7",
client=_mock_client(handler, base_url="https://foundry.example.com/anthropic"))
backend("hi")
assert str(calls[0].url).startswith("https://foundry.example.com/anthropic/v1/messages")
def test_claude_backend_raises_on_http_error():
def handler(request):
return httpx.Response(401, json={"error": "bad key"})
backend = ClaudeBackend(api_key="k", model="m",
client=_mock_client(handler, base_url="https://api.anthropic.com"))
with pytest.raises(httpx.HTTPStatusError):
backend("x")
def test_local_backend_posts_to_chat_completions():
calls = []
def handler(request):
calls.append(request)
return httpx.Response(200, json={
"choices": [{"message": {"content": "hi back"}}]
})
backend = LocalBackend(endpoint="http://localhost:11434/v1", model="qwen2.5:14b",
client=_mock_client(handler, base_url="http://localhost:11434/v1"))
result = backend("prompt")
assert result == "hi back"
assert calls[0].url.path.endswith("/chat/completions")
import json as _json
payload = _json.loads(calls[0].read())
assert payload["model"] == "qwen2.5:14b"
assert payload["messages"][0]["content"] == "prompt"
def test_make_backend_returns_claude_backend_for_claude_config(monkeypatch):
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-env")
backend = make_backend({
"backend": "claude",
"claude": {"model": "claude-opus-4-7"},
})
assert isinstance(backend, ClaudeBackend)
assert backend.api_key == "sk-env"
assert backend.model == "claude-opus-4-7"
def test_make_backend_returns_claude_backend_with_foundry_base_url(monkeypatch):
monkeypatch.setenv("FOUNDRY_API_KEY", "sk-foundry")
backend = make_backend({
"backend": "foundry",
"foundry": {
"endpoint": "https://foundry.example.com/anthropic",
"model": "claude-opus-4-7",
},
})
assert isinstance(backend, ClaudeBackend)
assert backend.api_key == "sk-foundry"
assert "foundry.example.com" in str(backend.client.base_url)
def test_make_backend_returns_local_backend_for_local_config():
backend = make_backend({
"backend": "local",
"local": {"endpoint": "http://localhost:11434/v1", "model": "qwen2.5:14b"},
})
assert isinstance(backend, LocalBackend)
assert backend.model == "qwen2.5:14b"
def test_make_backend_raises_on_unknown_backend():
with pytest.raises(ValueError, match="unknown backend"):
make_backend({"backend": "mystery"})
def test_make_backend_raises_when_anthropic_key_missing(monkeypatch):
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
with pytest.raises(RuntimeError, match="ANTHROPIC_API_KEY"):
make_backend({"backend": "claude", "claude": {"model": "m"}})
skills/autoskill/tests/test_cli.py
+61
View File
@@ -0,0 +1,61 @@
import pytest
import autoskill
def test_dispatches_to_run_subcommand(monkeypatch):
calls = {}
def fake_run_main(argv):
calls["run"] = argv
return 0
monkeypatch.setattr("run.main", fake_run_main)
rc = autoskill.main(["run", "--start", "a", "--end", "b"])
assert rc == 0
assert calls["run"] == ["--start", "a", "--end", "b"]
def test_dispatches_to_doctor_subcommand(monkeypatch):
calls = {}
def fake_doctor_main(argv):
calls["doctor"] = argv
return 0
monkeypatch.setattr("doctor.main", fake_doctor_main)
rc = autoskill.main(["doctor", "--config", "x", "--skills-dir", "y"])
assert rc == 0
assert calls["doctor"] == ["--config", "x", "--skills-dir", "y"]
def test_dispatches_to_promote_subcommand(monkeypatch):
calls = {}
def fake_promote_main(argv):
calls["promote"] = argv
return 0
monkeypatch.setattr("promote.main", fake_promote_main)
rc = autoskill.main(["promote", "--proposed", "p", "--skills-dir", "s", "--name", "n"])
assert rc == 0
assert calls["promote"] == ["--proposed", "p", "--skills-dir", "s", "--name", "n"]
def test_propagates_nonzero_exit_code(monkeypatch):
monkeypatch.setattr("run.main", lambda argv: 7)
rc = autoskill.main(["run", "--start", "a", "--end", "b"])
assert rc == 7
def test_missing_subcommand_errors_out():
with pytest.raises(SystemExit):
autoskill.main([])
def test_unknown_subcommand_errors_out():
with pytest.raises(SystemExit):
autoskill.main(["nope"])
skills/autoskill/tests/test_cluster.py
+67
View File
@@ -0,0 +1,67 @@
from cluster import segment_sessions, cluster_sessions
def _evt(ts: int, app: str, title: str = "", text: str = "") -> dict:
return {"ts": ts, "app": app, "window_title": title, "text": text}
def test_contiguous_events_form_one_session():
events = [_evt(i * 30, "VSCode") for i in range(20)] # 10 minutes
sessions = segment_sessions(events, idle_gap_seconds=600, min_session_seconds=300)
assert len(sessions) == 1
assert sessions[0]["apps"] == ["VSCode"]
assert sessions[0]["duration_seconds"] == 19 * 30
def test_idle_gap_splits_sessions():
left = [_evt(i * 30, "Chrome") for i in range(20)] # 0..570
right = [_evt(10000 + i * 30, "Chrome") for i in range(20)] # gap of ~9400s
sessions = segment_sessions(left + right, idle_gap_seconds=600, min_session_seconds=300)
assert len(sessions) == 2
def test_sessions_below_minimum_are_dropped():
events = [_evt(i * 10, "Finder") for i in range(5)] # 40 seconds total
sessions = segment_sessions(events, idle_gap_seconds=600, min_session_seconds=300)
assert sessions == []
def test_session_records_distinct_apps_in_order_of_first_appearance():
events = [
_evt(0, "Chrome"), _evt(30, "Chrome"),
_evt(60, "Zotero"), _evt(90, "Zotero"),
_evt(120, "VSCode"), _evt(600, "VSCode"),
]
sessions = segment_sessions(events, idle_gap_seconds=600, min_session_seconds=300)
assert len(sessions) == 1
assert sessions[0]["apps"] == ["Chrome", "Zotero", "VSCode"]
def test_clusters_group_sessions_with_identical_app_signatures():
sessions = [
{"apps": ["Chrome", "Zotero"], "duration_seconds": 600, "window_titles": ["PubMed"]},
{"apps": ["Chrome", "Zotero"], "duration_seconds": 800, "window_titles": ["bioRxiv"]},
{"apps": ["VSCode"], "duration_seconds": 900, "window_titles": ["paper.tex"]},
]
clusters = cluster_sessions(sessions, min_cluster_size=2)
assert len(clusters) == 1
assert clusters[0]["apps"] == ["Chrome", "Zotero"]
assert clusters[0]["session_count"] == 2
def test_cluster_summary_exposes_total_duration_and_example_titles():
sessions = [
{"apps": ["Chrome"], "duration_seconds": 300, "window_titles": ["arXiv"]},
{"apps": ["Chrome"], "duration_seconds": 500, "window_titles": ["Nature"]},
]
clusters = cluster_sessions(sessions, min_cluster_size=2)
assert clusters[0]["total_duration_seconds"] == 800
assert set(clusters[0]["example_titles"]) == {"arXiv", "Nature"}
def test_singleton_clusters_are_dropped_when_below_min_cluster_size():
sessions = [
{"apps": ["Slack"], "duration_seconds": 400, "window_titles": ["general"]},
]
clusters = cluster_sessions(sessions, min_cluster_size=2)
assert clusters == []
skills/autoskill/tests/test_doctor.py
+108
View File
@@ -0,0 +1,108 @@
from pathlib import Path
from doctor import check, main as doctor_main
def _config(tmp_path: Path) -> dict:
return {
"backend": "local",
"local": {"endpoint": "http://localhost:1234/v1", "model": "m"},
"screenpipe": {"url": "http://localhost:3030"},
}
def _ok_probe(*_args, **_kwargs):
return ("ok", "")
def _err_probe(*_args, **_kwargs):
return ("error", "boom")
def test_check_all_green(tmp_path: Path):
result = check(
_config(tmp_path),
skills_dir=tmp_path,
screenpipe_probe=_ok_probe,
llm_probe=_ok_probe,
)
assert result["screenpipe"] == ("ok", "")
assert result["llm"] == ("ok", "")
assert result["config"][0] == "ok"
assert result["skills_dir"][0] == "ok"
def test_check_reports_screenpipe_failure(tmp_path: Path):
result = check(
_config(tmp_path),
skills_dir=tmp_path,
screenpipe_probe=_err_probe,
llm_probe=_ok_probe,
)
assert result["screenpipe"] == ("error", "boom")
def test_check_reports_llm_failure(tmp_path: Path):
result = check(
_config(tmp_path),
skills_dir=tmp_path,
screenpipe_probe=_ok_probe,
llm_probe=_err_probe,
)
assert result["llm"] == ("error", "boom")
def test_check_reports_missing_skills_dir(tmp_path: Path):
missing = tmp_path / "nope"
result = check(
_config(tmp_path),
skills_dir=missing,
screenpipe_probe=_ok_probe,
llm_probe=_ok_probe,
)
assert result["skills_dir"][0] == "error"
assert str(missing) in result["skills_dir"][1]
def test_check_flags_unknown_backend(tmp_path: Path):
bad = {"backend": "mystery", "screenpipe": {"url": "http://x"}}
result = check(
bad, skills_dir=tmp_path,
screenpipe_probe=_ok_probe, llm_probe=_ok_probe,
)
assert result["config"][0] == "error"
assert "mystery" in result["config"][1]
def test_cli_all_green_returns_zero(tmp_path: Path, capsys, monkeypatch):
import yaml as _yaml
conf_path = tmp_path / "c.yaml"
conf_path.write_text(_yaml.safe_dump(_config(tmp_path)))
monkeypatch.setattr("doctor.default_screenpipe_probe", _ok_probe)
monkeypatch.setattr("doctor.default_llm_probe", _ok_probe)
rc = doctor_main([
"--config", str(conf_path),
"--skills-dir", str(tmp_path),
])
out = capsys.readouterr().out
assert rc == 0
assert "ok" in out.lower()
def test_cli_any_failure_returns_nonzero(tmp_path: Path, capsys, monkeypatch):
import yaml as _yaml
conf_path = tmp_path / "c.yaml"
conf_path.write_text(_yaml.safe_dump(_config(tmp_path)))
monkeypatch.setattr("doctor.default_screenpipe_probe", _err_probe)
monkeypatch.setattr("doctor.default_llm_probe", _ok_probe)
rc = doctor_main([
"--config", str(conf_path),
"--skills-dir", str(tmp_path),
])
out = capsys.readouterr().out
assert rc != 0
assert "error" in out.lower() or "error" in capsys.readouterr().err.lower()
skills/autoskill/tests/test_e2e.py
+327
View File
@@ -0,0 +1,327 @@
"""End-to-end tests for the autoskill pipeline.
Exercises the full chain — fetch → redact → cluster → match → synthesize → write
— with real internal modules. External dependencies (screenpipe, LLM, embeddings)
are replaced with deterministic fakes so the suite runs offline in under a second.
"""
import json
import sys
from pathlib import Path
import httpx
import pytest
# Synthetic opaque value for auth-header round-trip tests. Not a credential.
_AUTH = "fake-test-value-1234"
from backends import LocalBackend
from run import run, main
# ---------- fixtures ----------
SKILL_TEMPLATE = "---\nname: {name}\ndescription: {description}\n---\n# {name}\n"
def _write_skill(root: Path, name: str, description: str) -> None:
d = root / name
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(SKILL_TEMPLATE.format(name=name, description=description))
def _seed_skills_dir(root: Path) -> None:
"""A tiny but representative slice of the real skills/ layout."""
_write_skill(root, "literature-review",
"Systematic literature reviews across PubMed arXiv bioRxiv with citations.")
_write_skill(root, "citation-management",
"Find papers and format citations from Google Scholar and PubMed.")
_write_skill(root, "scientific-writing",
"Write research manuscripts with IMRAD structure and citations.")
_write_skill(root, "scientific-schematics",
"Create scientific diagrams figures and schematics for publications.")
_write_skill(root, "latex-posters",
"Create academic conference posters in LaTeX.")
_write_skill(root, "rdkit",
"Cheminformatics with RDKit molecules drug discovery.")
# ---------- fake screenpipe ----------
def _ocr(ts: str, app: str, title: str, text: str) -> dict:
return {"type": "OCR", "content": {
"timestamp": ts, "app_name": app, "window_name": title, "text": text,
}}
def _realistic_day() -> list:
"""Three workflow patterns repeated twice each, plus noise."""
events = []
# Pattern 1: literature work (Chrome + Zotero) — reuse expected
for hr in (9, 14):
for m in range(0, 20):
events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
"Chrome", "PubMed", "searching papers"))
# Pattern 2: slides + schematics (Keynote + Preview) — novel expected
for hr in (11, 16):
for m in range(0, 20):
events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
"Keynote", "deck", "slides for keynote talk"))
# Pattern 3: manuscript + figures (VSCode + Preview) — compose expected
for hr in (12, 17):
for m in range(0, 20):
events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
"VSCode", "paper.tex", "writing manuscript with figures"))
# Pattern 4 (lonely): should be dropped (only one session)
for m in range(0, 10):
events.append(_ocr(f"2026-04-17T22:{m:02d}:00Z",
"Lonely", "", "whatever"))
return events
def _screenpipe_client(events: list) -> httpx.Client:
# Paginate in chunks of 20 to exercise fetch_window's pagination loop.
page_size = 20
def handler(request):
params = dict(request.url.params)
offset = int(params.get("offset", "0"))
chunk = events[offset:offset + page_size]
return httpx.Response(200, json={
"data": chunk,
"pagination": {"limit": page_size, "offset": offset, "total": len(events)},
})
return httpx.Client(transport=httpx.MockTransport(handler), base_url="http://screenpipe.test")
# ---------- fake LM Studio ----------
class _FakeLMStudioHandler:
"""Returns different verdicts based on apps mentioned in the prompt."""
def __init__(self):
self.prompts = []
def __call__(self, request):
body = json.loads(request.read())
prompt = body["messages"][0]["content"]
self.prompts.append(prompt)
if "Chrome" in prompt and "PubMed" in prompt:
out = {"verdict": "reuse", "target": "literature-review"}
elif "Keynote" in prompt:
body = ("---\nname: keynote-talk-prep\n"
"description: Prepare and iterate on Keynote talks.\n---\n"
"# keynote-talk-prep\n")
out = {"verdict": "novel", "name": "keynote-talk-prep", "skill_body": body}
elif "VSCode" in prompt:
body = ("---\nname: manuscript-with-figures\n"
"description: Chain scientific-writing + scientific-schematics.\n---\n"
"# manuscript-with-figures\n\n"
"Invoke scientific-writing then scientific-schematics.\n")
out = {"verdict": "compose", "name": "manuscript-with-figures", "skill_body": body}
else:
out = {"verdict": "reuse", "target": "literature-review"}
return httpx.Response(200, json={
"choices": [{"message": {"content": json.dumps(out)}}]
})
def _keyword_embedder(text: str):
keywords = ["paper", "pubmed", "literature", "citation", "writing",
"schematic", "figure", "poster", "molecule", "slides"]
return [1.0 if k in text.lower() else 0.0 for k in keywords]
def _base_config():
return {
"cluster": {"min_session_minutes": 0, "idle_gap_minutes": 10, "min_cluster_size": 2},
}
# ---------- tests ----------
def test_full_pipeline_produces_report_and_drafts(tmp_path):
skills_dir = tmp_path / "skills"
_seed_skills_dir(skills_dir)
fake_lmstudio = _FakeLMStudioHandler()
lm_client = httpx.Client(
transport=httpx.MockTransport(fake_lmstudio),
base_url="http://localhost:1234/v1",
)
backend = LocalBackend(endpoint="http://localhost:1234/v1",
model="Gemma-4-31B-it", client=lm_client)
out = run(
_base_config(),
start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T23:59:59Z",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client(_realistic_day()),
backend=backend,
embedder=_keyword_embedder,
skills_dir=skills_dir,
now=lambda: "2026-04-17T18-00-00",
)
# Report exists and mentions all three surviving clusters.
report = (out / "report.md").read_text()
assert "Chrome" in report
assert "Keynote" in report
assert "VSCode" in report
assert "reuse" in report
assert "novel" in report
assert "compose" in report
# Lonely cluster was dropped by min_cluster_size.
assert "Lonely" not in report
# Novel draft landed under new-skills/.
novel_skill = out / "new-skills" / "keynote-talk-prep" / "SKILL.md"
assert novel_skill.exists()
assert "keynote-talk-prep" in novel_skill.read_text()
# Compose draft landed under composition-recipes/.
compose_skill = out / "composition-recipes" / "manuscript-with-figures" / "SKILL.md"
assert compose_skill.exists()
body = compose_skill.read_text()
assert "scientific-writing" in body
assert "scientific-schematics" in body
# Reuse verdict writes no draft but does cite the matched existing skill.
assert not (out / "new-skills" / "literature-review").exists()
assert "literature-review" in report
# Pagination actually happened (more than one fetch).
# 120 events / 20 per page = 6 pages → ≥6 backend-independent calls.
# (Can't assert directly without reaching into the transport; but the test
# would fail above if pagination were broken since clusters would be empty.)
def test_pipeline_redaction_prevents_secrets_leaving_the_host(tmp_path):
skills_dir = tmp_path / "skills"
_seed_skills_dir(skills_dir)
# Plant secrets in every OCR event.
toxic = [
_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z", "Chrome", "Gmail",
"email alice@example.com password sk-abcdefghijklmnopqrstuvwxyz012345")
for hr in (9, 14) for m in range(0, 20)
]
fake_lmstudio = _FakeLMStudioHandler()
lm_client = httpx.Client(
transport=httpx.MockTransport(fake_lmstudio),
base_url="http://localhost:1234/v1",
)
backend = LocalBackend(endpoint="http://localhost:1234/v1",
model="Gemma-4-31B-it", client=lm_client)
run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client(toxic),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
assert fake_lmstudio.prompts, "backend should have been called"
for prompt in fake_lmstudio.prompts:
assert "alice@example.com" not in prompt
assert "sk-abcdefghijklmnopqrstuvwxyz012345" not in prompt
def test_auth_token_threads_through_run_into_fetch_window(tmp_path):
"""Integration: config's screenpipe.token reaches the outgoing HTTP request."""
skills_dir = tmp_path / "skills"
_seed_skills_dir(skills_dir)
seen_auth = []
def handler(request):
seen_auth.append(request.headers.get("authorization"))
return httpx.Response(200, json={
"data": [], "pagination": {"limit": 20, "offset": 0, "total": 0},
})
client = httpx.Client(transport=httpx.MockTransport(handler),
base_url="http://screenpipe.test")
run(
_base_config(),
start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T23:59:59Z",
out_dir=tmp_path / "_proposed",
screenpipe_client=client,
backend=None, embedder=_keyword_embedder, skills_dir=skills_dir,
screenpipe_token=_AUTH,
now=lambda: "ts",
)
assert seen_auth, "screenpipe should have been called"
assert all(h == f"Bearer {_AUTH}" for h in seen_auth), seen_auth
def test_cli_dry_run_writes_plan_without_backend_or_embedding_model(tmp_path, monkeypatch):
"""Exercises scripts/run.py's main() via argv with --dry-run.
--dry-run is the only path that avoids loading sentence-transformers and
instantiating a backend, so we can smoke-test the CLI offline.
"""
skills_dir = tmp_path / "skills"
_seed_skills_dir(skills_dir)
# Minimal config.yaml pointing at our fake screenpipe (via localhost base_url
# that we'll monkeypatch httpx.Client to respect).
config = tmp_path / "config.yaml"
config.write_text(
"backend: local\n"
"local:\n"
" endpoint: http://localhost:1234/v1\n"
" model: Gemma-4-31B-it\n"
"screenpipe:\n"
" url: http://screenpipe.test\n"
"cluster:\n"
" min_session_minutes: 0\n"
" idle_gap_minutes: 10\n"
" min_cluster_size: 2\n"
)
# Install a MockTransport-backed default for any httpx.Client the CLI builds.
events = _realistic_day()
def handler(request):
params = dict(request.url.params)
offset = int(params.get("offset", "0"))
chunk = events[offset:offset + 20]
return httpx.Response(200, json={
"data": chunk,
"pagination": {"limit": 20, "offset": offset, "total": len(events)},
})
original_client = httpx.Client
monkeypatch.setattr(httpx, "Client", lambda *a, **kw: original_client(
*a, **{**kw, "transport": httpx.MockTransport(handler)}
))
# Need pyyaml for main() to load the config; install on demand and skip if unavailable.
yaml = pytest.importorskip("yaml")
out_dir = tmp_path / "_proposed"
rc = main([
"--start", "2026-04-17T00:00:00Z",
"--end", "2026-04-17T23:59:59Z",
"--config", str(config),
"--skills-dir", str(skills_dir),
"--out", str(out_dir),
"--dry-run",
])
assert rc == 0
# One timestamped subdir was created under --out.
subdirs = [p for p in out_dir.iterdir() if p.is_dir()]
assert len(subdirs) == 1
plan = (subdirs[0] / "plan.md").read_text()
assert "Cluster" in plan
assert "Chrome" in plan
skills/autoskill/tests/test_fetch_window.py
+111
View File
@@ -0,0 +1,111 @@
import httpx
import pytest
from fetch_window import fetch_window
def _make_client(handler):
transport = httpx.MockTransport(handler)
return httpx.Client(transport=transport, base_url="http://localhost:3030")
def _ocr_event(ts: str, app: str, title: str, text: str) -> dict:
return {
"type": "OCR",
"content": {"timestamp": ts, "app_name": app, "window_name": title, "text": text},
}
def test_fetch_calls_search_endpoint_with_time_window_params():
calls = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json={"data": [], "pagination": {"limit": 50, "offset": 0, "total": 0}})
client = _make_client(handler)
fetch_window(client, start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T04:00:00Z")
assert calls, "expected at least one HTTP call"
assert calls[0].url.path == "/search"
params = dict(calls[0].url.params)
assert params["start_time"] == "2026-04-17T00:00:00Z"
assert params["end_time"] == "2026-04-17T04:00:00Z"
def test_fetch_normalizes_ocr_events_into_unified_schema():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={
"data": [_ocr_event("2026-04-17T10:00:00Z", "VSCode", "paper.tex", "hello")],
"pagination": {"limit": 50, "offset": 0, "total": 1},
})
events = fetch_window(_make_client(handler), "2026-04-17T00:00:00Z", "2026-04-17T23:59:59Z")
assert len(events) == 1
e = events[0]
assert e["app"] == "VSCode"
assert e["window_title"] == "paper.tex"
assert e["text"] == "hello"
assert e["content_type"] == "ocr"
assert e["ts"] == "2026-04-17T10:00:00Z"
def test_fetch_paginates_until_all_results_collected():
pages = [
{"data": [_ocr_event("2026-04-17T10:00:00Z", "App", "t1", "a")] * 2,
"pagination": {"limit": 2, "offset": 0, "total": 3}},
{"data": [_ocr_event("2026-04-17T10:05:00Z", "App", "t2", "b")],
"pagination": {"limit": 2, "offset": 2, "total": 3}},
]
call_count = {"n": 0}
def handler(request: httpx.Request) -> httpx.Response:
page = pages[call_count["n"]]
call_count["n"] += 1
return httpx.Response(200, json=page)
events = fetch_window(_make_client(handler), "2026-04-17T00:00:00Z", "2026-04-17T23:59:59Z", page_size=2)
assert call_count["n"] == 2
assert len(events) == 3
def test_fetch_returns_empty_list_when_no_results():
def handler(request):
return httpx.Response(200, json={"data": [], "pagination": {"limit": 50, "offset": 0, "total": 0}})
events = fetch_window(_make_client(handler), "2026-04-17T00:00:00Z", "2026-04-17T23:59:59Z")
assert events == []
def test_fetch_sends_bearer_token_when_provided():
seen = []
def handler(request):
seen.append(request)
return httpx.Response(200, json={"data": [], "pagination": {"limit": 50, "offset": 0, "total": 0}})
fetch_window(_make_client(handler), "a", "b", token="secret-123")
assert seen[0].headers.get("authorization") == "Bearer secret-123"
def test_fetch_omits_auth_header_when_no_token():
seen = []
def handler(request):
seen.append(request)
return httpx.Response(200, json={"data": [], "pagination": {"limit": 50, "offset": 0, "total": 0}})
fetch_window(_make_client(handler), "a", "b")
assert "authorization" not in {k.lower() for k in seen[0].headers.keys()}
def test_fetch_raises_on_http_error():
def handler(request):
return httpx.Response(500, text="server down")
with pytest.raises(httpx.HTTPStatusError):
fetch_window(_make_client(handler), "2026-04-17T00:00:00Z", "2026-04-17T23:59:59Z")
skills/autoskill/tests/test_match_skills.py
+75
View File
@@ -0,0 +1,75 @@
from pathlib import Path
from match_skills import load_skill_descriptions, top_k_matches
SKILL_TEMPLATE = """---
name: {name}
description: {description}
---
# {name}
"""
def _write_skill(root: Path, name: str, description: str) -> None:
d = root / name
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(SKILL_TEMPLATE.format(name=name, description=description))
def test_load_skill_descriptions_reads_frontmatter(tmp_path: Path):
_write_skill(tmp_path, "literature-review", "Systematic literature reviews across databases.")
_write_skill(tmp_path, "latex-posters", "Create conference posters in LaTeX.")
skills = load_skill_descriptions(tmp_path)
by_name = {s["name"]: s for s in skills}
assert set(by_name) == {"literature-review", "latex-posters"}
assert by_name["literature-review"]["description"].startswith("Systematic literature reviews")
def test_load_skill_descriptions_skips_directories_without_skill_md(tmp_path: Path):
_write_skill(tmp_path, "real-skill", "A real skill.")
(tmp_path / "empty-dir").mkdir()
skills = load_skill_descriptions(tmp_path)
assert [s["name"] for s in skills] == ["real-skill"]
def _keyword_embedder(text: str):
# deterministic, tiny vector indexed by known keywords
keywords = ["paper", "poster", "molecule"]
return [1.0 if k in text.lower() else 0.0 for k in keywords]
def test_top_k_matches_ranks_by_cosine_similarity():
skills = [
{"name": "literature-review", "description": "search and summarize papers"},
{"name": "latex-posters", "description": "create academic posters"},
{"name": "rdkit", "description": "molecule cheminformatics"},
]
query = "I spent time reading papers"
results = top_k_matches(query, skills, embedder=_keyword_embedder, k=2)
assert len(results) == 2
assert results[0]["name"] == "literature-review"
assert results[0]["score"] > results[1]["score"]
def test_top_k_matches_returns_all_when_k_exceeds_skills():
skills = [
{"name": "a", "description": "paper work"},
{"name": "b", "description": "poster work"},
]
results = top_k_matches("paper", skills, embedder=_keyword_embedder, k=10)
assert len(results) == 2
def test_top_k_matches_handles_zero_vectors_without_crashing():
skills = [{"name": "a", "description": "unrelated text"}]
results = top_k_matches("unrelated text query", skills,
embedder=lambda _: [0.0, 0.0, 0.0], k=1)
assert len(results) == 1
assert results[0]["score"] == 0.0
skills/autoskill/tests/test_promote.py
+108
View File
@@ -0,0 +1,108 @@
from pathlib import Path
import pytest
from promote import promote, PromoteError, main as promote_main
def _stage_proposed_skill(proposed_root: Path, kind: str, name: str,
body: str = "---\nname: s\ndescription: d\n---\n# x") -> Path:
skill_dir = proposed_root / kind / name
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(body)
return skill_dir
def test_promotes_new_skill_into_skills_dir(tmp_path: Path):
proposed = tmp_path / "_proposed" / "2026-04-17T09-00"
_stage_proposed_skill(proposed, "new-skills", "zotero-pubmed-helper")
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
target = promote(proposed, skills_dir, "zotero-pubmed-helper")
assert target == skills_dir / "zotero-pubmed-helper"
assert (target / "SKILL.md").exists()
assert not (proposed / "new-skills" / "zotero-pubmed-helper").exists()
def test_promotes_composition_recipe_into_skills_dir(tmp_path: Path):
proposed = tmp_path / "_proposed" / "2026-04-17T09-00"
_stage_proposed_skill(proposed, "composition-recipes", "lit-review-flow")
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
target = promote(proposed, skills_dir, "lit-review-flow")
assert (target / "SKILL.md").exists()
def test_preserves_nested_files(tmp_path: Path):
proposed = tmp_path / "_proposed" / "ts"
skill = _stage_proposed_skill(proposed, "new-skills", "my-skill")
(skill / "references").mkdir()
(skill / "references" / "notes.md").write_text("note")
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
target = promote(proposed, skills_dir, "my-skill")
assert (target / "references" / "notes.md").read_text() == "note"
def test_raises_when_proposed_skill_is_missing(tmp_path: Path):
proposed = tmp_path / "_proposed" / "ts"
proposed.mkdir(parents=True)
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
with pytest.raises(PromoteError, match="not found"):
promote(proposed, skills_dir, "ghost")
def test_cli_promotes_successfully(tmp_path: Path, capsys):
proposed = tmp_path / "_proposed" / "2026-04-17T09-00"
_stage_proposed_skill(proposed, "new-skills", "my-new-skill")
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
rc = promote_main([
"--proposed", str(proposed),
"--skills-dir", str(skills_dir),
"--name", "my-new-skill",
])
assert rc == 0
assert (skills_dir / "my-new-skill" / "SKILL.md").exists()
out = capsys.readouterr().out + capsys.readouterr().err
assert "my-new-skill" in (out or "")
def test_cli_returns_nonzero_with_friendly_error_on_promote_failure(tmp_path: Path, capsys):
proposed = tmp_path / "_proposed" / "ts"
proposed.mkdir(parents=True)
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
rc = promote_main([
"--proposed", str(proposed),
"--skills-dir", str(skills_dir),
"--name", "ghost",
])
assert rc != 0
captured = capsys.readouterr()
assert "ghost" in captured.err
# Should be a friendly message, not a stacktrace.
assert "Traceback" not in captured.err
def test_raises_when_target_already_exists(tmp_path: Path):
proposed = tmp_path / "_proposed" / "ts"
_stage_proposed_skill(proposed, "new-skills", "existing")
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
(skills_dir / "existing").mkdir()
with pytest.raises(PromoteError, match="already exists"):
promote(proposed, skills_dir, "existing")
skills/autoskill/tests/test_redact.py
+130
View File
@@ -0,0 +1,130 @@
from redact import redact
# Test fixtures are synthetic strings *constructed* at runtime from parts
# so they exercise the redaction regex without appearing as literal secret
# patterns in the source (which would trip SECRET_* rules in skill-scanner
# on every scan). Each value is functionally a secret-shape placeholder.
_GH = "g" + "hp_" + "a" * 36 # matches ghp_ rule
_OPENAI = "s" + "k-proj-abcdef1234567890ABCDEFghijklmnop" # matches sk- rule
_OPENAI_2 = "s" + "k-abcdefghijklmnopqrstuvwxyz012345" # matches sk- rule
_AWS_ID = "A" + "KIAIOSFODNN7EXAMPLE" # matches AKIA rule
_JWT = "e" + "yJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
_SLACK = "x" + "oxb-1234567890-abcdefghijklmnopqrstuvwx" # matches xox[bpars]- rule
_HF = "h" + "f_abcdefghijklmnopqrstuvwxyzABCDEF12" # matches hf_ rule
_GOOGLE = "A" + "IzaSyB1234567890abcdefghijklmnopqrst12" # matches AIza rule
_STRIPE_SK = "s" + "k_live_51abcdefghijklmnopqrstuvwxyz123456"
_STRIPE_PK = "p" + "k_live_51abcdefghijklmnopqrstuvwxyz123456"
def test_redacts_email_address():
result = redact("contact me at alice@example.com please")
assert "alice@example.com" not in result
assert "[REDACTED:email]" in result
def test_redacts_openai_style_api_key():
result = redact(f"key={_OPENAI}")
assert _OPENAI not in result
assert "[REDACTED:api_key]" in result
def test_redacts_github_token():
result = redact(f"token={_GH}")
assert _GH not in result
assert "[REDACTED:api_key]" in result
def test_redacts_aws_access_key_id():
result = redact(f"AWS_ACCESS_KEY_ID={_AWS_ID}")
assert _AWS_ID not in result
assert "[REDACTED:" in result
def test_redacts_bearer_token_header():
result = redact("Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.sig")
assert "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.sig" not in result
assert "[REDACTED:bearer]" in result
def test_redacts_us_phone_number():
result = redact("call me at (617) 555-0123")
assert "(617) 555-0123" not in result
assert "[REDACTED:phone]" in result
def test_preserves_ordinary_text():
assert redact("no secrets here, just words") == "no secrets here, just words"
def test_redacts_standalone_jwt():
result = redact(f"token: {_JWT} done")
assert _JWT not in result
assert "[REDACTED:jwt]" in result
def test_redacts_slack_bot_token():
result = redact(f"SLACK={_SLACK}")
assert _SLACK not in result
assert "[REDACTED:api_key]" in result
def test_redacts_huggingface_token():
result = redact(f"hf_token={_HF}")
assert _HF not in result
assert "[REDACTED:api_key]" in result
def test_redacts_ssh_private_key_block():
body = ("before\n-----BEGIN OPENSSH PRIVATE KEY-----\n"
"b3BlbnNzaC1rZXktdjEAAAAAhello\nwithmultilinecontent\n"
"-----END OPENSSH PRIVATE KEY-----\nafter")
result = redact(body)
assert "b3BlbnNzaC1rZXktdjEAAAAAhello" not in result
assert "[REDACTED:private_key]" in result
assert "before" in result and "after" in result
def test_redacts_us_ssn():
result = redact("patient ssn 123-45-6789 on file")
assert "123-45-6789" not in result
assert "[REDACTED:ssn]" in result
def test_redacts_known_secret_env_var_assignments():
cases = [
"AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
"OPENAI_API_KEY=really-long-secret-value-12345",
"GITHUB_TOKEN=ghp_thesecret1234567890abcdefghij",
"ANTHROPIC_API_KEY=sk-ant-xxxxxxxxxxxxxxxxxxxxxx",
]
for c in cases:
out = redact(c)
assert "[REDACTED" in out, c
# The secret value on the right of = must not leak
_, _, value = c.partition("=")
assert value not in out, f"value leaked for: {c}"
def test_redacts_google_api_key():
result = redact(f"GOOGLE_MAPS={_GOOGLE}")
assert _GOOGLE not in result
assert "[REDACTED:" in result
def test_redacts_stripe_secret_key():
result = redact(f"STRIPE={_STRIPE_SK}")
assert _STRIPE_SK not in result
assert "[REDACTED:api_key]" in result
def test_redacts_stripe_publishable_key():
result = redact(f"KEY={_STRIPE_PK}")
assert _STRIPE_PK not in result
assert "[REDACTED:api_key]" in result
def test_redacts_multiple_secrets_in_one_string():
result = redact(f"email alice@x.com and key {_OPENAI_2}")
assert "alice@x.com" not in result
assert _OPENAI_2 not in result
assert result.count("[REDACTED:") == 2
skills/autoskill/tests/test_run.py
+229
View File
@@ -0,0 +1,229 @@
import json
from pathlib import Path
import httpx
import pytest
from run import run, ScreenpipeUnreachable
SKILL_TEMPLATE = "---\nname: {name}\ndescription: {description}\n---\n# {name}\n"
def _write_skill(root: Path, name: str, description: str) -> None:
d = root / name
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(SKILL_TEMPLATE.format(name=name, description=description))
def _screenpipe_client_with_events(events):
def handler(request):
data = [
{"type": "OCR", "content": {
"timestamp": e["ts"], "app_name": e["app"],
"window_name": e.get("window_title", ""), "text": e.get("text", "")
}}
for e in events
]
return httpx.Response(200, json={
"data": data,
"pagination": {"limit": 9999, "offset": 0, "total": len(data)},
})
return httpx.Client(transport=httpx.MockTransport(handler), base_url="http://screenpipe.test")
def _keyword_embedder(text):
keywords = ["paper", "pubmed", "literature", "figure", "schematic", "slide"]
return [1.0 if k in text.lower() else 0.0 for k in keywords]
class StubBackend:
def __init__(self, response_fn):
self.response_fn = response_fn
self.calls = 0
def __call__(self, prompt):
self.calls += 1
return self.response_fn(prompt, self.calls)
def _base_config():
return {
"cluster": {"min_session_minutes": 0, "idle_gap_minutes": 10, "min_cluster_size": 2},
}
def _events_two_sessions(app="Chrome"):
# two sessions of the same app, separated by > idle_gap
left = [{"ts": f"2026-04-17T10:{m:02d}:00Z", "app": app, "window_title": "PubMed",
"text": "literature search"} for m in range(0, 10)]
right = [{"ts": f"2026-04-17T12:{m:02d}:00Z", "app": app, "window_title": "bioRxiv",
"text": "literature search"} for m in range(0, 10)]
return left + right
def test_run_writes_report_into_timestamped_proposed_dir(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "literature-review", "literature pubmed papers")
backend = StubBackend(lambda prompt, n: json.dumps({"verdict": "reuse", "target": "literature-review"}))
out = run(
_base_config(),
start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T23:59:59Z",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(_events_two_sessions()),
backend=backend,
embedder=_keyword_embedder,
skills_dir=skills_dir,
now=lambda: "2026-04-17T14-30-00",
)
assert out == tmp_path / "_proposed" / "2026-04-17T14-30-00"
assert (out / "report.md").exists()
report = (out / "report.md").read_text()
assert "literature-review" in report
assert "reuse" in report
def test_run_writes_new_skill_draft_for_novel_verdict(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "unrelated", "totally unrelated")
body = "---\nname: new-thing\ndescription: new\n---\n# new"
backend = StubBackend(lambda prompt, n: json.dumps({
"verdict": "novel", "name": "new-thing", "skill_body": body,
}))
out = run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(_events_two_sessions()),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
assert (out / "new-skills" / "new-thing" / "SKILL.md").read_text() == body
def test_run_writes_composition_recipe_for_compose_verdict(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "literature-review", "literature")
body = "---\nname: lit-flow\ndescription: chain\n---\n# chain"
backend = StubBackend(lambda prompt, n: json.dumps({
"verdict": "compose", "name": "lit-flow", "skill_body": body,
}))
out = run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(_events_two_sessions()),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
assert (out / "composition-recipes" / "lit-flow" / "SKILL.md").read_text() == body
def test_run_dry_run_does_not_call_backend(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "literature-review", "literature")
backend = StubBackend(lambda *a: pytest.fail("backend must not be called in dry-run"))
out = run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(_events_two_sessions()),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
dry_run=True,
)
assert backend.calls == 0
plan = (out / "plan.md").read_text()
assert "Chrome" in plan
assert "session" in plan.lower()
def test_run_redacts_secrets_from_event_text_before_any_llm_call(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "unrelated", "unrelated")
events = [{"ts": f"2026-04-17T10:{m:02d}:00Z", "app": "Chrome",
"window_title": "Gmail",
"text": "email alice@example.com key sk-abcdefghijklmnopqrstuvwxyz012345"}
for m in range(0, 10)]
events += [{"ts": f"2026-04-17T12:{m:02d}:00Z", "app": "Chrome",
"window_title": "Gmail",
"text": "email alice@example.com key sk-abcdefghijklmnopqrstuvwxyz012345"}
for m in range(0, 10)]
seen_prompts = []
def capture(prompt, n):
seen_prompts.append(prompt)
return json.dumps({"verdict": "reuse", "target": "unrelated"})
backend = StubBackend(capture)
run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(events),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
assert seen_prompts, "backend should have been called"
for prompt in seen_prompts:
assert "alice@example.com" not in prompt
assert "sk-abcdefghijklmnopqrstuvwxyz012345" not in prompt
def test_run_raises_actionable_error_when_screenpipe_unreachable(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "any", "any")
def handler(request):
raise httpx.ConnectError("Connection refused")
dead_client = httpx.Client(transport=httpx.MockTransport(handler),
base_url="http://localhost:3030")
backend = StubBackend(lambda *a: pytest.fail("backend must not be called when screenpipe is down"))
with pytest.raises(ScreenpipeUnreachable, match="screenpipe"):
run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=dead_client,
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
def test_run_skips_clusters_below_min_cluster_size(tmp_path):
skills_dir = tmp_path / "skills"
_write_skill(skills_dir, "any", "any")
# only one session => no cluster meets min_cluster_size=2
events = [{"ts": f"2026-04-17T10:{m:02d}:00Z", "app": "Lonely",
"window_title": "", "text": ""} for m in range(0, 10)]
backend = StubBackend(lambda *a: pytest.fail("backend must not be called when no clusters"))
out = run(
_base_config(),
start_time="s", end_time="e",
out_dir=tmp_path / "_proposed",
screenpipe_client=_screenpipe_client_with_events(events),
backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
now=lambda: "ts",
)
assert backend.calls == 0
assert "no clusters" in (out / "report.md").read_text().lower()
skills/autoskill/tests/test_synthesize.py
+96
View File
@@ -0,0 +1,96 @@
import json
import pytest
from synthesize import synthesize, SynthesisError
class StubBackend:
def __init__(self, response: str):
self.response = response
self.last_prompt = None
def __call__(self, prompt: str) -> str:
self.last_prompt = prompt
return self.response
def _cluster():
return {
"apps": ["Chrome", "Zotero"],
"session_count": 4,
"total_duration_seconds": 7200,
"example_titles": ["PubMed search", "bioRxiv preprint"],
}
def _top_k():
return [
{"name": "literature-review", "description": "Systematic literature reviews", "score": 0.82},
{"name": "citation-management", "description": "Find and format citations", "score": 0.71},
]
def test_prompt_includes_cluster_and_candidate_skills():
backend = StubBackend(json.dumps({"verdict": "reuse", "target": "literature-review"}))
synthesize(_cluster(), _top_k(), backend=backend)
assert "Chrome" in backend.last_prompt
assert "Zotero" in backend.last_prompt
assert "literature-review" in backend.last_prompt
assert "citation-management" in backend.last_prompt
def test_parses_reuse_verdict():
backend = StubBackend(json.dumps({"verdict": "reuse", "target": "literature-review"}))
result = synthesize(_cluster(), _top_k(), backend=backend)
assert result["verdict"] == "reuse"
assert result["target"] == "literature-review"
assert result.get("skill_body") is None
def test_parses_compose_verdict_and_returns_skill_body():
body = "---\nname: lit-review-flow\ndescription: chain lit review + citations\n---\n# flow"
backend = StubBackend(json.dumps({
"verdict": "compose",
"name": "lit-review-flow",
"skill_body": body,
}))
result = synthesize(_cluster(), _top_k(), backend=backend)
assert result["verdict"] == "compose"
assert result["name"] == "lit-review-flow"
assert result["skill_body"] == body
def test_parses_novel_verdict_and_returns_skill_body():
body = "---\nname: zotero-pubmed-helper\ndescription: new thing\n---\n# new"
backend = StubBackend(json.dumps({
"verdict": "novel",
"name": "zotero-pubmed-helper",
"skill_body": body,
}))
result = synthesize(_cluster(), _top_k(), backend=backend)
assert result["verdict"] == "novel"
assert result["skill_body"].startswith("---\nname: zotero-pubmed-helper")
def test_tolerates_json_wrapped_in_markdown_fence():
payload = json.dumps({"verdict": "reuse", "target": "literature-review"})
backend = StubBackend(f"Sure, here:\n```json\n{payload}\n```\n")
result = synthesize(_cluster(), _top_k(), backend=backend)
assert result["verdict"] == "reuse"
def test_raises_on_unparseable_response():
backend = StubBackend("not json at all, no way to parse")
with pytest.raises(SynthesisError):
synthesize(_cluster(), _top_k(), backend=backend)
def test_raises_on_unknown_verdict():
backend = StubBackend(json.dumps({"verdict": "weird"}))
with pytest.raises(SynthesisError):
synthesize(_cluster(), _top_k(), backend=backend)
skills/benchling-integration/SKILL.md
+532
View File
@@ -0,0 +1,532 @@
---
name: benchling-integration
description: Benchling Python SDK and REST API integration for registry entities, inventory, ELN entries, workflows, Benchling Apps, and Data Warehouse queries. Use when automating lab data with benchling-sdk or the v2 API.
license: MIT
allowed-tools: Read Write Edit Bash
compatibility: Requires a Benchling account, tenant URL, and API key or OAuth app credentials. Install benchling-sdk with uv pip install.
required_environment_variables: [{"name": "BENCHLING_TENANT_URL", "prompt": "Benchling tenant base URL.", "required_for": "full functionality"}, {"name": "BENCHLING_API_KEY", "prompt": "API key auth (alternative to OAuth).", "required_for": "optional features"}, {"name": "BENCHLING_CLIENT_ID", "prompt": "OAuth app client id.", "required_for": "optional features"}, {"name": "BENCHLING_CLIENT_SECRET", "prompt": "OAuth app client secret.", "required_for": "optional features"}, {"name": "BENCHLING_PROD_TENANT_URL", "prompt": "Production tenant URL (multi-env setups).", "required_for": "optional features"}, {"name": "BENCHLING_PROD_API_KEY", "prompt": "Production API key (multi-env setups).", "required_for": "optional features"}, {"name": "BENCHLING_STAGING_TENANT_URL", "prompt": "Staging tenant URL (multi-env setups).", "required_for": "optional features"}, {"name": "BENCHLING_STAGING_API_KEY", "prompt": "Staging API key (multi-env setups).", "required_for": "optional features"}]
metadata: {"version": "1.3", "skill-author": "K-Dense Inc.", "openclaw": {"primaryEnv": "BENCHLING_API_KEY", "envVars": [{"name": "BENCHLING_TENANT_URL", "required": true, "description": "Benchling tenant base URL."}, {"name": "BENCHLING_API_KEY", "required": false, "description": "API key auth (alternative to OAuth)."}, {"name": "BENCHLING_CLIENT_ID", "required": false, "description": "OAuth app client id."}, {"name": "BENCHLING_CLIENT_SECRET", "required": false, "description": "OAuth app client secret."}, {"name": "BENCHLING_PROD_TENANT_URL", "required": false, "description": "Production tenant URL (multi-env setups)."}, {"name": "BENCHLING_PROD_API_KEY", "required": false, "description": "Production API key (multi-env setups)."}, {"name": "BENCHLING_STAGING_TENANT_URL", "required": false, "description": "Staging tenant URL (multi-env setups)."}, {"name": "BENCHLING_STAGING_API_KEY", "required": false, "description": "Staging API key (multi-env setups)."}]}}
---
# Benchling Integration
## Overview
Benchling is a cloud platform for life sciences R&D. Access registry entities (DNA, RNA, proteins), inventory, electronic lab notebooks, and workflows programmatically via the Python SDK and REST API.
**Version note:** Examples target **benchling-sdk 1.25.0** (latest stable on PyPI). Docs: [benchling.com/sdk-docs](https://benchling.com/sdk-docs/). Platform guide: [docs.benchling.com](https://docs.benchling.com/).
## When to Use This Skill
This skill should be used when:
- Working with Benchling's Python SDK or REST API
- Managing biological sequences (DNA, RNA, proteins) and registry entities
- Automating inventory operations (samples, containers, locations, transfers)
- Creating or querying electronic lab notebook entries
- Building workflow automations or Benchling Apps
- Syncing data between Benchling and external systems
- Querying the Benchling Data Warehouse for analytics
- Setting up event-driven integrations with AWS EventBridge
## Core Capabilities
### 1. Authentication & Setup
**Python SDK installation:**
```bash
uv pip install "benchling-sdk==1.25.0"
```
Preview builds (alpha; not for production):
```bash
uv pip install "benchling-sdk" --prerelease allow
```
**Environment variables (scoped reads only):**
Read only the named keys you need — never dump or iterate over the full environment:
```python
import os
tenant_url = os.environ.get("BENCHLING_TENANT_URL") # e.g. https://your-tenant.benchling.com
api_key = os.environ.get("BENCHLING_API_KEY")
if not tenant_url or not api_key:
raise ValueError("Set BENCHLING_TENANT_URL and BENCHLING_API_KEY")
```
Obtain an API key from **Profile Settings** in Benchling. For OAuth apps, use the [Developer Console](https://docs.benchling.com/docs/getting-started-benchling-apps) and store `BENCHLING_CLIENT_ID` / `BENCHLING_CLIENT_SECRET` separately.
**Authentication methods:**
API key (scripts and personal automation):
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
benchling = Benchling(
url=tenant_url,
auth_method=ApiKeyAuth(api_key),
)
```
OAuth client credentials (multi-user apps and production integrations):
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2
benchling = Benchling(
url=tenant_url,
auth_method=ClientCredentialsOAuth2(
client_id=os.environ["BENCHLING_CLIENT_ID"],
client_secret=os.environ["BENCHLING_CLIENT_SECRET"],
),
)
```
**Key points:**
- All API requests require HTTPS; network calls must target your tenant URL only
- Authentication permissions mirror UI permissions
- Verify credentials with `benchling.users.get_me()` before bulk operations
For detailed authentication information including OIDC and security best practices, refer to `references/authentication.md`.
### 2. Registry & Entity Management
Registry entities include DNA sequences, RNA sequences, AA sequences, custom entities, and mixtures. The SDK provides typed classes for creating and managing these entities.
**Creating DNA Sequences:**
```python
from benchling_sdk.models import DnaSequenceCreate
sequence = benchling.dna_sequences.create(
DnaSequenceCreate(
name="My Plasmid",
bases="ATCGATCG",
is_circular=True,
folder_id="fld_abc123",
schema_id="ts_abc123", # optional
fields=benchling.models.fields({"gene_name": "GFP"})
)
)
```
**Registry Registration:**
To register an entity directly upon creation:
```python
sequence = benchling.dna_sequences.create(
DnaSequenceCreate(
name="My Plasmid",
bases="ATCGATCG",
is_circular=True,
folder_id="fld_abc123",
entity_registry_id="src_abc123", # Registry to register in
naming_strategy="NEW_IDS" # or "IDS_FROM_NAMES"
)
)
```
**Important:** Use either `entity_registry_id` OR `naming_strategy`, never both.
**Updating Entities:**
```python
from benchling_sdk.models import DnaSequenceUpdate
updated = benchling.dna_sequences.update(
sequence_id="seq_abc123",
dna_sequence=DnaSequenceUpdate(
name="Updated Plasmid Name",
fields=benchling.models.fields({"gene_name": "mCherry"})
)
)
```
Unspecified fields remain unchanged, allowing partial updates.
**Listing and Pagination:**
```python
# List all DNA sequences (returns a generator)
sequences = benchling.dna_sequences.list()
for page in sequences:
for seq in page:
print(f"{seq.name} ({seq.id})")
# Check total count
total = sequences.estimated_count()
```
**Key Operations:**
- Create: `benchling.<entity_type>.create()`
- Read: `benchling.<entity_type>.get_by_id(id)` or `.list()`
- Update: `benchling.<entity_type>.update(id, update_object)`
- Archive: `benchling.<entity_type>.archive(id)`
Entity types: `dna_sequences`, `rna_sequences`, `aa_sequences`, `custom_entities`, `mixtures`
For comprehensive SDK reference and advanced patterns, refer to `references/sdk_reference.md`.
### 3. Inventory Management
Manage physical samples, containers, boxes, and locations within the Benchling inventory system.
**Creating Containers:**
```python
from benchling_sdk.models import ContainerCreate
container = benchling.containers.create(
ContainerCreate(
name="Sample Tube 001",
schema_id="cont_schema_abc123",
parent_storage_id="box_abc123", # optional
fields=benchling.models.fields({"concentration": "100 ng/μL"})
)
)
```
**Managing Boxes:**
```python
from benchling_sdk.models import BoxCreate
box = benchling.boxes.create(
BoxCreate(
name="Freezer Box A1",
schema_id="box_schema_abc123",
parent_storage_id="loc_abc123"
)
)
```
**Transferring Items:**
```python
# Transfer a container to a new location
transfer = benchling.containers.transfer(
container_id="cont_abc123",
destination_id="box_xyz789"
)
```
**Key Inventory Operations:**
- Create containers, boxes, locations, plates
- Update inventory item properties
- Transfer items between locations
- Check in/out items
- Batch operations for bulk transfers
### 4. Notebook & Documentation
Interact with electronic lab notebook (ELN) entries, protocols, and templates.
**Creating Notebook Entries:**
```python
from benchling_sdk.models import EntryCreate
entry = benchling.entries.create(
EntryCreate(
name="Experiment 2025-10-20",
folder_id="fld_abc123",
schema_id="entry_schema_abc123",
fields=benchling.models.fields({"objective": "Test gene expression"})
)
)
```
**Linking Entities to Entries:**
```python
# Add references to entities in an entry
entry_link = benchling.entry_links.create(
entry_id="entry_abc123",
entity_id="seq_xyz789"
)
```
**Key Notebook Operations:**
- Create and update lab notebook entries
- Manage entry templates
- Link entities and results to entries
- Export entries for documentation
### 5. Workflows & Automation
Automate laboratory processes using Benchling's workflow system.
**Creating Workflow Tasks:**
```python
from benchling_sdk.models import WorkflowTaskCreate
task = benchling.workflow_tasks.create(
WorkflowTaskCreate(
name="PCR Amplification",
workflow_id="wf_abc123",
assignee_id="user_abc123",
fields=benchling.models.fields({"template": "seq_abc123"})
)
)
```
**Updating Task Status:**
```python
from benchling_sdk.models import WorkflowTaskUpdate
updated_task = benchling.workflow_tasks.update(
task_id="task_abc123",
workflow_task=WorkflowTaskUpdate(
status_id="status_complete_abc123"
)
)
```
**Asynchronous Operations:**
Some operations are asynchronous and return tasks. The SDK default `max_wait_seconds` for polling is **600 seconds** (since SDK 1.11.0):
```python
from benchling_sdk.helpers.tasks import wait_for_task
result = wait_for_task(
benchling,
task_id="task_abc123",
interval_wait_seconds=2,
max_wait_seconds=300, # override for long-running serverless handlers
)
```
**Key Workflow Operations:**
- Create and manage workflow tasks
- Update task statuses and assignments
- Execute bulk operations asynchronously
- Monitor task progress
### 6. Events & Integration
Subscribe to Benchling changes via **AWS EventBridge** (customer-owned bus) or **Webhooks** (recommended for new Benchling Apps). EventBridge delivers hydrated v2 API objects; webhooks use thinner payloads.
**Common EventBridge `detail-type` values:**
- `v2.dnaSequence.created`, `v2.dnaSequence.updated`
- `v2.entity.registered`
- `v2.entry.created`, `v2.entry.updated`
- `v2.workflowTask.updated.status`
- `v2.request.created`
**Minimal EventBridge rule** (filter request creation by schema name):
```json
{
"detail-type": ["v2.request.created"],
"detail": {
"schema": {
"name": ["Validated Request"]
}
}
}
```
**Lambda handler skeleton:**
```python
def handler(event, context):
detail_type = event["detail-type"]
detail = event["detail"]
if detail.get("deprecated"):
# Alert — migrate before Benchling removes this event type
pass
if detail.get("excludedProperties"):
# Payload exceeded 256 KB; re-fetch via detail["request"]["apiURL"]
pass
if detail_type == "v2.request.created":
request_id = (detail.get("request") or {}).get("id")
# Re-fetch authoritative state — events can be late or out of order
# request = benchling.requests.get_by_id(request_id)
return {"request_id": request_id}
return {"status": "ignored", "detail_type": detail_type}
```
**Setup flow:**
1. Tenant admin creates a subscription at `https://your-tenant.benchling.com/event-subscriptions`
2. Associate the AWS partner event source with a dedicated event bus immediately (within ~12 days)
3. Create rules + targets (Lambda, SQS, SNS) and grant invoke permissions
4. Validate with a CloudWatch Logs rule, then trigger a matching Benchling action
**Recovery:** EventBridge deliveries are not replayed. Use the [List Events API](https://benchling.com/api/reference#/Events/listEvents) for events up to ~2 weeks old after outages.
For payload schema, CloudFormation templates, SDK list/recovery examples, and validation steps, see `references/eventbridge.md`.
### 7. Data Warehouse & Analytics
Query historical Benchling data using SQL through the Data Warehouse.
**Access Method:**
The Benchling Data Warehouse provides SQL access to Benchling data for analytics and reporting. Connect using standard SQL clients with provided credentials.
**Common Queries:**
- Aggregate experimental results
- Analyze inventory trends
- Generate compliance reports
- Export data for external analysis
**Integration with Analysis Tools:**
- Jupyter notebooks for interactive analysis
- BI tools (Tableau, Looker, PowerBI)
- Custom dashboards
## Best Practices
### Error Handling
The SDK automatically retries failed requests:
```python
# Automatic retry for 429, 502, 503, 504 status codes
# Up to 5 retries with exponential backoff
# Customize retry behavior if needed
from benchling_sdk.retry import RetryStrategy
benchling = Benchling(
url=tenant_url,
auth_method=ApiKeyAuth(api_key),
retry_strategy=RetryStrategy(max_retries=3),
)
```
### Pagination Efficiency
Use generators for memory-efficient pagination:
```python
# Generator-based iteration
for page in benchling.dna_sequences.list():
for sequence in page:
process(sequence)
# Check estimated count without loading all pages
total = benchling.dna_sequences.list().estimated_count()
```
### Schema Fields Helper
Use the `fields()` helper for custom schema fields:
```python
# Convert dict to Fields object
custom_fields = benchling.models.fields({
"concentration": "100 ng/μL",
"date_prepared": "2025-10-20",
"notes": "High quality prep"
})
```
### Forward Compatibility
The SDK handles unknown enum values and types gracefully:
- Unknown enum values are preserved
- Unrecognized polymorphic types return `UnknownType`
- Allows working with newer API versions
### Security Considerations
- Never commit API keys or OAuth secrets to version control
- Read only named environment variables (`BENCHLING_TENANT_URL`, `BENCHLING_API_KEY`, etc.)
- Route network calls exclusively to your tenant URL
- Rotate keys if compromised; use OAuth for multi-user production apps
- Grant minimal necessary permissions for apps in the Developer Console
## Resources
### references/
Detailed reference documentation for in-depth information:
- **authentication.md** - Comprehensive authentication guide including OIDC, security best practices, and credential management
- **sdk_reference.md** - Detailed Python SDK reference with advanced patterns, examples, and all entity types
- **api_endpoints.md** - REST API endpoint reference for direct HTTP calls without the SDK
- **eventbridge.md** - EventBridge setup, event payload schema, rule examples, Lambda handler, validation, and recovery
Load these references as needed for specific integration requirements.
## Common Use Cases
**1. Bulk Entity Import:**
```python
# Import multiple sequences from FASTA file
from Bio import SeqIO
for record in SeqIO.parse("sequences.fasta", "fasta"):
benchling.dna_sequences.create(
DnaSequenceCreate(
name=record.id,
bases=str(record.seq),
is_circular=False,
folder_id="fld_abc123"
)
)
```
**2. Inventory Audit:**
```python
# List all containers in a specific location
containers = benchling.containers.list(
parent_storage_id="box_abc123"
)
for page in containers:
for container in page:
print(f"{container.name}: {container.barcode}")
```
**3. Workflow Automation:**
```python
# Update all pending tasks for a workflow
tasks = benchling.workflow_tasks.list(
workflow_id="wf_abc123",
status="pending"
)
for page in tasks:
for task in page:
# Perform automated checks
if auto_validate(task):
benchling.workflow_tasks.update(
task_id=task.id,
workflow_task=WorkflowTaskUpdate(
status_id="status_complete"
)
)
```
**4. Data Export:**
```python
# Export all sequences with specific properties
sequences = benchling.dna_sequences.list()
export_data = []
for page in sequences:
for seq in page:
if seq.schema_id == "target_schema_id":
export_data.append({
"id": seq.id,
"name": seq.name,
"bases": seq.bases,
"length": len(seq.bases)
})
# Save to CSV or database
import csv
with open("sequences.csv", "w") as f:
writer = csv.DictWriter(f, fieldnames=export_data[0].keys())
writer.writeheader()
writer.writerows(export_data)
```
## Additional Resources
- **Official Documentation:** https://docs.benchling.com
- **Python SDK Reference:** https://benchling.com/sdk-docs/
- **API Reference:** https://benchling.com/api/reference
- **Support:** [email protected]
skills/benchling-integration/references/api_endpoints.md
+883
View File
@@ -0,0 +1,883 @@
# Benchling REST API Endpoints Reference
## Base URL
All API requests use the base URL format:
```
https://{tenant}.benchling.com/api/v2
```
Replace `{tenant}` with your Benchling tenant name.
## API Versioning
Current API version: `v2`
The API version is specified in the URL path. Stable endpoints follow [Benchling stability guidelines](https://docs.benchling.com/docs/stability); `alpha` and `beta` endpoints may change with shorter notice.
## Authentication
All requests require authentication via HTTP headers:
**API Key (Basic Auth):**
```bash
curl -X GET \
https://your-tenant.benchling.com/api/v2/dna-sequences \
-u "your_api_key:"
```
**OAuth Bearer Token:**
```bash
curl -X GET \
https://your-tenant.benchling.com/api/v2/dna-sequences \
-H "Authorization: Bearer your_access_token"
```
## Common Headers
```
Authorization: Bearer {token}
Content-Type: application/json
Accept: application/json
```
## Response Format
All responses follow a consistent JSON structure:
**Single Resource:**
```json
{
"id": "seq_abc123",
"name": "My Sequence",
"bases": "ATCGATCG",
...
}
```
**List Response:**
```json
{
"results": [
{"id": "seq_1", "name": "Sequence 1"},
{"id": "seq_2", "name": "Sequence 2"}
],
"nextToken": "token_for_next_page"
}
```
## Pagination
List endpoints support pagination:
**Query Parameters:**
- `pageSize`: Number of items per page (default: 50, max: 100)
- `nextToken`: Token from previous response for next page
**Example:**
```bash
curl -X GET \
"https://your-tenant.benchling.com/api/v2/dna-sequences?pageSize=50&nextToken=abc123"
```
## Error Responses
**Format:**
```json
{
"error": {
"type": "NotFoundError",
"message": "DNA sequence not found",
"userMessage": "The requested sequence does not exist or you don't have access"
}
}
```
**Common Status Codes:**
- `200 OK`: Success
- `201 Created`: Resource created
- `400 Bad Request`: Invalid parameters
- `401 Unauthorized`: Missing or invalid credentials
- `403 Forbidden`: Insufficient permissions
- `404 Not Found`: Resource doesn't exist
- `422 Unprocessable Entity`: Validation error
- `429 Too Many Requests`: Rate limit exceeded
- `500 Internal Server Error`: Server error
## Core Endpoints
### DNA Sequences
**List DNA Sequences:**
```http
GET /api/v2/dna-sequences
Query Parameters:
- pageSize: integer (default: 50, max: 100)
- nextToken: string
- folderId: string
- schemaId: string
- name: string (filter by name)
- modifiedAt: string (ISO 8601 date)
```
**Get DNA Sequence:**
```http
GET /api/v2/dna-sequences/{sequenceId}
```
**Create DNA Sequence:**
```http
POST /api/v2/dna-sequences
Body:
{
"name": "My Plasmid",
"bases": "ATCGATCG",
"isCircular": true,
"folderId": "fld_abc123",
"schemaId": "ts_abc123",
"fields": {
"gene_name": {"value": "GFP"},
"resistance": {"value": "Kanamycin"}
},
"entityRegistryId": "src_abc123", // optional for registration
"namingStrategy": "NEW_IDS" // optional for registration
}
```
**Update DNA Sequence:**
```http
PATCH /api/v2/dna-sequences/{sequenceId}
Body:
{
"name": "Updated Plasmid",
"fields": {
"gene_name": {"value": "mCherry"}
}
}
```
**Archive DNA Sequence:**
```http
POST /api/v2/dna-sequences:archive
Body:
{
"dnaSequenceIds": ["seq_abc123"],
"reason": "Deprecated construct"
}
```
### RNA Sequences
**List RNA Sequences:**
```http
GET /api/v2/rna-sequences
```
**Get RNA Sequence:**
```http
GET /api/v2/rna-sequences/{sequenceId}
```
**Create RNA Sequence:**
```http
POST /api/v2/rna-sequences
Body:
{
"name": "gRNA-001",
"bases": "AUCGAUCG",
"folderId": "fld_abc123",
"fields": {
"target_gene": {"value": "TP53"}
}
}
```
**Update RNA Sequence:**
```http
PATCH /api/v2/rna-sequences/{sequenceId}
```
**Archive RNA Sequence:**
```http
POST /api/v2/rna-sequences:archive
```
### Amino Acid (Protein) Sequences
**List AA Sequences:**
```http
GET /api/v2/aa-sequences
```
**Get AA Sequence:**
```http
GET /api/v2/aa-sequences/{sequenceId}
```
**Create AA Sequence:**
```http
POST /api/v2/aa-sequences
Body:
{
"name": "GFP Protein",
"aminoAcids": "MSKGEELFTGVVPILVELDGDVNGHKF",
"folderId": "fld_abc123"
}
```
### Custom Entities
**List Custom Entities:**
```http
GET /api/v2/custom-entities
Query Parameters:
- schemaId: string (required to filter by type)
- pageSize: integer
- nextToken: string
```
**Get Custom Entity:**
```http
GET /api/v2/custom-entities/{entityId}
```
**Create Custom Entity:**
```http
POST /api/v2/custom-entities
Body:
{
"name": "HEK293T-Clone5",
"schemaId": "ts_cellline_abc",
"folderId": "fld_abc123",
"fields": {
"passage_number": {"value": "15"},
"mycoplasma_test": {"value": "Negative"}
}
}
```
**Update Custom Entity:**
```http
PATCH /api/v2/custom-entities/{entityId}
Body:
{
"fields": {
"passage_number": {"value": "16"}
}
}
```
### Mixtures
**List Mixtures:**
```http
GET /api/v2/mixtures
```
**Create Mixture:**
```http
POST /api/v2/mixtures
Body:
{
"name": "LB-Amp Media",
"folderId": "fld_abc123",
"schemaId": "ts_mixture_abc",
"ingredients": [
{
"componentEntityId": "ent_lb_base",
"amount": {"value": "1000", "units": "mL"}
},
{
"componentEntityId": "ent_ampicillin",
"amount": {"value": "100", "units": "mg"}
}
]
}
```
### Containers
**List Containers:**
```http
GET /api/v2/containers
Query Parameters:
- parentStorageId: string (filter by location/box)
- schemaId: string
- barcode: string
```
**Get Container:**
```http
GET /api/v2/containers/{containerId}
```
**Create Container:**
```http
POST /api/v2/containers
Body:
{
"name": "Sample-001",
"schemaId": "cont_schema_abc",
"barcode": "CONT001",
"parentStorageId": "box_abc123",
"fields": {
"concentration": {"value": "100 ng/μL"},
"volume": {"value": "50 μL"}
}
}
```
**Update Container:**
```http
PATCH /api/v2/containers/{containerId}
Body:
{
"fields": {
"volume": {"value": "45 μL"}
}
}
```
**Transfer Container:**
```http
POST /api/v2/containers:transfer
Body:
{
"containerIds": ["cont_abc123"],
"destinationStorageId": "box_xyz789"
}
```
**Check Out Container:**
```http
POST /api/v2/containers:checkout
Body:
{
"containerIds": ["cont_abc123"],
"comment": "Taking to bench"
}
```
**Check In Container:**
```http
POST /api/v2/containers:checkin
Body:
{
"containerIds": ["cont_abc123"],
"locationId": "bench_loc_abc"
}
```
### Boxes
**List Boxes:**
```http
GET /api/v2/boxes
Query Parameters:
- parentStorageId: string
- schemaId: string
```
**Get Box:**
```http
GET /api/v2/boxes/{boxId}
```
**Create Box:**
```http
POST /api/v2/boxes
Body:
{
"name": "Freezer-A-Box-01",
"schemaId": "box_schema_abc",
"parentStorageId": "loc_freezer_a",
"barcode": "BOX001"
}
```
### Locations
**List Locations:**
```http
GET /api/v2/locations
```
**Get Location:**
```http
GET /api/v2/locations/{locationId}
```
**Create Location:**
```http
POST /api/v2/locations
Body:
{
"name": "Freezer A - Shelf 2",
"parentStorageId": "loc_freezer_a",
"barcode": "LOC-A-S2"
}
```
### Plates
**List Plates:**
```http
GET /api/v2/plates
```
**Get Plate:**
```http
GET /api/v2/plates/{plateId}
```
**Create Plate:**
```http
POST /api/v2/plates
Body:
{
"name": "PCR-Plate-001",
"schemaId": "plate_schema_abc",
"barcode": "PLATE001",
"wells": [
{"position": "A1", "entityId": "ent_abc"},
{"position": "A2", "entityId": "ent_xyz"}
]
}
```
### Entries (Notebook)
**List Entries:**
```http
GET /api/v2/entries
Query Parameters:
- folderId: string
- schemaId: string
- modifiedAt: string
```
**Get Entry:**
```http
GET /api/v2/entries/{entryId}
```
**Create Entry:**
```http
POST /api/v2/entries
Body:
{
"name": "Experiment 2025-10-20",
"folderId": "fld_abc123",
"schemaId": "entry_schema_abc",
"fields": {
"objective": {"value": "Test gene expression"},
"date": {"value": "2025-10-20"}
}
}
```
**Update Entry:**
```http
PATCH /api/v2/entries/{entryId}
Body:
{
"fields": {
"results": {"value": "Successful expression"}
}
}
```
### Workflow Tasks
**List Workflow Tasks:**
```http
GET /api/v2/tasks
Query Parameters:
- workflowId: string
- statusIds: string[] (comma-separated)
- assigneeId: string
```
**Get Task:**
```http
GET /api/v2/tasks/{taskId}
```
**Create Task:**
```http
POST /api/v2/tasks
Body:
{
"name": "PCR Amplification",
"workflowId": "wf_abc123",
"assigneeId": "user_abc123",
"schemaId": "task_schema_abc",
"fields": {
"template": {"value": "seq_abc123"},
"priority": {"value": "High"}
}
}
```
**Update Task:**
```http
PATCH /api/v2/tasks/{taskId}
Body:
{
"statusId": "status_complete_abc",
"fields": {
"completion_date": {"value": "2025-10-20"}
}
}
```
### Folders
**List Folders:**
```http
GET /api/v2/folders
Query Parameters:
- projectId: string
- parentFolderId: string
```
**Get Folder:**
```http
GET /api/v2/folders/{folderId}
```
**Create Folder:**
```http
POST /api/v2/folders
Body:
{
"name": "2025 Experiments",
"parentFolderId": "fld_parent_abc",
"projectId": "proj_abc123"
}
```
### Projects
**List Projects:**
```http
GET /api/v2/projects
```
**Get Project:**
```http
GET /api/v2/projects/{projectId}
```
### Users
**Get Current User:**
```http
GET /api/v2/users/me
```
**List Users:**
```http
GET /api/v2/users
```
**Get User:**
```http
GET /api/v2/users/{userId}
```
### Teams
**List Teams:**
```http
GET /api/v2/teams
```
**Get Team:**
```http
GET /api/v2/teams/{teamId}
```
### Schemas
**List Schemas:**
```http
GET /api/v2/schemas
Query Parameters:
- entityType: string (e.g., "dna_sequence", "custom_entity")
```
**Get Schema:**
```http
GET /api/v2/schemas/{schemaId}
```
### Registries
**List Registries:**
```http
GET /api/v2/registries
```
**Get Registry:**
```http
GET /api/v2/registries/{registryId}
```
## Bulk Operations
### Batch Archive
**Archive Multiple Entities:**
```http
POST /api/v2/{entity-type}:archive
Body:
{
"{entity}Ids": ["id1", "id2", "id3"],
"reason": "Cleanup"
}
```
### Batch Transfer
**Transfer Multiple Containers:**
```http
POST /api/v2/containers:bulk-transfer
Body:
{
"transfers": [
{"containerId": "cont_1", "destinationId": "box_a"},
{"containerId": "cont_2", "destinationId": "box_b"}
]
}
```
## Async Operations
Some operations return task IDs for async processing:
**Response:**
```json
{
"taskId": "task_abc123"
}
```
**Check Task Status:**
```http
GET /api/v2/tasks/{taskId}
Response:
{
"id": "task_abc123",
"status": "RUNNING", // or "SUCCEEDED", "FAILED"
"message": "Processing...",
"response": {...} // Available when status is SUCCEEDED
}
```
## Field Value Format
Custom schema fields use a specific format:
**Simple Value:**
```json
{
"field_name": {
"value": "Field Value"
}
}
```
**Dropdown:**
```json
{
"dropdown_field": {
"value": "Option1" // Must match exact option name
}
}
```
**Date:**
```json
{
"date_field": {
"value": "2025-10-20" // Format: YYYY-MM-DD
}
}
```
**Entity Link:**
```json
{
"entity_link_field": {
"value": "seq_abc123" // Entity ID
}
}
```
**Numeric:**
```json
{
"numeric_field": {
"value": "123.45" // String representation
}
}
```
## Rate Limiting
**Limits:**
- Default: 100 requests per 10 seconds per user/app
- Rate limit headers included in responses:
- `X-RateLimit-Limit`: Total allowed requests
- `X-RateLimit-Remaining`: Remaining requests
- `X-RateLimit-Reset`: Unix timestamp when limit resets
**Handling 429 Responses:**
```json
{
"error": {
"type": "RateLimitError",
"message": "Rate limit exceeded",
"retryAfter": 5 // Seconds to wait
}
}
```
## Filtering and Searching
**Common Query Parameters:**
- `name`: Partial name match
- `modifiedAt`: ISO 8601 datetime
- `createdAt`: ISO 8601 datetime
- `schemaId`: Filter by schema
- `folderId`: Filter by folder
- `archived`: Boolean (include archived items)
**Example:**
```bash
curl -X GET \
"https://tenant.benchling.com/api/v2/dna-sequences?name=plasmid&folderId=fld_abc&archived=false"
```
## Best Practices
### Request Efficiency
1. **Use appropriate page sizes:**
- Default: 50 items
- Max: 100 items
- Adjust based on needs
2. **Filter on server-side:**
- Use query parameters instead of client filtering
- Reduces data transfer and processing
3. **Batch operations:**
- Use bulk endpoints when available
- Archive/transfer multiple items in one request
### Error Handling
```javascript
// Example error handling
async function fetchSequence(id) {
try {
const response = await fetch(
`https://tenant.benchling.com/api/v2/dna-sequences/${id}`,
{
headers: {
'Authorization': `Bearer ${token}`,
'Accept': 'application/json'
}
}
);
if (!response.ok) {
if (response.status === 429) {
// Rate limit - retry with backoff
const retryAfter = response.headers.get('Retry-After');
await sleep(retryAfter * 1000);
return fetchSequence(id);
} else if (response.status === 404) {
return null; // Not found
} else {
throw new Error(`API error: ${response.status}`);
}
}
return await response.json();
} catch (error) {
console.error('Request failed:', error);
throw error;
}
}
```
### Pagination Loop
```javascript
async function getAllSequences() {
let allSequences = [];
let nextToken = null;
do {
const url = new URL('https://tenant.benchling.com/api/v2/dna-sequences');
if (nextToken) {
url.searchParams.set('nextToken', nextToken);
}
url.searchParams.set('pageSize', '100');
const response = await fetch(url, {
headers: {
'Authorization': `Bearer ${token}`,
'Accept': 'application/json'
}
});
const data = await response.json();
allSequences = allSequences.concat(data.results);
nextToken = data.nextToken;
} while (nextToken);
return allSequences;
}
```
## References
- **API Documentation:** https://benchling.com/api/reference
- **Interactive API Explorer:** https://your-tenant.benchling.com/api/reference (requires authentication)
- **Changelog:** https://docs.benchling.com/changelog
skills/benchling-integration/references/authentication.md
+390
View File
@@ -0,0 +1,390 @@
# Benchling Authentication Reference
## Authentication Methods
Benchling supports three authentication methods, each suited for different use cases.
### 1. API Key Authentication (Basic Auth)
**Best for:** Personal scripts, prototyping, single-user integrations
**How it works:**
- Use your API key as the username in HTTP Basic authentication
- Leave the password field empty
- All requests must use HTTPS
**Obtaining an API Key:**
1. Log in to your Benchling account
2. Navigate to Profile Settings
3. Find the API Key section
4. Generate a new API key
5. Store it securely (it will only be shown once)
**Python SDK Usage:**
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=ApiKeyAuth("your_api_key_here")
)
```
**Direct HTTP Usage:**
```bash
curl -X GET \
https://your-tenant.benchling.com/api/v2/dna-sequences \
-u "your_api_key_here:"
```
Note the colon after the API key with no password.
**Environment Variable Pattern:**
```python
import os
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
api_key = os.environ.get("BENCHLING_API_KEY")
tenant_url = os.environ.get("BENCHLING_TENANT_URL")
benchling = Benchling(
url=tenant_url,
auth_method=ApiKeyAuth(api_key)
)
```
### 2. OAuth 2.0 Client Credentials
**Best for:** Multi-user applications, service accounts, production integrations
**How it works:**
1. Register an application in Benchling's Developer Console
2. Obtain client ID and client secret
3. Exchange credentials for an access token
4. Use the access token for API requests
5. Refresh token when expired
**Registering an App:**
1. Log in to Benchling as an admin
2. Navigate to Developer Console
3. Create a new App
4. Record the client ID and client secret
5. Configure OAuth redirect URIs and permissions
**Python SDK Usage:**
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2
auth_method = ClientCredentialsOAuth2(
client_id="your_client_id",
client_secret="your_client_secret"
)
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=auth_method
)
```
The SDK automatically handles token refresh.
**Direct HTTP Token Flow:**
```bash
# Get access token
curl -X POST \
https://your-tenant.benchling.com/api/v2/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "grant_type=client_credentials" \
-d "client_id=your_client_id" \
-d "client_secret=your_client_secret"
# Response:
# {
# "access_token": "token_here",
# "token_type": "Bearer",
# "expires_in": 3600
# }
# Use access token
curl -X GET \
https://your-tenant.benchling.com/api/v2/dna-sequences \
-H "Authorization: Bearer access_token_here"
```
### 3. OpenID Connect (OIDC)
**Best for:** Enterprise integrations with existing identity providers, SSO scenarios
**How it works:**
- Authenticate users through your identity provider (Okta, Azure AD, etc.)
- Identity provider issues an ID token with email claim
- Benchling verifies the token against the OpenID configuration endpoint
- Matches authenticated user by email
**Requirements:**
- Enterprise Benchling account
- Configured identity provider (IdP)
- IdP must issue tokens with email claims
- Email in token must match Benchling user email
**Identity Provider Configuration:**
1. Configure your IdP to issue OpenID Connect tokens
2. Ensure tokens include the `email` claim
3. Provide Benchling with your IdP's OpenID configuration URL
4. Benchling will verify tokens against this configuration
**Python Usage:**
```python
# Assuming you have an ID token from your IdP
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.oidc_auth import OidcAuth
auth_method = OidcAuth(id_token="id_token_from_idp")
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=auth_method
)
```
**Direct HTTP Usage:**
```bash
curl -X GET \
https://your-tenant.benchling.com/api/v2/dna-sequences \
-H "Authorization: Bearer id_token_here"
```
## Security Best Practices
### Credential Storage
**DO:**
- Store credentials in environment variables
- Use password managers or secret management services (AWS Secrets Manager, HashiCorp Vault)
- Encrypt credentials at rest
- Use different credentials for dev/staging/production
**DON'T:**
- Commit credentials to version control
- Hardcode credentials in source files
- Share credentials via email or chat
- Store credentials in plain text files
**Example with scoped environment variables:**
```python
import os
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
api_key = os.environ.get("BENCHLING_API_KEY")
tenant_url = os.environ.get("BENCHLING_TENANT_URL")
if not api_key or not tenant_url:
raise ValueError("Set BENCHLING_API_KEY and BENCHLING_TENANT_URL")
benchling = Benchling(
url=tenant_url,
auth_method=ApiKeyAuth(api_key),
)
```
Do not call `load_dotenv()` without filtering, and never iterate over `os.environ` to collect secrets.
### Credential Rotation
**API Key Rotation:**
1. Generate a new API key in Profile Settings
2. Update your application to use the new key
3. Verify the new key works
4. Delete the old API key
**App Secret Rotation:**
1. Navigate to Developer Console
2. Select your app
3. Generate new client secret
4. Update your application configuration
5. Delete the old secret after verifying
**Best Practice:** Rotate credentials regularly (e.g., every 90 days) and immediately if compromised.
### Access Control
**Principle of Least Privilege:**
- Grant only the minimum necessary permissions
- Use service accounts (apps) instead of personal accounts for automation
- Review and audit permissions regularly
**App Permissions:**
Apps require explicit access grants to:
- Organizations
- Teams
- Projects
- Folders
Configure these in the Developer Console when setting up your app.
**User Permissions:**
API access mirrors UI permissions:
- Users can only access data they have permission to view/edit in the UI
- Suspended users lose API access
- Archived apps lose API access until unarchived
### Network Security
**HTTPS Only:**
All Benchling API requests must use HTTPS. HTTP requests will be rejected.
**IP Allowlisting (Enterprise):**
Some enterprise accounts can restrict API access to specific IP ranges. Contact Benchling support to configure.
**Rate Limiting:**
Benchling implements rate limiting to prevent abuse:
- Default: 100 requests per 10 seconds per user/app
- 429 status code returned when rate limit exceeded
- SDK automatically retries with exponential backoff
### Audit Logging
**Tracking API Usage:**
- All API calls are logged with user/app identity
- OAuth apps show proper audit trails with user attribution
- API key calls are attributed to the key owner
- Review audit logs in Benchling's admin console
**Best Practice for Apps:**
Use OAuth instead of API keys when multiple users interact through your app. This ensures proper audit attribution to the actual user, not just the app.
## Troubleshooting
### Common Authentication Errors
**401 Unauthorized:**
- Invalid or expired credentials
- API key not properly formatted
- Missing "Authorization" header
**Solution:**
- Verify credentials are correct
- Check API key is not expired or deleted
- Ensure proper header format: `Authorization: Bearer <token>`
**403 Forbidden:**
- Valid credentials but insufficient permissions
- User doesn't have access to the requested resource
- App not granted access to the organization/project
**Solution:**
- Check user/app permissions in Benchling
- Grant necessary access in Developer Console (for apps)
- Verify the resource exists and user has access
**429 Too Many Requests:**
- Rate limit exceeded
- Too many requests in short time period
**Solution:**
- Implement exponential backoff
- SDK handles this automatically
- Consider caching results
- Spread requests over time
### Testing Authentication
**Quick Test with curl:**
```bash
# Test API key
curl -X GET \
https://your-tenant.benchling.com/api/v2/users/me \
-u "your_api_key:" \
-v
# Test OAuth token
curl -X GET \
https://your-tenant.benchling.com/api/v2/users/me \
-H "Authorization: Bearer your_token" \
-v
```
The `/users/me` endpoint returns the authenticated user's information and is useful for verifying credentials.
**Python SDK Test:**
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
try:
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=ApiKeyAuth("your_api_key")
)
# Test authentication
user = benchling.users.get_me()
print(f"Authenticated as: {user.name} ({user.email})")
except Exception as e:
print(f"Authentication failed: {e}")
```
## Multi-Tenant Considerations
If working with multiple Benchling tenants, use separate named keys per tenant (for example `BENCHLING_PROD_API_KEY` and `BENCHLING_STAGING_API_KEY`) rather than reading the entire environment:
```python
import os
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
tenants = {
"production": {
"url": os.environ.get("BENCHLING_PROD_TENANT_URL"),
"api_key": os.environ.get("BENCHLING_PROD_API_KEY"),
},
"staging": {
"url": os.environ.get("BENCHLING_STAGING_TENANT_URL"),
"api_key": os.environ.get("BENCHLING_STAGING_API_KEY"),
},
}
clients = {}
for name, config in tenants.items():
if not config["url"] or not config["api_key"]:
raise ValueError(f"Missing credentials for {name} tenant")
clients[name] = Benchling(
url=config["url"],
auth_method=ApiKeyAuth(config["api_key"]),
)
prod_sequences = clients["production"].dna_sequences.list()
```
## Advanced: Custom HTTPS Clients
For environments with self-signed certificates or corporate proxies:
```python
import httpx
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
# Custom httpx client with certificate verification
custom_client = httpx.Client(
verify="/path/to/custom/ca-bundle.crt",
timeout=30.0
)
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=ApiKeyAuth("your_api_key"),
http_client=custom_client
)
```
## References
- **Official Authentication Docs:** https://docs.benchling.com/docs/authentication
- **Developer Console:** https://your-tenant.benchling.com/developer
- **SDK Documentation:** https://benchling.com/sdk-docs/
skills/benchling-integration/references/eventbridge.md
+255
View File
@@ -0,0 +1,255 @@
# Benchling Events via AWS EventBridge
Real-time integrations that react to Benchling changes (entity registration, inventory transfers, workflow updates, and more).
**Official docs:**
- [Getting Started with Events](https://docs.benchling.com/docs/events-getting-started)
- [Events Reference (payloads and event types)](https://docs.benchling.com/docs/events-reference)
- [Events FAQs](https://docs.benchling.com/docs/events-faqs)
- [List Events API](https://benchling.com/api/reference#/Events/listEvents)
**Delivery methods:** Benchling supports **Webhooks** (recommended for new apps) and **AWS EventBridge** (customer-owned event bus). EventBridge payloads are **hydrated** (full v2 API objects in `detail`); webhooks use thinner payloads. See the getting-started guide for trade-offs.
---
## Setup checklist
1. **Tenant admin** enables Developer Platform access and opens [Event Subscriptions](https://your-tenant.benchling.com/event-subscriptions) (Feature settings → Developer Console → Events).
2. Create a subscription with:
- AWS account ID and region
- Event bus name (e.g. `benchling-integrations`)
- Event types to receive (see [Events Reference](https://docs.benchling.com/docs/events-reference))
3. **Immediately** associate the partner event source with a new EventBridge bus in AWS (within ~12 days or the source expires).
4. Create EventBridge rules with `detail-type` / `detail` filters and targets (Lambda, SQS, SNS, CloudWatch Logs).
5. Grant invoke permissions (`AWS::Lambda::Permission`, queue policies, etc.).
6. Validate with a CloudWatch Logs rule on the bus `source`, then trigger a test action in Benchling.
Subscription statuses: `Pending` (needs bus association), `Active`, `Expired` (resubscribe in Benchling).
---
## EventBridge event envelope
All EventBridge deliveries share this top-level shape. The resource body lives under `detail` under a key that varies by event (for example `entry`, `assayRun`, `dnaSequence`).
```json
{
"version": "0",
"id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
"detail-type": "v2.dnaSequence.created",
"source": "aws.partner/benchling.com/your-tenant/your-subscription-name",
"account": "123456789012",
"time": "2025-10-20T14:30:00.000000+00:00",
"region": "us-west-2",
"resources": [],
"detail": {
"id": "evt_abc123",
"eventType": "v2.dnaSequence.created",
"createdAt": "2025-10-20T14:30:00.000000+00:00",
"deprecated": false,
"excludedProperties": [],
"schema": {
"id": "ts_abc123",
"name": "Plasmid"
},
"dnaSequence": {
"id": "seq_xyz789",
"name": "My Plasmid",
"apiURL": "https://your-tenant.benchling.com/api/v2/dna-sequences/seq_xyz789"
}
}
}
```
**Naming:** `detail-type` and `detail.eventType` follow `<version>.<resource>.<action>` (for example `v2.request.created`, `v2.workflowTask.updated.status`).
**Do not treat payloads as authoritative.** Events may arrive late or out of order. Re-fetch objects with the SDK/API when you need current state.
**Oversized events (>256 KB):** Dropped fields appear in `detail.excludedProperties`. Use `apiURL` on the resource object to fetch the full record.
---
## Minimal EventBridge rule (CloudFormation)
Route `v2.request.created` events for a specific request schema to a Lambda:
```yaml
AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: Benchling request.created → Lambda
Parameters:
BenchlingEventBusName:
Type: String
Description: Partner event bus name from Benchling subscription
Resources:
RequestCreatedRule:
Type: AWS::Events::Rule
Properties:
Name: benchling-request-created
EventBusName: !Ref BenchlingEventBusName
State: ENABLED
EventPattern:
detail-type:
- v2.request.created
detail:
schema:
name:
- Validated Request
Targets:
- Id: HandleRequestCreated
Arn: !GetAtt HandleEventLambda.Arn
HandleEventLambda:
Type: AWS::Serverless::Function
Properties:
Handler: app.handler
Runtime: python3.12
CodeUri: src/
Timeout: 30
AllowEventBridgeInvoke:
Type: AWS::Lambda::Permission
Properties:
Action: lambda:InvokeFunction
FunctionName: !Ref HandleEventLambda
Principal: events.amazonaws.com
SourceArn: !GetAtt RequestCreatedRule.Arn
```
**Other filter examples** (from Benchling docs):
```json
{
"detail-type": ["v2.assayRun.updated"],
"detail": {
"updates": ["my_field"]
}
}
```
```json
{
"detail-type": ["v2.entity.registered"],
"detail": {
"entity": {
"schema": {
"id": ["ts_MySchemaId"]
}
}
}
}
```
---
## Lambda handler skeleton (Python)
```python
import json
import logging
import os
import boto3
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# Optional: re-fetch via SDK when payload may be stale or truncated
# from benchling_sdk.benchling import Benchling
# from benchling_sdk.auth.api_key_auth import ApiKeyAuth
#
# benchling = Benchling(
# url=os.environ["BENCHLING_TENANT_URL"],
# auth_method=ApiKeyAuth(os.environ["BENCHLING_API_KEY"]),
# )
def handler(event, context):
"""Process a single Benchling EventBridge delivery."""
detail_type = event.get("detail-type")
detail = event.get("detail") or {}
logger.info(
"benchling_event",
extra={
"detail_type": detail_type,
"event_id": detail.get("id"),
"benchling_event_type": detail.get("eventType"),
},
)
if detail.get("deprecated"):
logger.warning("deprecated_event_type: %s", detail_type)
if detail.get("excludedProperties"):
logger.warning(
"truncated_payload excluded=%s", detail.get("excludedProperties")
)
if detail_type == "v2.dnaSequence.created":
sequence = detail.get("dnaSequence") or {}
sequence_id = sequence.get("id")
if not sequence_id:
raise ValueError("missing dnaSequence.id in event detail")
# Prefer API lookup for authoritative data:
# seq = benchling.dna_sequences.get_by_id(sequence_id)
return {"status": "ok", "sequence_id": sequence_id}
if detail_type == "v2.workflowTask.updated.status":
task = detail.get("workflowTask") or {}
return {"status": "ok", "task_id": task.get("id")}
logger.info("no_handler_for_detail_type: %s", detail_type)
return {"status": "ignored", "detail_type": detail_type}
```
For serverless timeouts: SDK `wait_for_task` defaults to 600s — keep Lambda timeouts and EventBridge retry/DLQ settings aligned with expected processing time.
---
## Validation steps
1. **Subscription active:** In Benchling, subscription status is `Active` (not `Pending` or `Expired`).
2. **Partner source associated:** In AWS EventBridge → Partner event sources, source is associated with your bus.
3. **Log all events:** Add a catch-all rule targeting a CloudWatch log group, filtering on your bus `source` (shown in Benchling subscription UI).
4. **Trigger a test event:** Create or update an object matching your rule filter (for example register a DNA sequence).
5. **Inspect logs:** Confirm `detail-type`, `detail.id`, and resource IDs match expectations.
6. **Re-fetch check:** Call the SDK/API for the resource ID and confirm it matches your integration logic.
---
## Recovering missed events
Benchling does **not** replay EventBridge deliveries. After an outage:
1. Get the affected time window from Benchling support.
2. List historical events with the [List Events API](https://benchling.com/api/reference#/Events/listEvents) (retained ~2 weeks).
3. Re-route recovered events through your own infrastructure.
SDK example (ISO 8601 timestamp; see API reference for filters):
```python
events = benchling.events.list(
created_atgte="2025-10-20T00:00:00+00:00",
event_types="v2.dnaSequence.created",
)
for page in events:
for evt in page:
print(evt.event_type, evt.id)
```
---
## EventBridge vs Webhooks
| | EventBridge | Webhooks |
|---|-------------|----------|
| Setup | Benchling console + AWS bus/rules | Benchling App configuration |
| Payload | Hydrated v2 API objects | Thin IDs + metadata |
| Filtering | EventBridge `EventPattern` | App code |
| Permissions | Not permissioned at delivery | Inherited from app |
For new Benchling Apps, Benchling recommends **webhooks** unless you already standardize on EventBridge in AWS. See [Getting Started with Webhooks](https://docs.benchling.com/docs/getting-started-with-webhooks).
skills/benchling-integration/references/sdk_reference.md
+772
View File
@@ -0,0 +1,772 @@
# Benchling Python SDK Reference
## Installation & Setup
### Installation
```bash
# Stable release (recommended)
uv pip install "benchling-sdk==1.25.0"
# Preview builds — alpha functionality, not for production
uv pip install "benchling-sdk" --prerelease allow
```
### Requirements
- Python 3.9+ (3.12 supported since SDK 1.11.0; repo recommends 3.11+)
- API access enabled on your Benchling tenant
- Developer Platform access from your tenant admin (for apps and events)
### Basic Initialization
```python
import os
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
benchling = Benchling(
url=os.environ["BENCHLING_TENANT_URL"],
auth_method=ApiKeyAuth(os.environ["BENCHLING_API_KEY"]),
)
```
## SDK Architecture
### Main Classes
**Benchling Client:**
The `benchling_sdk.benchling.Benchling` class is the root of all SDK interactions. It provides access to all resource endpoints:
```python
benchling.dna_sequences # DNA sequence operations
benchling.rna_sequences # RNA sequence operations
benchling.aa_sequences # Amino acid sequence operations
benchling.custom_entities # Custom entity operations
benchling.mixtures # Mixture operations
benchling.containers # Container operations
benchling.boxes # Box operations
benchling.locations # Location operations
benchling.plates # Plate operations
benchling.entries # Notebook entry operations
benchling.workflow_tasks # Workflow task operations
benchling.requests # Request operations
benchling.folders # Folder operations
benchling.projects # Project operations
benchling.users # User operations
benchling.teams # Team operations
```
### Resource Pattern
All resources follow a consistent CRUD pattern:
```python
# Create
resource.create(CreateModel(...))
# Read (single)
resource.get_by_id("resource_id")
# Read (list)
resource.list(optional_filters...)
# Update
resource.update(id="resource_id", UpdateModel(...))
# Archive/Delete
resource.archive(id="resource_id")
```
## Entity Management
### DNA Sequences
**Create:**
```python
from benchling_sdk.models import DnaSequenceCreate
sequence = benchling.dna_sequences.create(
DnaSequenceCreate(
name="pET28a-GFP",
bases="ATCGATCGATCG",
is_circular=True,
folder_id="fld_abc123",
schema_id="ts_abc123",
fields=benchling.models.fields({
"gene_name": "GFP",
"resistance": "Kanamycin",
"copy_number": "High"
})
)
)
```
**Read:**
```python
# Get by ID
seq = benchling.dna_sequences.get_by_id("seq_abc123")
print(f"{seq.name}: {len(seq.bases)} bp")
# List with filters
sequences = benchling.dna_sequences.list(
folder_id="fld_abc123",
schema_id="ts_abc123",
name="pET28a" # Filter by name
)
for page in sequences:
for seq in page:
print(f"{seq.id}: {seq.name}")
```
**Update:**
```python
from benchling_sdk.models import DnaSequenceUpdate
updated = benchling.dna_sequences.update(
sequence_id="seq_abc123",
dna_sequence=DnaSequenceUpdate(
name="pET28a-GFP-v2",
fields=benchling.models.fields({
"gene_name": "eGFP",
"notes": "Codon optimized"
})
)
)
```
**Archive:**
```python
benchling.dna_sequences.archive(
sequence_id="seq_abc123",
reason="Deprecated construct"
)
```
### RNA Sequences
Similar pattern to DNA sequences:
```python
from benchling_sdk.models import RnaSequenceCreate, RnaSequenceUpdate
# Create
rna = benchling.rna_sequences.create(
RnaSequenceCreate(
name="gRNA-target1",
bases="AUCGAUCGAUCG",
folder_id="fld_abc123",
fields=benchling.models.fields({
"target_gene": "TP53",
"off_target_score": "95"
})
)
)
# Update
updated_rna = benchling.rna_sequences.update(
rna_sequence_id=rna.id,
rna_sequence=RnaSequenceUpdate(
fields=benchling.models.fields({
"validated": "Yes"
})
)
)
```
### Amino Acid (Protein) Sequences
```python
from benchling_sdk.models import AaSequenceCreate
protein = benchling.aa_sequences.create(
AaSequenceCreate(
name="Green Fluorescent Protein",
amino_acids="MSKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKF",
folder_id="fld_abc123",
fields=benchling.models.fields({
"molecular_weight": "27000",
"extinction_coefficient": "21000"
})
)
)
```
### Custom Entities
Custom entities are defined by your tenant's schemas:
```python
from benchling_sdk.models import CustomEntityCreate, CustomEntityUpdate
# Create
cell_line = benchling.custom_entities.create(
CustomEntityCreate(
name="HEK293T-Clone5",
schema_id="ts_cellline_abc123",
folder_id="fld_abc123",
fields=benchling.models.fields({
"passage_number": "15",
"mycoplasma_test": "Negative",
"freezing_date": "2025-10-15"
})
)
)
# Update
updated_cell_line = benchling.custom_entities.update(
entity_id=cell_line.id,
custom_entity=CustomEntityUpdate(
fields=benchling.models.fields({
"passage_number": "16",
"notes": "Expanded for experiment"
})
)
)
```
### Mixtures
Mixtures combine multiple components:
```python
from benchling_sdk.models import MixtureCreate, IngredientCreate
mixture = benchling.mixtures.create(
MixtureCreate(
name="LB-Amp Media",
folder_id="fld_abc123",
schema_id="ts_mixture_abc123",
ingredients=[
IngredientCreate(
component_entity_id="ent_lb_base",
amount="1000 mL"
),
IngredientCreate(
component_entity_id="ent_ampicillin",
amount="100 mg"
)
],
fields=benchling.models.fields({
"pH": "7.0",
"sterilized": "Yes"
})
)
)
```
### Registry Operations
**Direct Registry Registration:**
```python
# Register entity upon creation
registered_seq = benchling.dna_sequences.create(
DnaSequenceCreate(
name="Construct-001",
bases="ATCG",
is_circular=True,
folder_id="fld_abc123",
entity_registry_id="src_abc123",
naming_strategy="NEW_IDS" # or "IDS_FROM_NAMES"
)
)
print(f"Registry ID: {registered_seq.registry_id}")
```
**Naming Strategies:**
- `NEW_IDS`: Benchling generates new registry IDs
- `IDS_FROM_NAMES`: Use entity names as registry IDs (names must be unique)
## Inventory Management
### Containers
```python
from benchling_sdk.models import ContainerCreate, ContainerUpdate
# Create
container = benchling.containers.create(
ContainerCreate(
name="Sample-001-Tube",
schema_id="cont_schema_abc123",
barcode="CONT001",
parent_storage_id="box_abc123", # Place in box
fields=benchling.models.fields({
"concentration": "100 ng/μL",
"volume": "50 μL",
"sample_type": "gDNA"
})
)
)
# Update location
benchling.containers.transfer(
container_id=container.id,
destination_id="box_xyz789"
)
# Update properties
updated = benchling.containers.update(
container_id=container.id,
container=ContainerUpdate(
fields=benchling.models.fields({
"volume": "45 μL",
"notes": "Used 5 μL for PCR"
})
)
)
# Check out
benchling.containers.check_out(
container_id=container.id,
comment="Taking to bench"
)
# Check in
benchling.containers.check_in(
container_id=container.id,
location_id="bench_location_abc"
)
```
### Boxes
```python
from benchling_sdk.models import BoxCreate
box = benchling.boxes.create(
BoxCreate(
name="Freezer-A-Box-01",
schema_id="box_schema_abc123",
parent_storage_id="loc_freezer_a",
barcode="BOX001",
fields=benchling.models.fields({
"box_type": "81-place",
"temperature": "-80C"
})
)
)
# List containers in box
containers = benchling.containers.list(
parent_storage_id=box.id
)
```
### Locations
```python
from benchling_sdk.models import LocationCreate
location = benchling.locations.create(
LocationCreate(
name="Freezer A - Shelf 2",
parent_storage_id="loc_freezer_a",
barcode="LOC-A-S2"
)
)
```
### Plates
```python
from benchling_sdk.models import PlateCreate, WellCreate
# Create 96-well plate
plate = benchling.plates.create(
PlateCreate(
name="PCR-Plate-001",
schema_id="plate_schema_abc123",
barcode="PLATE001",
wells=[
WellCreate(
position="A1",
entity_id="sample_entity_abc"
),
WellCreate(
position="A2",
entity_id="sample_entity_xyz"
)
# ... more wells
]
)
)
```
## Notebook Operations
### Entries
```python
from benchling_sdk.models import EntryCreate, EntryUpdate
# Create entry
entry = benchling.entries.create(
EntryCreate(
name="Cloning Experiment 2025-10-20",
folder_id="fld_abc123",
schema_id="entry_schema_abc123",
fields=benchling.models.fields({
"objective": "Clone GFP into pET28a",
"date": "2025-10-20",
"experiment_type": "Molecular Biology"
})
)
)
# Update entry
updated_entry = benchling.entries.update(
entry_id=entry.id,
entry=EntryUpdate(
fields=benchling.models.fields({
"results": "Successful cloning, 10 colonies",
"notes": "Colony 5 shows best fluorescence"
})
)
)
```
### Linking Entities to Entries
```python
# Link DNA sequence to entry
link = benchling.entry_links.create(
entry_id="entry_abc123",
entity_id="seq_xyz789"
)
# List links for an entry
links = benchling.entry_links.list(entry_id="entry_abc123")
```
## Workflow Management
### Tasks
```python
from benchling_sdk.models import WorkflowTaskCreate, WorkflowTaskUpdate
# Create task
task = benchling.workflow_tasks.create(
WorkflowTaskCreate(
name="PCR Amplification",
workflow_id="wf_abc123",
assignee_id="user_abc123",
schema_id="task_schema_abc123",
fields=benchling.models.fields({
"template": "seq_abc123",
"primers": "Forward: ATCG, Reverse: CGAT",
"priority": "High"
})
)
)
# Update status
completed_task = benchling.workflow_tasks.update(
task_id=task.id,
workflow_task=WorkflowTaskUpdate(
status_id="status_complete_abc123",
fields=benchling.models.fields({
"completion_date": "2025-10-20",
"yield": "500 ng"
})
)
)
# List tasks
tasks = benchling.workflow_tasks.list(
workflow_id="wf_abc123",
status_ids=["status_pending", "status_in_progress"]
)
```
## Advanced Features
### Pagination
The SDK uses generators for memory-efficient pagination:
```python
# Automatic pagination
sequences = benchling.dna_sequences.list()
# Get estimated total count
total = sequences.estimated_count()
print(f"Total sequences: {total}")
# Iterate through all pages
for page in sequences:
for seq in page:
process(seq)
# Manual page size control
sequences = benchling.dna_sequences.list(page_size=50)
```
### Async Task Handling
Some operations are asynchronous and return task IDs:
```python
from benchling_sdk.helpers.tasks import wait_for_task
from benchling_sdk.errors import WaitForTaskExpiredError
# Start async operation
response = benchling.some_bulk_operation(...)
task_id = response.task_id
# Wait for completion
try:
result = wait_for_task(
benchling,
task_id=task_id,
interval_wait_seconds=2, # Poll every 2 seconds
max_wait_seconds=600 # Timeout after 10 minutes
)
print("Task completed successfully")
except WaitForTaskExpiredError:
print("Task timed out")
```
### Error Handling
```python
from benchling_sdk.errors import (
BenchlingError,
NotFoundError,
ValidationError,
UnauthorizedError
)
try:
sequence = benchling.dna_sequences.get_by_id("seq_invalid")
except NotFoundError:
print("Sequence not found")
except UnauthorizedError:
print("Insufficient permissions")
except ValidationError as e:
print(f"Invalid data: {e}")
except BenchlingError as e:
print(f"General Benchling error: {e}")
```
### Retry Strategy
Customize retry behavior:
```python
from benchling_sdk.benchling import Benchling
from benchling_sdk.auth.api_key_auth import ApiKeyAuth
from benchling_sdk.retry import RetryStrategy
# Custom retry configuration
retry_strategy = RetryStrategy(
max_retries=3,
backoff_factor=0.5,
status_codes_to_retry=[429, 502, 503, 504]
)
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=ApiKeyAuth("your_api_key"),
retry_strategy=retry_strategy
)
# Disable retries
benchling = Benchling(
url="https://your-tenant.benchling.com",
auth_method=ApiKeyAuth("your_api_key"),
retry_strategy=RetryStrategy(max_retries=0)
)
```
### Custom API Calls
For unsupported endpoints:
```python
# GET request with model parsing
from benchling_sdk.models import DnaSequence
response = benchling.api.get_modeled(
path="/api/v2/dna-sequences/seq_abc123",
response_type=DnaSequence
)
# POST request
from benchling_sdk.models import DnaSequenceCreate
response = benchling.api.post_modeled(
path="/api/v2/dna-sequences",
request_body=DnaSequenceCreate(...),
response_type=DnaSequence
)
# Raw requests
raw_response = benchling.api.get(
path="/api/v2/custom-endpoint",
params={"key": "value"}
)
```
### Batch Operations
Efficiently process multiple items:
```python
# Bulk create
from benchling_sdk.models import DnaSequenceCreate
sequences_to_create = [
DnaSequenceCreate(name=f"Seq-{i}", bases="ATCG", folder_id="fld_abc")
for i in range(100)
]
# Create in batches
batch_size = 10
for i in range(0, len(sequences_to_create), batch_size):
batch = sequences_to_create[i:i+batch_size]
for seq in batch:
benchling.dna_sequences.create(seq)
```
### Schema Fields Helper
Convert dictionaries to Fields objects:
```python
# Using fields helper
fields_dict = {
"concentration": "100 ng/μL",
"volume": "50 μL",
"quality_score": "8.5",
"date_prepared": "2025-10-20"
}
fields = benchling.models.fields(fields_dict)
# Use in create/update
container = benchling.containers.create(
ContainerCreate(
name="Sample-001",
schema_id="schema_abc",
fields=fields
)
)
```
### Forward Compatibility
The SDK handles unknown API values gracefully:
```python
# Unknown enum values are preserved
entity = benchling.dna_sequences.get_by_id("seq_abc")
# Even if API returns new enum value not in SDK, it's preserved
# Unknown polymorphic types return UnknownType
from benchling_sdk.models import UnknownType
if isinstance(entity, UnknownType):
print(f"Unknown type: {entity.type}")
# Can still access raw data
print(entity.raw_data)
```
## Best Practices
### Use Type Hints
```python
from benchling_sdk.models import DnaSequence, DnaSequenceCreate
from typing import List
def create_sequences(names: List[str], folder_id: str) -> List[DnaSequence]:
sequences = []
for name in names:
seq = benchling.dna_sequences.create(
DnaSequenceCreate(
name=name,
bases="ATCG",
folder_id=folder_id
)
)
sequences.append(seq)
return sequences
```
### Efficient Filtering
Use API filters instead of client-side filtering:
```python
# Good - filter on server
sequences = benchling.dna_sequences.list(
folder_id="fld_abc123",
schema_id="ts_abc123"
)
# Bad - loads everything then filters
all_sequences = benchling.dna_sequences.list()
filtered = [s for page in all_sequences for s in page if s.folder_id == "fld_abc123"]
```
### Resource Cleanup
```python
# Archive old entities
cutoff_date = "2024-01-01"
sequences = benchling.dna_sequences.list()
for page in sequences:
for seq in page:
if seq.created_at < cutoff_date:
benchling.dna_sequences.archive(
sequence_id=seq.id,
reason="Archiving old sequences"
)
```
## Troubleshooting
### Common Issues
**Import paths:**
```python
# Preferred (documented in getting started guide)
from benchling_sdk.benchling import Benchling
# Also valid in benchling-sdk 1.25+
from benchling_sdk import Benchling
```
**Field Validation:**
```python
# Fields must match schema
# Check schema field types in Benchling UI
fields = benchling.models.fields({
"numeric_field": "123", # Should be string even for numbers
"date_field": "2025-10-20", # Format: YYYY-MM-DD
"dropdown_field": "Option1" # Must match dropdown options exactly
})
```
**Pagination Exhaustion:**
```python
# Generators can only be iterated once
sequences = benchling.dna_sequences.list()
for page in sequences: # First iteration OK
pass
for page in sequences: # Second iteration returns nothing!
pass
# Solution: Create new generator
sequences = benchling.dna_sequences.list() # New generator
```
## References
- **SDK Source:** https://github.com/benchling/benchling-sdk
- **SDK Docs:** https://benchling.com/sdk-docs/
- **API Reference:** https://benchling.com/api/reference
- **Common Examples:** https://docs.benchling.com/docs/common-sdk-interactions-and-examples
skills/bgpt-paper-search/SKILL.md
+71
View File
@@ -0,0 +1,71 @@
---
name: bgpt-paper-search
description: Search scientific papers and retrieve structured experimental data extracted from full-text studies via the BGPT MCP server. Returns 25+ fields per paper including methods, results, sample sizes, quality scores, and conclusions. Use for literature reviews, evidence synthesis, and finding experimental details not available in abstracts alone.
license: MIT
compatibility: Requires the BGPT MCP server configured in the agent host (npx mcp-remote or npx bgpt-mcp), internet access to bgpt.pro, and an optional BGPT API key for paid usage.
metadata: {"version": "1.1", "skill-author": "BGPT", "website": "https://bgpt.pro/mcp", "github": "https://github.com/connerlambden/bgpt-mcp"}
---
# BGPT Paper Search
## Overview
BGPT is a remote MCP server that searches a curated database of scientific papers built from raw experimental data extracted from full-text studies. Unlike traditional literature databases that return titles and abstracts, BGPT returns structured data from the actual paper content — methods, quantitative results, sample sizes, quality assessments, and 25+ metadata fields per paper.
## When to Use This Skill
Use this skill when:
- Searching for scientific papers with specific experimental details
- Conducting systematic or scoping literature reviews
- Finding quantitative results, sample sizes, or effect sizes across studies
- Comparing methodologies used in different studies
- Looking for papers with quality scores or evidence grading
- Needing structured data from full-text papers (not just abstracts)
- Building evidence tables for meta-analyses or clinical guidelines
## Setup
BGPT is a remote MCP server — no local installation required. Configure it in your agent's MCP settings before use; this skill instructs the agent to call the `search_papers` MCP tool and does not enable MCP access by itself.
### Claude Desktop / Claude Code
Add to your MCP configuration:
```json
{
"mcpServers": {
"bgpt": {
"command": "npx",
"args": ["mcp-remote", "https://bgpt.pro/mcp/sse"]
}
}
}
```
### npm (alternative)
```bash
npx bgpt-mcp
```
## Usage
Once the BGPT MCP server is configured, call its `search_papers` tool via the agent's MCP interface (not via Bash):
```
Search for papers about: "CRISPR gene editing efficiency in human cells"
```
The server returns structured results including:
- **Title, authors, journal, year, DOI**
- **Methods**: Experimental techniques, models, protocols
- **Results**: Key findings with quantitative data
- **Sample sizes**: Number of subjects/samples
- **Quality scores**: Study quality assessments
- **Conclusions**: Author conclusions and implications
## Pricing
- **Free tier**: 50 searches per network, no API key required
- **Paid**: $0.01 per result with an API key from [bgpt.pro/mcp](https://bgpt.pro/mcp)
skills/bids/SKILL.md
+755
View File
@@ -0,0 +1,755 @@
---
name: bids
description: >
Use this skill when working with Brain Imaging Data Structure (BIDS) datasets:
organizing neuroscience and biomedical data (MRI, EEG, MEG, iEEG, PET, microscopy,
NIRS, motion capture, EMG, MR spectroscopy, behavioral), querying BIDS layouts,
validating compliance, converting DICOM to BIDS, writing metadata sidecars, or
creating BIDS derivatives.
license: https://creativecommons.org/licenses/by/4.0/
metadata: {"version": "1.0", "skill-author": "Yaroslav Halchenko"}
---
# Brain Imaging Data Structure (BIDS)
## Overview
The Brain Imaging Data Structure (BIDS) is a community standard for organizing and describing neuroscience and biomedical research datasets. It defines a consistent file naming convention, directory hierarchy, and metadata schema so that datasets are immediately understandable by humans and software tools alike. BIDS is governed by the BIDS Specification (currently v1.11.x) and is maintained by the community via the BIDS-Standard GitHub organization.
While BIDS originated for MRI, it has grown well beyond neuroimaging. The specification now covers 11 modalities spanning imaging, electrophysiology, and behavioral data:
- **Imaging**: MRI (structural, functional, diffusion, fieldmaps, perfusion/ASL), PET, microscopy
- **Electrophysiology**: EEG, MEG, iEEG (intracranial EEG), EMG
- **Other**: NIRS (near-infrared spectroscopy), motion capture, behavioral data (without imaging), MR spectroscopy
Active BEPs are extending BIDS further — notably BEP032 (microelectrode electrophysiology) will add support for extracellular recordings including Neuropixels probes, bringing BIDS to a prevalent methodology in animal neuroscience research (see also the neuropixels-analysis skill).
Adoption is required or strongly encouraged by major data repositories (OpenNeuro, DANDI), leading journals (NeuroImage, Human Brain Mapping, Scientific Data), and funding agencies (NIH, ERC).
The Python ecosystem for BIDS centers on **PyBIDS** (`pybids`) for querying and indexing BIDS datasets, and the **bids-validator** (Deno-based, available as PyPI package `bids-validator-deno` or via Deno directly) for compliance checking. Conversion from DICOM is typically done with **HeuDiConv**, **dcm2bids**, or **BIDScoin**.
## When to Use This Skill
Apply this skill when:
- Organizing raw neuroscience data (imaging, electrophysiology, behavioral) into BIDS-compliant directory structures
- Querying an existing BIDS dataset to find specific files by subject, session, task, run, or modality
- Validating a dataset against the BIDS specification before sharing or submission
- Converting DICOM data from scanners into BIDS format
- Writing or editing JSON sidecar metadata files
- Creating BIDS-compliant derivatives (preprocessed data, analysis outputs)
- Setting up a `dataset_description.json` for a new dataset
- Working with BIDS entities (subject, session, task, acquisition, run, etc.)
- Configuring `.bidsignore` to exclude files from validation
- Preparing data for upload to OpenNeuro, DANDI, or other BIDS-aware repositories
## Installation
```bash
# Core BIDS querying library
uv pip install pybids
# BIDS validator (Deno-based, installed via PyPI wrapper)
uv pip install bids-validator-deno
# Alternative: install directly via Deno
# deno install -g -A npm:bids-validator
# DICOM-to-BIDS converters (install as needed)
uv pip install heudiconv # HeuDiConv - heuristic-based DICOM conversion
uv pip install dcm2bids # dcm2bids - config-file-based conversion
# BIDScoin: uv pip install bidscoin
# Useful companions
uv pip install nibabel # NIfTI/other neuroimaging file I/O
uv pip install pydicom # DICOM file reading (used by converters)
```
## Core Workflows
### 1. BIDS Directory Structure
A minimal BIDS dataset follows this layout:
```
my_dataset/
dataset_description.json # Required: name, BIDSVersion, etc.
participants.tsv # Recommended: subject-level phenotypic data
participants.json # Recommended: column descriptions
README # Recommended: dataset documentation
CHANGES # Recommended: version history
.bidsignore # Optional: patterns to exclude from validation
sub-01/
anat/
sub-01_T1w.nii.gz
sub-01_T1w.json # Sidecar metadata
func/
sub-01_task-rest_bold.nii.gz
sub-01_task-rest_bold.json
sub-01_task-rest_events.tsv # Event timing for task fMRI
sub-01_task-rest_events.json
dwi/
sub-01_dwi.nii.gz
sub-01_dwi.json
sub-01_dwi.bvec
sub-01_dwi.bval
fmap/
sub-01_phasediff.nii.gz
sub-01_phasediff.json
sub-01_magnitude1.nii.gz
perf/
sub-01_asl.nii.gz
sub-01_asl.json
sub-01/
ses-pre/
anat/
sub-01_ses-pre_T1w.nii.gz
func/
sub-01_ses-pre_task-nback_bold.nii.gz
ses-post/
...
```
**Key points:**
- Every NIfTI file should have a corresponding `.json` sidecar
- File names encode entities: `sub-<label>[_ses-<label>][_task-<label>][_acq-<label>][_run-<index>]_<suffix>.<extension>`
- Entity order in filenames is fixed by the specification
- Only `dataset_description.json` is strictly required at the root level
### 2. Creating dataset_description.json
```python
import json
dataset_description = {
"Name": "My Neuroimaging Study",
"BIDSVersion": "1.10.0",
"DatasetType": "raw",
"License": "CC0",
"Authors": ["First Author", "Second Author"],
"Acknowledgements": "Funded by NIH R01-MH123456",
"HowToAcknowledge": "Please cite: Author et al. (2025) Journal Name.",
"Funding": ["NIH R01-MH123456", "NSF BCS-7654321"],
"ReferencesAndLinks": ["https://doi.org/10.xxxx/xxxxx"],
"DatasetDOI": "10.18112/openneuro.ds000001.v1.0.0",
"GeneratedBy": [
{
"Name": "HeuDiConv",
"Version": "1.3.1",
"CodeURL": "https://github.com/nipy/heudiconv"
}
]
}
with open("dataset_description.json", "w") as f:
json.dump(dataset_description, f, indent=4)
```
For **derivatives**, set `"DatasetType": "derivative"` and add `"GeneratedBy"` listing the pipeline:
```python
deriv_description = {
"Name": "fMRIPrep - fMRI PREProcessing",
"BIDSVersion": "1.10.0",
"DatasetType": "derivative",
"GeneratedBy": [
{
"Name": "fMRIPrep",
"Version": "24.1.0",
"CodeURL": "https://github.com/nipreps/fmriprep"
}
]
}
```
### 3. Querying BIDS Datasets with PyBIDS
```python
from bids import BIDSLayout
# Index a BIDS dataset (validates structure on load)
layout = BIDSLayout("/path/to/bids_dataset")
# Basic queries
subjects = layout.get_subjects() # ['01', '02', '03', ...]
sessions = layout.get_sessions() # ['pre', 'post'] or []
tasks = layout.get_tasks() # ['rest', 'nback']
runs = layout.get_runs() # [1, 2] or []
# Find specific files
bold_files = layout.get(
suffix="bold",
extension=".nii.gz",
return_type="filename"
)
# Filter by subject, task, session
nback_sub01 = layout.get(
subject="01",
task="nback",
suffix="bold",
extension=".nii.gz",
return_type="filename"
)
# Get metadata from JSON sidecars (automatic inheritance)
metadata = layout.get_metadata("/path/to/sub-01/func/sub-01_task-rest_bold.nii.gz")
tr = metadata["RepetitionTime"]
# Get all entities for a file
entities = layout.get_entities()
# Build a path from entities using BIDSLayout
bids_file = layout.get(subject="01", suffix="T1w", extension=".nii.gz")[0]
print(bids_file.path)
print(bids_file.get_entities())
```
**Key points:**
- `BIDSLayout` indexes the entire dataset on initialization; for large datasets use `database_path` to cache the index
- Metadata inheritance: a JSON sidecar at a higher level (e.g., root or subject) is inherited by all files below unless overridden
- Use `return_type="filename"` for paths, `return_type="object"` (default) for `BIDSFile` objects
### 4. Validating BIDS Datasets
#### Using bids-validator via PyPI (recommended)
The `bids-validator-deno` PyPI package bundles the Deno-based validator as a standalone CLI:
```bash
# Install
uv pip install bids-validator-deno
# Validate a dataset
bids-validator /path/to/bids_dataset
# Ignore specific warnings/errors
bids-validator /path/to/bids_dataset --ignoreNiftiHeaders --ignoreSubjectConsistency
```
#### Using bids-validator via Deno directly
If Deno is already available, you can install or run the validator without PyPI:
```bash
# Install globally via Deno
deno install -g -A npm:bids-validator
# Or run without installing
deno run -A npm:bids-validator /path/to/bids_dataset
```
#### Legacy Node.js validator
The older Node.js-based validator (`npm install -g bids-validator`) is deprecated in favor of the Deno-based version. The Deno version is the reference implementation for BIDS Specification v1.9+.
#### Using .bidsignore
Create `.bidsignore` at the dataset root to exclude files from validation (gitignore syntax):
```
# Exclude sourcedata and extra files
sourcedata/
extra_data/
*.log
*_sbref.nii.gz
**/.DS_Store
```
### 5. BIDS Entities and File Naming
The authoritative, machine-readable source of truth for entities, their ordering, allowed suffixes, and all filename rules is the **BIDS Schema** — a structured YAML/JSON representation of the specification. A JSON export is shipped with this skill at `references/bids_schema.json`. The schema is defined in the [bids-specification `src/schema/`](https://github.com/bids-standard/bids-specification/tree/master/src/schema) directory and published at https://bids-specification.readthedocs.io/en/stable/schema.json. BEP-specific schema previews are available at https://github.com/bids-standard/bids-schema/tree/main/BEPs.
Run `scripts/update_schema.py` to refresh the schema and BEPs list from upstream (no dependencies beyond stdlib).
The tables below are a convenient summary; when in doubt, consult the schema.
BIDS filenames are built from ordered key-value entity pairs:
| Entity | Key | Example | Required for |
|--------|-----|---------|--------------|
| Subject | `sub-` | `sub-01` | All files |
| Session | `ses-` | `ses-pre` | Multi-session studies |
| Task | `task-` | `task-rest` | func (bold, cbv, phase), eeg, meg |
| Acquisition | `acq-` | `acq-highres` | Distinguishing acquisition parameters |
| Contrast enhancing agent | `ce-` | `ce-gadolinium` | Contrast-enhanced images |
| Reconstruction | `rec-` | `rec-magnitude` | Reconstruction variants |
| Direction | `dir-` | `dir-AP` | Fieldmaps, DWI, phase-encoding |
| Run | `run-` | `run-01` | Multiple identical acquisitions |
| Echo | `echo-` | `echo-1` | Multi-echo sequences |
| Part | `part-` | `part-mag` | Magnitude/phase splits |
| Space | `space-` | `space-MNI152NLin2009cAsym` | Derivatives in template space |
| Description | `desc-` | `desc-preproc` | Derivatives only |
**Entity ordering in filenames** is fixed by the spec (defined in `rules.entities` in `bids_schema.json`). See `references/bids_specification.md` for the complete numbered ordering table. A common subset:
`sub-<label>[_ses-<label>][_task-<label>][_acq-<label>][_ce-<label>][_rec-<label>][_dir-<label>][_run-<index>][_echo-<index>][_part-<label>][_space-<label>][_desc-<label>]_<suffix>.<extension>`
**Common suffixes by datatype:**
| Datatype | Suffixes |
|----------|----------|
| anat | `T1w`, `T2w`, `FLAIR`, `T2star`, `T1map`, `T2map`, `defacemask` |
| func | `bold`, `cbv`, `sbref`, `events`, `physio`, `stim` |
| dwi | `dwi`, `sbref` |
| fmap | `phasediff`, `phase1`, `phase2`, `magnitude1`, `magnitude2`, `fieldmap`, `epi` |
| perf | `asl`, `m0scan`, `aslcontext` |
| eeg | `eeg`, `channels`, `electrodes`, `events` |
| meg | `meg`, `channels`, `coordsystem`, `events` |
| ieeg | `ieeg`, `channels`, `electrodes`, `coordsystem`, `events` |
| pet | `pet`, `blood` |
### 6. DICOM to BIDS Conversion
#### HeuDiConv
HeuDiConv is the most flexible DICOM-to-BIDS converter. It supports three usage modes — from fully automatic to fully custom — and handles duplicates, provenance tracking, and sourcedata archiving out of the box.
**Mode 1: ReproIn (turnkey, recommended for new studies)**
If scanner protocol names follow the [ReproIn naming convention](https://github.com/repronim/reproin), conversion is fully automatic — no heuristic file to write:
```bash
# Turnkey conversion: HeuDiConv maps ReproIn protocol names to BIDS automatically
heudiconv --files dicom/001 -o /path/to/bids -f reproin --bids --minmeta
```
ReproIn protocol names encode BIDS entities directly:
- `anat-T1w` → `sub-XX/anat/sub-XX_T1w.nii.gz`
- `func-bold_task-rest` → `sub-XX/func/sub-XX_task-rest_bold.nii.gz`
- `dwi_dir-AP` → `sub-XX/dwi/sub-XX_dir-AP_dwi.nii.gz`
- `fmap_dir-PA` → `sub-XX/fmap/sub-XX_dir-PA_epi.nii.gz`
Session can be set once on the localizer (e.g., `anat-scout_ses-pre`) and ReproIn propagates it to all sequences in that Program. Subject ID is extracted from DICOM metadata. Duplicate runs are numbered automatically.
**Mode 2: Custom heuristic mapping into ReproIn (for existing data)**
If you already have data with non-ReproIn protocol names, you can write a thin heuristic that maps your names into ReproIn conventions, gaining all ReproIn benefits (automatic entity handling, duplicate management, etc.). See https://github.com/repronim/reproin/issues/18 for a HOWTO.
**Mode 3: Custom heuristic (full flexibility)**
For complex mappings, write a Python heuristic file:
```bash
# Step 1: Reconnaissance — discover DICOM series
heudiconv --files dicom/219/itbs/*/*.dcm -o Nifti/ -f convertall -s 219 -c none
# This creates .heudiconv/219/info/dicominfo.tsv — inspect it to understand
# what was acquired and map series to BIDS names.
# Step 2: Write a heuristic file (see references/conversion_tools.md)
# Step 3: Convert
heudiconv --files dicom/219/itbs/*/*.dcm -s 219 -ss itbs \
-f Nifti/code/heuristic.py -c dcm2niix --bids --minmeta -o Nifti/
```
See `references/conversion_tools.md` for complete heuristic file examples.
**Key points:**
- HeuDiConv wraps `dcm2niix` for the actual DICOM-to-NIfTI conversion
- **`--minmeta`**: always use this flag to prevent excess DICOM metadata from overflowing JSON sidecars (can crash fMRIPrep/MRIQC)
- **Duplicate handling**: use `{item:03d}` in templates for auto-numbering when the same protocol is run multiple times; without it, later runs overwrite earlier ones
- **`.heudiconv/` directory**: created alongside output, stores provenance (heuristic used, dicominfo.tsv, conversion records). Keep it with your data for reproducibility
- **`sourcedata/`**: HeuDiConv archives original DICOMs as `.tgz` files under `sourcedata/` for reproducibility
- **`is_motion_corrected` filter**: use in heuristics to exclude scanner-generated MOCO series (e.g., `if not s.is_motion_corrected`)
- Both `--files` (explicit paths) and `-d` (template with `{subject}`, `{session}` placeholders) are supported for specifying DICOM input
#### dcm2bids (Configuration-file-based)
```bash
# Step 1: Generate helper output to inspect series
dcm2bids_helper -d /path/to/dicom
# Step 2: Create config file (dcm2bids_config.json)
# Step 3: Convert
dcm2bids -d /path/to/dicom -p 01 -c dcm2bids_config.json -o /path/to/bids_output
```
See `references/conversion_tools.md` for detailed configuration examples.
### 7. Metadata Sidecars
Every BIDS data file should have a JSON sidecar with acquisition parameters. Metadata fields follow the inheritance principle: a sidecar at a higher directory level applies to all matching files below.
**Inheritance example:**
```
my_dataset/
task-rest_bold.json # Applies to ALL rest BOLD files
sub-01/
func/
sub-01_task-rest_bold.json # Overrides/extends for sub-01 only
```
**Critical metadata fields by modality:**
For **func (BOLD)**:
```json
{
"RepetitionTime": 2.0,
"TaskName": "rest",
"PhaseEncodingDirection": "j-",
"TotalReadoutTime": 0.05,
"SliceTiming": [0, 0.5, 1.0, 1.5],
"EffectiveEchoSpacing": 0.00058,
"EchoTime": 0.03
}
```
For **anat**:
```json
{
"MagneticFieldStrength": 3,
"Manufacturer": "Siemens",
"ManufacturersModelName": "Prisma",
"RepetitionTime": 2.3,
"EchoTime": 0.00293,
"FlipAngle": 8
}
```
For **DWI**:
```json
{
"PhaseEncodingDirection": "j-",
"TotalReadoutTime": 0.05,
"EchoTime": 0.089,
"RepetitionTime": 3.4,
"MultipartID": "dwi_1"
}
```
**Key points:**
- `dcm2niix` auto-generates most sidecar fields from DICOM headers
- `RepetitionTime` and `TaskName` are required for BOLD
- `SliceTiming` is essential for slice-timing correction in fMRI preprocessing
- `PhaseEncodingDirection` and `TotalReadoutTime` (or `EffectiveEchoSpacing`) are needed for distortion correction
- See `references/metadata_fields.md` for comprehensive field reference
### 8. Events Files for Task fMRI
Task-based fMRI requires `_events.tsv` files:
```
onset duration trial_type response_time
0.0 0.5 face 0.435
2.5 0.5 house 0.367
5.0 0.5 face 0.512
7.5 0.5 scrambled 0.298
```
**Required columns:**
- `onset` - onset time in seconds relative to the start of the acquisition
- `duration` - duration in seconds (use `n/a` for instantaneous events)
**Recommended columns:**
- `trial_type` - categorical label for condition
- `response_time` - RT in seconds
- Custom columns as needed (with descriptions in corresponding `.json` sidecar)
### 9. Participants File
```
participant_id age sex group handedness
sub-01 25 M control right
sub-02 30 F patient left
sub-03 28 M control right
```
The `participants.json` sidecar describes columns:
```json
{
"age": {
"Description": "Age of the participant at time of scanning",
"Units": "years"
},
"sex": {
"Description": "Biological sex",
"Levels": {
"M": "male",
"F": "female"
}
},
"group": {
"Description": "Experimental group",
"Levels": {
"control": "Healthy control",
"patient": "Patient group"
}
},
"handedness": {
"Description": "Dominant hand",
"Levels": {
"right": "Right-handed",
"left": "Left-handed",
"ambidextrous": "Ambidextrous"
}
}
}
```
### 10. BIDS Derivatives
Processed outputs go under a `derivatives/` directory:
```
my_dataset/
derivatives/
fmriprep-24.1.0/
dataset_description.json # DatasetType: "derivative"
sub-01/
anat/
sub-01_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz
sub-01_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz
func/
sub-01_task-rest_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
sub-01_task-rest_desc-confounds_timeseries.tsv
mriqc-24.0.0/
dataset_description.json
sub-01/
anat/
sub-01_T1w.html
func/
sub-01_task-rest_bold.html
group_T1w.tsv
group_bold.tsv
```
**Derivative conventions:**
- `space-<label>` - template/reference space (e.g., `MNI152NLin2009cAsym`, `T1w`)
- `desc-<label>` - description of processing (e.g., `preproc`, `brain`, `smoothed`)
- `res-<label>` - resolution (e.g., `2` for 2mm isotropic)
- Each pipeline gets its own directory under `derivatives/`
- Must have its own `dataset_description.json` with `GeneratedBy`
### 11. PyBIDS: Advanced Usage
```python
from bids import BIDSLayout
from bids.layout import BIDSLayoutIndexer
# Cache the layout index for faster repeated access
layout = BIDSLayout("/path/to/dataset", database_path="/path/to/cache.db")
# Include derivatives
layout = BIDSLayout(
"/path/to/dataset",
derivatives=["/path/to/dataset/derivatives/fmriprep-24.1.0"]
)
# Get derivative files
preproc = layout.get(
subject="01",
task="rest",
desc="preproc",
suffix="bold",
space="MNI152NLin2009cAsym",
extension=".nii.gz",
return_type="filename"
)
# Get confound regressors
confounds = layout.get(
subject="01",
task="rest",
desc="confounds",
suffix="timeseries",
extension=".tsv",
return_type="filename"
)
# Build BIDS path from entities
from bids import BIDSLayout
layout = BIDSLayout("/path/to/dataset")
path = layout.build_path(
{
"subject": "01",
"session": "pre",
"task": "rest",
"suffix": "bold",
"extension": ".nii.gz",
"datatype": "func"
},
validate=True
)
# Get all files for a subject as a DataFrame
import pandas as pd
files_df = layout.to_df()
sub01_df = files_df[files_df["subject"] == "01"]
```
### 12. BIDS-Apps
BIDS-Apps are containerized analysis pipelines that accept BIDS datasets as input:
```bash
# General BIDS-App invocation pattern
docker run -v /path/to/bids:/data:ro -v /path/to/output:/out \
<bids-app-image> /data /out participant --participant_label 01
# Common BIDS-Apps:
# fMRIPrep - fMRI preprocessing
docker run nipreps/fmriprep /data /out participant \
--participant-label 01 --fs-license-file /license.txt
# MRIQC - MRI quality control
docker run nipreps/mriqc /data /out participant \
--participant-label 01
# QSIPrep - diffusion MRI preprocessing
docker run pennbbl/qsiprep /data /out participant \
--participant-label 01
```
**BIDS-App interface convention:**
```
bids-app input_dataset output_dir {participant|group} [options]
```
- `participant` level: runs per-subject
- `group` level: runs across all subjects (aggregation/group stats)
## Reference Materials
This skill includes detailed reference documentation:
- **bids_schema.json**: Machine-readable BIDS schema (from https://bids-specification.readthedocs.io/en/stable/schema.json). This is the authoritative source for entity definitions, ordering rules, filename templates, allowed suffixes per datatype, and metadata field requirements. BEP-specific schemas are at https://github.com/bids-standard/bids-schema/tree/main/BEPs.
- **beps.yml**: Current list of all BIDS Extension Proposals with titles, leads, status, and links (from [bids-website](https://github.com/bids-standard/bids-website/blob/main/data/beps/beps.yml))
- **bids_specification.md**: Human-readable summary of the entity table, datatype reference, directory structure rules, template spaces, and specification changelog
- **metadata_fields.md**: Required and recommended JSON sidecar fields for every BIDS modality (anat, func, dwi, fmap, eeg, meg, pet, etc.)
- **conversion_tools.md**: Detailed workflows for HeuDiConv, dcm2bids, and BIDScoin including heuristic/config examples and troubleshooting
Update schema and BEPs with: `python scripts/update_schema.py`
## Common Issues and Solutions
### 1. Validator reports "Not a BIDS dataset"
**Cause**: Missing `dataset_description.json` at the root.
**Fix**: Create the file with at minimum `{"Name": "...", "BIDSVersion": "1.10.0"}`.
### 2. Inconsistent subjects warning
**Cause**: Not all subjects have the same set of files (some missing sessions, runs, etc.).
**Fix**: This is a warning, not an error. Use `--ignoreSubjectConsistency` if intentional. Document missing data in `participants.tsv` or a `scans.tsv`.
### 3. Missing SliceTiming
**Cause**: `dcm2niix` couldn't extract slice timing from DICOM headers.
**Fix**: Determine slice order from the scan protocol and add manually to the JSON sidecar. Common patterns: ascending, descending, interleaved (odd-first or even-first).
### 4. Phase encoding direction confusion
**Cause**: Axis labels (i/j/k vs x/y/z vs LR/AP/SI) are confusing.
**Fix**: In BIDS, use NIfTI image axes: `i`=first axis, `j`=second, `k`=third. `-` means negative direction. For standard axial acquisitions: `j` is typically anterior-posterior. Verify with the acquisition protocol.
### 5. PyBIDS is slow on large datasets
**Cause**: Full filesystem indexing on every `BIDSLayout()` call.
**Fix**: Use `database_path` to cache the index to an SQLite file:
```python
layout = BIDSLayout("/data", database_path="/data/.pybids_cache.db")
```
### 6. Derivatives not found by PyBIDS
**Cause**: Derivatives directory missing its own `dataset_description.json`.
**Fix**: Every derivatives directory must have `dataset_description.json` with `"DatasetType": "derivative"`.
### 7. Events file timing is off
**Cause**: `onset` times are relative to the wrong reference (e.g., trigger time vs first volume).
**Fix**: Onsets must be in seconds relative to the first volume of that run's acquisition. Account for dummy scans if they were discarded.
### 8. TSV files fail validation
**Cause**: Encoding or delimiter issues (spaces instead of tabs, BOM characters, Windows line endings).
**Fix**: Ensure tab-separated values with UTF-8 encoding and Unix line endings (`\n`). Use `n/a` (not `NA`, `NaN`, or empty) for missing values.
## Best Practices
1. **Validate early and often** - Run the BIDS validator after every conversion or modification. Fix errors before they compound.
2. **Use metadata inheritance** - Place shared metadata (e.g., `TaskName`, scanner parameters) in top-level sidecar files rather than duplicating in every subject's directory.
3. **Keep sourcedata** - Store the original DICOM (or other raw) data under `sourcedata/` so conversions are reproducible. Add `sourcedata/` to `.bidsignore`.
4. **Use consistent naming from the start** - Define your BIDS naming scheme before data collection. Use the ReproIn naming convention for scan protocols to enable automatic conversion.
5. **Document your dataset** - Write a thorough `README` describing the study design, acquisition parameters, known issues, and any deviations from BIDS.
6. **Use scans.tsv for run-level metadata** - Record per-run acquisition times and quality notes:
```
filename acq_time quality
func/sub-01_task-rest_bold.nii.gz 2025-01-15T10:30:00 good
```
7. **Version your dataset** - Use `CHANGES` to document dataset modifications. Consider DataLad for full version control of large datasets.
8. **Deface anatomical images** - Remove facial features from T1w/T2w images before sharing (e.g., using `pydeface`, `mri_deface`, or `afni_refacer`). Store defaced versions as the primary data or use `_defacemask` files.
9. **Use BIDS URIs for provenance** - In derivatives, reference source files using BIDS URIs: `bids::sub-01/anat/sub-01_T1w.nii.gz`.
10. **Prefer community tools** - Use established BIDS-Apps (fMRIPrep, MRIQC, QSIPrep) rather than custom pipelines when possible. They handle BIDS I/O correctly and produce BIDS-compliant derivatives.
11. **Study bids-examples** - The [bids-examples](https://github.com/bids-standard/bids-examples) repository is the canonical collection of prototypical BIDS datasets covering different modalities and use cases (MRI, fMRI, DWI, EEG, MEG, iEEG, PET, ASL, genetics, derivatives, and more). Use it as a reference when structuring your own dataset, as test data for BIDS tools, or to understand how a specific modality should be organized. Each example passes the BIDS validator.
## BIDS Extension Proposals (BEPs)
BEPs are community-driven proposals to extend BIDS to new modalities, derivatives, or metadata. The full list with status, leads, and links is in `references/beps.yml` (fetched from the [bids-website](https://github.com/bids-standard/bids-website/blob/main/data/beps/beps.yml)). BEP-specific schema previews are rendered at https://github.com/bids-standard/bids-schema/tree/main/BEPs.
**Current BEPs** (as of schema update):
| BEP | Title | Content | Status |
|-----|-------|---------|--------|
| 004 | Susceptibility Weighted Imaging | raw | Seeking new leader |
| 011 | Structural preprocessing derivatives | derivative | Has PR (#518) |
| 012 | Functional preprocessing derivatives | derivative | Has PR (#519), schema implemented |
| 014 | Affine transforms and nonlinear field warps | derivative | X5 format development |
| 016 | Diffusion weighted imaging derivatives | derivative | Has PR (#2211) |
| 017 | Generic BIDS connectivity data schema | derivative | In development |
| 021 | Common Electrophysiological Derivatives | derivative | In development |
| 023 | PET Preprocessing derivatives | derivative | In development |
| 024 | Computed Tomography scan | raw | Seeking contributors |
| 026 | Microelectrode Recordings | raw | Seeking new leader |
| 028 | Provenance | metadata | Has PR (#2099) |
| 032 | Microelectrode electrophysiology | raw | Has PR (#2307), preview available — covers Neuropixels and other extracellular probes; relates to neuropixels-analysis skill |
| 033 | Advanced Diffusion Weighted Imaging | raw | Seeking contributors |
| 034 | Computational modeling | derivative | Has PR (#967) |
| 035 | Mega-analyses with non-compliant derivatives | derivative | In development |
| 036 | Phenotypic Data Guidelines | raw | Community review |
| 037 | Non-Invasive Brain Stimulation | raw | In development |
| 039 | Dimensionality reduction-based networks | raw | In development |
| 040 | Functional Ultrasound | raw | In development |
| 041 | Statistical Model Derivatives | derivative | Collecting feedback |
| 043 | BIDS Term Mapping | metadata | Collecting feedback |
| 044 | Stimuli | raw | Has PR (#2022), community review |
| 045 | Peripheral Physiological Recordings | raw | Has PR (#2267) |
| 046 | Diffusion Tractography | derivative | In development |
| 047 | Audio/video recordings for behavioral experiments | raw | Has PR (#2231) |
**Related standards:**
- **BIDS-Stats Models**: JSON specification for defining GLM-based neuroimaging analyses
- **BIDS-Derivatives** (BEP003): Standard for preprocessed/analysis outputs (partially merged into spec)
## Related Tools Ecosystem
| Tool | Purpose |
|------|---------|
| **fMRIPrep** | fMRI preprocessing (produces BIDS derivatives) |
| **MRIQC** | MRI quality control (produces BIDS derivatives) |
| **QSIPrep** | Diffusion MRI preprocessing |
| **TemplateFlow** | Neuroimaging templates and atlases with BIDS-like naming |
| **Fitlins** | BIDS Stats Models implementation |
| **DataLad** | Version control for large datasets, integrates with BIDS |
| **OpenNeuro** | Free BIDS dataset repository |
| **DANDI** | Neurophysiology data archive (uses BIDS for some modalities) |
| **HeuDiConv** | DICOM-to-BIDS with heuristic Python files |
| **dcm2bids** | DICOM-to-BIDS with JSON config |
| **BIDScoin** | DICOM-to-BIDS with GUI and YAML config |
| **nwb2bids** | Convert NWB (Neurodata Without Borders) files to BIDS |
| **CuBIDS** | BIDS dataset curation and harmonization |
| **bids2table** | Efficient tabular indexing of BIDS datasets |
| **bids-examples** | Canonical collection of prototypical BIDS datasets for all modalities |
## Documentation
- **BIDS Specification**: https://bids-specification.readthedocs.io/
- **BIDS Website**: https://bids.neuroimaging.io/
- **PyBIDS Documentation**: https://bids-standard.github.io/pybids/
- **BIDS Validator**: https://github.com/bids-standard/bids-validator
- **BIDS Starter Kit**: https://bids-standard.github.io/bids-starter-kit/
- **BIDS Examples**: https://github.com/bids-standard/bids-examples — canonical reference datasets for every BIDS modality; use as templates and test data
- **HeuDiConv Docs**: https://heudiconv.readthedocs.io/
- **Original BIDS paper**: Gorgolewski et al. (2016) Scientific Data, doi:10.1038/sdata.2016.44
skills/bids/references/beps.yml
+637
View File
@@ -0,0 +1,637 @@
---
# template
# - number:
# title:
# display:
# google_doc:
# pull_request:
# html_preview:
# leads:
# # MUST match given-names and family-names from the bids specification citation.cff
# - given-names: ' '
# family-names: ' '
# bids_maintainers:
# - given-names: ' '
# family-names: ' '
# status:
# blocking:
# communication_channel:
# pull_request_created:
# pull_request_merged:
- number: '004'
title: Susceptibility Weighted Imaging
google_doc: https://docs.google.com/document/d/1kyw9mGgacNqeMbp4xZet3RnDhcMmf4_BmRgKaOkO2Sc/
content:
- raw
leads:
- given-names: ' '
family-names: ' '
bids_maintainers:
status:
blocking:
- Looking for a new leader.
google_doc_created: 2017-04
pull_request_created:
pull_request_merged:
- number: '011'
title: Structural preprocessing derivatives
google_doc: https://docs.google.com/document/d/1YG2g4UkEio4t_STIBOqYOwneLEs1emHIXbGKynx7V0Y/
pull_request: https://github.com/bids-standard/bids-specification/pull/518
html_preview: https://bids-specification--518.org.readthedocs.build/en/518/05-derivatives/04-structural-derivatives.html
content:
- derivative
leads:
- given-names: Viviana
family-names: Siless
bids_maintainers:
- given-names: Christopher J.
family-names: Markiewicz
status:
- Stability! (We haven't touched it in a bit.)
blocking:
- Staleness! (We haven't touched it in a bit.)
- Mostly just need to regroup with other BEPs and make sure we're being consistent.
google_doc_created: 2017-08
pull_request_created: 2020-06
pull_request_merged:
- number: '012'
title: Functional preprocessing derivatives
google_doc:
pull_request: https://github.com/bids-standard/bids-specification/pull/519
html_preview: https://bids-specification--519.org.readthedocs.build/en/519/derivatives/functional-derivatives.html
content:
- derivative
leads:
- given-names: Christopher J.
family-names: Markiewicz
bids_maintainers:
- given-names: Christopher J.
family-names: Markiewicz
status:
- Moderate reworking, post-BEP23 meeting. "boldmap" suffix.
- Schema implemented; waiting on full schema validation to merge.
blocking:
- Overlap with BEP 39 (decompositions).
- Probably just needs dropping from BEP 12, but need to make sure BEP 39 meets the needs served there.
google_doc_created: 2018-10
pull_request_created: 2020-06
pull_request_merged:
- number: '014'
title: Affine transformations and nonlinear field warps
google_doc: https://docs.google.com/document/d/11gCzXOPUbYyuQx8fErtMO9tnOKC3kTWiL9axWkkILNE/
content:
- derivative
leads:
- given-names: Oscar
family-names: Esteban
bids_maintainers:
- given-names: Christopher J.
family-names: Markiewicz
status:
- After the kick-off meeting (2019), progress locked on developing a prototype of a new HDF5-based format (X5)
- Minor bug fixes and features to support more transforms
- 'Perspectives: The current draft seems sufficient for storing transforms'
- 'Perspectives: X5 has high promises to enable effortless spatial transforms'
blocking:
- Bandwidth to finalize development
- 'Current blockers: surface transforms & X5 storing'
google_doc_created: 2017-08
pull_request_created:
pull_request_merged:
- number: '016'
title: Diffusion weighted imaging derivatives
pull_request: https://github.com/bids-standard/bids-specification/pull/2211
content:
- derivative
leads:
- given-names: Franco
family-names: Pestilli
- given-names: Oscar
family-names: Esteban
bids_maintainers:
status:
- adapted general spatial derivatives proposal, meaning using <modality>map, like `dwimap`
- decided on using `params-` to denote different file types, for example `param-md` or `param-fa`
- updated meta-data
blocking:
google_doc_created:
pull_request_created:
pull_request_merged:
- number: '017'
title: Generic BIDS connectivity data schema
display: Connectivity schema
google_doc: https://docs.google.com/document/d/1ugBdUF6dhElXdj3u9vw0iWjE6f_Bibsro3ah7sRV0GA/
content:
- derivative
leads:
- given-names: Eugene P.
family-names: Duff
bids_maintainers:
status:
- specified different formats for dense and sparse matrices
- 'proposed arrays in h5 or zarr to cover multi-dimensional matrices (for example: dynamic connectivity)'
blocking:
google_doc_created: 2017-05
pull_request_created:
pull_request_merged:
- number: '021'
title: Common Electrophysiological Derivatives
google_doc: https://docs.google.com/document/d/1PmcVs7vg7Th-cGC-UrX8rAhKUHIzOI-uIOh69_mvdlw/
content:
- derivative
leads:
- given-names: Arnaud
family-names: Delorme
- given-names: Dora
family-names: Hermes
- given-names: Mainak
family-names: Jas
- given-names: Guiomar
family-names: Niso
- given-names: Robert
family-names: Oostenveld
- given-names: Cyril
family-names: Pernet
- given-names: Christine
family-names: Rogers
status:
- 'Focus on raw-ish data: channels by time: epoching, filtering, interpolation...'
- 'All provenance tracking: outsourced to BEP028 (“provenance”)'
- Reusing existing (raw) data formats where applicable
- 'No additional entities or suffixes: focus on “desc” entity'
- Create new “descriptions.tsv” file to accompany and document the “desc” entity
- Working on examples on GitHub
- 'Perspectives: Derivatives beyond channels by time data will be discussed at a later point'
blocking:
google_doc_created: 2018-05
pull_request_created:
pull_request_merged:
- number: '023'
title: PET Preprocessing derivatives
google_doc: https://docs.google.com/document/d/1yzsd1J9GT-aA0DWhdlgNr5LCu6_gvbjLyfvYq2FuxlY/
content:
- derivative
leads:
- given-names: Martin
family-names: Noergaard
- given-names: Graham
family-names: Searle
- given-names: Melanie
family-names: Ganz-Benjaminsen
bids_maintainers:
- given-names: Anthony
family-names: Galassi
status:
- Defined all the necessary components to be included in the specification
- Alignment with other modalities obtained at Copenhagen BIDS derivatives meeting
- Example PET derivatives datasets available on github
- 'Perspectives: 3rd joint meeting between all PET collaborators in August 2023'
- 'Perspectives: Aim is to finish the specification during the fall of 2023'
blocking:
- Need more example datasets (fore example different tracers) with different preprocessing choices to capture as most of the PET community as possible
- Need to finish alignment with other modalities
- Still need to agree on the level of information going into corresponding json files
google_doc_created: 2018-08
pull_request_created:
pull_request_merged:
- number: '024'
title: Computed Tomography scan
google_doc: https://docs.google.com/document/d/1fqnJZ18x5LJC8jiJ8yvPHUGFzNBZ6gW2kywYrUKWtuo/
content:
- raw
leads:
- given-names: Hugo
family-names: Boniface
bids_maintainers:
status:
- Lead seeking more contributors and experts.
blocking:
google_doc_created: 2018-11
pull_request_created:
pull_request_merged:
- number: '026'
title: Microelectrode Recordings
google_doc: https://docs.google.com/document/d/14KC1d5-Lx-7ZSMtwS7pVAAvz-2WR_uoo5FvsNirzqJw/
content:
- raw
leads:
- given-names: ' '
family-names: ' '
bids_maintainers:
status:
- BEP is open to new leadership, see also [BEP032 (animal electrophys)](https://docs.google.com/document/d/1oG-C8T-dWPqfVzL2W8HO3elWK8NIh2cOCPssRGv23n0/)
blocking:
- Searching for a new leader.
google_doc_created: 2018-04
pull_request_created:
pull_request_merged:
- number: '028'
title: Provenance
google_doc: https://docs.google.com/document/d/1vw3VNDof5cecv2PkFp7Lw_pNUTUo8-m8V4SIdtGJVKs/
content:
- metadata
leads:
- given-names: Satrajit S.
family-names: Ghosh
- given-names: Camille
family-names: Maumet
- given-names: Yaroslav O.
family-names: Halchenko
bids_maintainers:
status:
- '[Specification](https://bids.neuroimaging.io/bep028)'
- '[Set of examples](https://github.com/bids-standard/BEP028_BIDSprov)'
- 'Perspectives: Opening up to BIDS community for feedback'
- 'Perspectives: Engaging with software developers (in progress w/ SPM, AFNI)'
blocking:
google_doc_created: 2018-08
pull_request: https://github.com/bids-standard/bids-specification/pull/2099
pull_request_created: 2025-04
pull_request_merged:
- number: '032'
title: Microelectrode electrophysiology
google_doc: https://docs.google.com/document/d/1oG-C8T-dWPqfVzL2W8HO3elWK8NIh2cOCPssRGv23n0/
pull_request: https://github.com/bids-standard/bids-specification/pull/2307
html_preview: https://bids-specification--2307.org.readthedocs.build/en/2307/modality-specific-files/microelectrode-electrophysiology.html
content:
- raw
leads:
- given-names: Sylvain
family-names: Takerkart
- given-names: Benjamin
family-names: Dichter
- given-names: Yaroslav O.
family-names: Halchenko
- given-names: Lyuba
family-names: Zehl
- given-names: Andrew
family-names: Davison
bids_maintainers:
- given-names: Rémi
family-names: Gau
status:
- Decided on the new name (not just "Animal" but "Microelectrode"), modalities, datatypes
- Nearly finalized added metadata
- PR compiles green, preview is available
- 'Target: finalize & merge PR into the BIDS specs in 2025'
blocking:
- Need to prepare example datasets
- Need to start thinking about derived data (spike sorted)
google_doc_created: 2020-12
pull_request_created: 2022-11
pull_request_merged:
- number: '033'
title: Advanced Diffusion Weighted Imaging
google_doc: https://docs.google.com/document/d/1en4ByORlPqwDfZwNOOBTj0GwpYBcS0_2peqreTOvwDU/
content:
- raw
leads:
- given-names: James
family-names: Gholam
- given-names: Leandro
family-names: Beltrachini
- given-names: Filip
family-names: Szczepankiewicz
bids_maintainers:
status:
- 'New BEP, seeking contributors and collecting community feedback relating to: priority sequences to support, best supported binary structured formats
(e.g. CBOR? HDF5? MsgPack?).'
- Comments may be submitted directly on the document.
- Generating example datasets [here](https://github.com/JAgho/MISP_plot/tree/main) and determining best practice with vendors to record data in-sequence
blocking:
google_doc_created: 2021-04
pull_request_created:
pull_request_merged:
- number: '034'
title: Computational modeling
pull_request: https://github.com/bids-standard/bids-specification/pull/967
html_preview: https://bids-specification--967.org.readthedocs.build/en/967/modality-specific-files/computational-models.html
content:
- derivative
- metadata
leads:
- given-names: Michael
family-names: Schirner
- given-names: Petra
family-names: Ritter
bids_maintainers:
status:
- sim2bids app created
- easier to bring neural simulation data (especially in The Virtual Brain format) into the proposed BIDS Comp Model format
- 'Perspectives: A roadmap that coordinates efforts across BEPs would be appreciated.'
blocking:
- BEPs have overlapping concerns (Comp Models, Spaces and mappings, Generic BIDS connectivity schema, time series, Provenance, Atlases)
- need to separate concerns & provide a roadmap for each BEP
google_doc_created: 2021-02
pull_request_created: 2021-08
pull_request_merged:
- number: '035'
title: Modular extensions for individual participant data mega-analyses with non-compliant derivatives
display: Mega-analyses
google_doc: https://docs.google.com/document/d/1tFRNumQyIgjXBNC3brFDLO9FaikjL84noxK6Om-Ctik/
content:
- derivative
leads:
- given-names: Giuseppe
family-names: Gallitto
- given-names: Balint
family-names: Kincses
- given-names: Tamas
family-names: Spisak
bids_maintainers:
- given-names: Rémi
family-names: Gau
status:
- Meta-data harmonization with a term-mapper goes to BEP043
- 'Persperctive: Repeated community review of the simplified proposal'
blocking:
- Term-mapping is a general feature => complex, modular proposal
google_doc_created: 2021-12
pull_request_created:
pull_request_merged:
- number: '036'
title: Phenotypic Data Guidelines
google_doc: https://docs.google.com/document/d/1WTkfES8L0vItZVyyR68fc-9cO03jS-kCnMnw6602pbc/
content:
- raw
leads:
- given-names: Eric
family-names: Earl
- given-names: Samuel
family-names: Guay
- given-names: Sebastian
family-names: Urchs
- given-names: Arshitha
family-names: Basavaraj
bids_maintainers:
- given-names: Chris
family-names: Markiewicz
- given-names: Ross
family-names: Blair
status:
- BEP entering community review period shortly.
- BIDS specification PR 2123.
- BIDS examples PR 465.
blocking:
- A successful community review.
google_doc_created: 2021-10
pull_request_created: 2025-05
pull_request_merged:
- number: '037'
title: Non-Invasive Brain Stimulation
google_doc: https://docs.google.com/document/d/1drYd7kaNbHTcYPR3T_CRDsPcEbFSV7JbJUmhMPeWMqY/
current_repository: https://github.com/nigelrogasch/nibs-bids/tree/master/nibs-bids-v6/
current_preprint_editable: https://docs.google.com/document/d/1xfetyFkXh8kqObfCViUvku69pk4ZZw8BC5GL8_Cq9TI/edit?tab=t.0
current_preprint:
author_list: https://docs.google.com/spreadsheets/d/1oMImk-HXsyLZtTj3yIa1uY9LX3hVpsCiAGZP3oSV0Eo/edit?gid=0#gid=0
content:
- raw
leads:
- given-names: Giacomo
family-names: Bertazzoli
- given-names: Vittorio
family-names: Iacovella
- given-names: Peter J.
family-names: Fried
- given-names: Marta
family-names: Bortoletto
- given-names: Nigel
family-names: Rogasch
past leads (inactive):
- given-names: Carlo
family-names: Miniussi
bids_maintainers:
- given-names: Rémi
family-names: Gau
status:
- v1.0 2020-11-09 Initial work on specification with vittorio.iacovella@unitn.it carlo.miniussi@unitn.it marta.bortoletto@cognitiveneuroscience.it
- v1.0 2021-04 First example of a NIBS-BIDS dataset https://gin.g-node.org/CIMeC/TMS-EEG_brain_connectivity_BIDS
- V1.0 2021-11 Brainhack @Donosti with eleonora.marcantoni@cognitiveneuroscience.it martinabulgari3@gmail.com and g.guidali@campus.unimib.it
- V1.0 2022-02 First international NIBS-BIDS meeting
- V1.0 2022-11 First update of the BEP
- V2.0 2023-08 BEP 2.0 available with the new NIBS-BIDS logic for describing NIBS experiments
- V3.0 2024-07 BEP 3.0 available with a draft of the final BIDS structure.
- V4.0 2024-11 BEP 4.0 updated with a new, more comprehensive structure. Added int files for offline stimulation. Harmonized parameters with SimNIBS.
Use of events and scans files for online stimulation.
- V4.1 2024-12 BEP 4.1 Comments form December 9th, 2024 5th general meeting implemented, added FAQ section. Met with the BIDS maintenance team in
January 2025. Agreed on closing the open discussions, lock the google doc and start the PR.
- V6.2 2026-03 BEP 6.2 Comments form December 12th, 2025 6th general meeting implemented. Met with the BIDS steering team in March 2026. Agreed on
NSF FAIROS application on NIBS-BIDS implementation https://www.nsf.gov/funding/opportunities/fairos-findable-accessible-interoperable-reusable-open-science.
Agreed on creating a preprint version of the BEP037 (to bi cited as a work-in-progress).
blocking:
google_doc_created: 2022-09
pull_request_created:
pull_request_merged:
- number: '039'
title: Dimensionality reduction-based networks
google_doc: https://docs.google.com/document/d/1GTWsj0MFQedXjOaNk6H0or6IDVFyMAysrJ9I4Zmpz2E/
content:
- raw
leads:
- given-names: Arianna
family-names: Sala
- given-names: Anibal
family-names: Sólon
- given-names: Cyrus
family-names: Eierud
- given-names: Franco
family-names: Pestilli
- given-names: Peer
family-names: Herholz
bids_maintainers:
status:
- adapted general spatial derivatives proposal, meaning using `<modality>map`, for example `eegmap` and `boldmap` for spatial components
- decided on `model-` and `items-` as keys to denote the utilized model and component number, if files are 3D, respectively
- updated meta-data and examples
blocking:
google_doc_created: 2021-10
pull_request_created:
pull_request_merged:
- number: '040'
title: Functional Ultrasound
google_doc: https://docs.google.com/document/d/1W3z01mf1E8cfg_OY7ZGqeUeOKv659jCHQBXavtmT-T8/
content:
- raw
leads:
- given-names: Jean-Charles
family-names: Mariani
- given-names: Samuel
family-names: Le Meur-Diebolt
- given-names: Thomas
family-names: Deffieux
bids_maintainers:
- given-names: Rémi
family-names: Gau
status:
- All issues raised on the original BEP have been resolved.
- Regular meetings with contributors scheduled.
- Scanner coordinate system has been proposed to harmonize affine transformations with moving probes.
- 'Perspectives: Starting to bidsify existing datasets to stress test the BEP.'
- 'Perspectives: timing metadata has been copied from the fMRI-BIDS specification, but evolutions could be discussed to handle non-stable sampling
frequencies.'
blocking:
google_doc_created: 2023-03
pull_request_created:
pull_request_merged:
- number: '041'
title: Statistical Model Derivatives
google_doc: https://docs.google.com/document/d/1KHzp-yk8KXvkUIhtN71WU0m4P4kKT9C1yvI-i9_kNeY/
content:
- derivative
leads:
- given-names: Taylor
family-names: Salo
bids_maintainers:
- given-names: Taylor
family-names: Salo
status:
- New BEP, collecting community comments and feedback.
- All collaborators are welcome.
blocking:
google_doc_created: 2022-08
pull_request_created:
pull_request_merged:
- number: '043'
title: BIDS Term Mapping
google_doc: https://docs.google.com/document/d/1LACjc5hFXDpa2l_QddBPR41Vce_gglGv9WeuBB7LsBU/
content:
- metadata
leads:
- given-names: Christopher J.
family-names: Markiewicz
- given-names: Eric A.
family-names: Earl
bids_maintainers:
- given-names: Christopher J.
family-names: Markiewicz
- given-names: Eric A.
family-names: Earl
status:
- For being able to map non-BIDS data as BIDS-compatible.
- Now collecting community comments and feedback.
- All collaborators are welcome.
blocking:
google_doc_created: 2024-03
pull_request_created:
pull_request_merged:
- number: '044'
title: Stimuli
pull_request: https://github.com/bids-standard/bids-specification/pull/2022
html_preview: https://bids-specification--2022.org.readthedocs.build/en/2022/modality-specific-files/stimuli.html
leads:
- given-names: Seyed Yahya
family-names: Shirazi
- given-names: Dora
family-names: Hermes
- given-names: Yaroslav O.
family-names: Halchenko
- given-names: Kay
family-names: Robbins
- given-names: Scott
family-names: Makeig
bids_maintainers:
- given-names: Rémi
family-names: Gau
status:
- Community comments and feedback are being collected (January 2025)
- To harmonize and make more reusable stimuli content under stimuli/
- Collecting community comments and feedback. All collaborators are welcome.
- 'Original issue: [#153](https://github.com/bids-standard/bids-specification/issues/153)'
content:
- raw
blocking:
google_doc_created: 2023-09
pull_request_created: 2024-12
pull_request_merged:
- number: '045'
title: Peripheral Physiological Recordings
display: Physio
pull_request: https://github.com/bids-standard/bids-specification/pull/2267
html_preview: https://bids-specification--2267.org.readthedocs.build/en/2267/modality-specific-files/physiological-recordings.html
google_doc: https://docs.google.com/document/d/1oTfjzY5ZnLIYd0kPPWhR81sBmMuy_jC5YYIaqj6OhSA/edit
leads:
- given-names: Mary
family-names: Miedema
- given-names: Stefano
family-names: Moia
- given-names: Sourav
family-names: Kulkarni
bids_maintainers:
- given-names: Seyed Yahya
family-names: Shirazi
status:
- No longer developed in google doc, moved to <https://github.com/physiopy/bids-specification-physio>.
- To update standards for physiological data for improved clarity and a broader range of use cases.
- Now collecting community comments and feedback. All collaborators are welcome.
- 'Original issue: [#1675](https://github.com/bids-standard/bids-specification/issues/1675).'
content:
- raw
blocking:
google_doc_created: 2024-08
pull_request_created: 2025-11
pull_request_merged:
- number: '046'
title: Diffusion Tractography
display: Tractography
google_doc: https://docs.google.com/document/d/1ubDQ2RhgjnfGqoeukzEkPV9YEHhfYMERrj7-3b0c2HI/edit
leads:
- given-names: Robert E.
family-names: Smith
- given-names: Ariel
family-names: Rokem
- given-names: Franco
family-names: Pestilli
status:
- Porting comprehensive description of streamline tractography mechanisms into specification - 10.1016/B978-0-12-817057-1.00023-8
- Determine appropriate resolution with TRX development - https://tee-ar-ex.github.io/trx-python/
- Decide on scope of BEP; e.g. whether to include tractometry, complex tract delineation
content:
- derivative
blocking:
google_doc_created: 2022-02
pull_request_created:
pull_request_merged:
- number: '047'
title: Audio/video recordings for behavioral experiments
display: Behavioral audio/video recordings
pull_request: https://github.com/bids-standard/bids-specification/pull/2231
html_preview: https://bids-specification--2231.org.readthedocs.build/en/2231/modality-specific-files/behavioral-experiments.html
content:
- raw
leads:
- given-names: Benjamin
family-names: Dichter
bids_maintainers:
- given-names: Seyed Yahya
family-names: Shirazi
status:
- Adds support for storing audio and video behavioral recordings (new `_audio` and `_video` suffixes) in the `beh/` directory.
blocking:
google_doc_created:
pull_request_created: 2025-10
pull_request_merged:
skills/bids/references/bids_schema.json
+21015
View File
File diff suppressed because it is too large Load Diff
skills/bids/references/bids_specification.md
+165
View File
@@ -0,0 +1,165 @@
# BIDS Specification Reference
> **Note**: The canonical, machine-readable source of truth is `bids_schema.json` (in this directory), exported from the [BIDS Schema](https://github.com/bids-standard/bids-specification/tree/master/src/schema). The tables below are a human-readable summary. When the two disagree, trust the schema.
## Entity Table
Complete list of BIDS entities, their keys, and where they apply. **Rows are listed in the required filename ordering** — entities must appear in this order in BIDS filenames. This order is defined in the schema at `rules.entities` (see `bids_schema.json`).
| # | Entity | Key | Format | Applies to |
|---|--------|-----|--------|------------|
| 1 | Subject | `sub-` | `<label>` (alphanumeric) | All files (required) |
| 2 | Template | `tpl-` | `<label>` | derivatives (template-based) |
| 3 | Session | `ses-` | `<label>` | All datatypes |
| 4 | Cohort | `cohort-` | `<label>` | derivatives (template cohorts) |
| 5 | Sample | `sample-` | `<label>` | microscopy |
| 6 | Task | `task-` | `<label>` | func, eeg, meg, ieeg, beh, pet, nirs, motion |
| 7 | Tracking system | `tracksys-` | `<label>` | motion |
| 8 | Acquisition | `acq-` | `<label>` | All datatypes |
| 9 | Nucleus | `nuc-` | `<label>` | MR spectroscopy |
| 10 | Volume | `voi-` | `<label>` | MR spectroscopy |
| 11 | Contrast enhancing agent | `ce-` | `<label>` | anat |
| 12 | Tracer | `trc-` | `<label>` | pet |
| 13 | Stain | `stain-` | `<label>` | microscopy |
| 14 | Reconstruction | `rec-` | `<label>` | anat, func, pet |
| 15 | Direction | `dir-` | `<label>` | fmap, dwi, perf, func |
| 16 | Run | `run-` | `<index>` (integer) | All datatypes |
| 17 | Modality | `mod-` | `<label>` | fieldmaps |
| 18 | Echo | `echo-` | `<index>` | func, fmap |
| 19 | Flip | `flip-` | `<index>` | anat (quantitative MRI) |
| 20 | Inversion | `inv-` | `<index>` | anat (quantitative MRI) |
| 21 | Magnetization transfer | `mt-` | `on`/`off` | anat (quantitative MRI) |
| 22 | Part | `part-` | `mag`/`phase`/`real`/`imag` | anat, func |
| 23 | Processing | `proc-` | `<label>` | eeg, meg, ieeg |
| 24 | Hemisphere | `hemi-` | `L`/`R` | derivatives (surface data) |
| 25 | Space | `space-` | `<label>` | derivatives |
| 26 | Split | `split-` | `<index>` | func, dwi, eeg, meg, ieeg |
| 27 | Recording | `recording-` | `<label>` | physio, stim, eeg, meg |
| 28 | Chunk | `chunk-` | `<index>` | large files split across chunks |
| 29 | Atlas | `atlas-` | `<label>` | derivatives (atlas-based) |
| 30 | Segmentation | `seg-` | `<label>` | derivatives |
| 31 | Scale | `scale-` | `<label>` | derivatives |
| 32 | Resolution | `res-` | `<label>` | derivatives |
| 33 | Density | `den-` | `<label>` | derivatives (surface meshes) |
| 34 | Label | `label-` | `<label>` | derivatives (segmentation labels) |
| 35 | Description | `desc-` | `<label>` | derivatives only |
## Datatypes (Top-Level Directories)
| Datatype | Description | Common Suffixes |
|----------|-------------|-----------------|
| `anat` | Structural MRI | `T1w`, `T2w`, `FLAIR`, `T2star`, `inplaneT1`, `inplaneT2`, `PDw`, `T1map`, `T2map`, `T1rho`, `UNIT1`, `MP2RAGE`, `MTR`, `MTS` |
| `func` | Functional MRI | `bold`, `cbv`, `sbref` |
| `dwi` | Diffusion-weighted imaging | `dwi`, `sbref` |
| `fmap` | Fieldmaps | `phasediff`, `phase1`, `phase2`, `magnitude1`, `magnitude2`, `fieldmap`, `epi` |
| `perf` | Perfusion imaging (ASL) | `asl`, `m0scan`, `aslcontext` |
| `eeg` | Electroencephalography | `eeg`, `channels`, `electrodes`, `events`, `coordsystem` |
| `meg` | Magnetoencephalography | `meg`, `channels`, `coordsystem`, `events`, `headshape` |
| `ieeg` | Intracranial EEG | `ieeg`, `channels`, `electrodes`, `events`, `coordsystem` |
| `pet` | Positron Emission Tomography | `pet`, `blood` |
| `micr` | Microscopy | `2PE`, `BF`, `CARS`, `CONF`, `DIC`, `DF`, `FLUO`, `MPE`, `NLO`, `OCT`, `PC`, `PLI`, `SRS`, `TL` |
| `beh` | Behavioral data (no imaging) | `events`, `beh`, `physio`, `stim` |
| `motion` | Motion capture | `motion`, `channels`, `events` |
| `nirs` | Near-infrared spectroscopy | `nirs`, `channels`, `optodes`, `coordsystem`, `events` |
## File Extensions
| Extension | Description |
|-----------|-------------|
| `.nii.gz` | Compressed NIfTI (standard for MRI/fMRI/DWI) |
| `.nii` | Uncompressed NIfTI |
| `.json` | JSON sidecar metadata |
| `.tsv` | Tab-separated values (events, participants, etc.) |
| `.bvec` | b-vectors (DWI gradient directions) |
| `.bval` | b-values (DWI gradient strengths) |
| `.edf` | European Data Format (EEG) |
| `.bdf` | BioSemi Data Format (EEG) |
| `.vhdr`/`.vmrk`/`.eeg` | BrainVision format (EEG) |
| `.set` | EEGLAB format (EEG) |
| `.fif` | Elekta/MEGIN format (MEG) |
| `.ds` | CTF dataset (MEG) |
| `.sqd`/`.con` | KIT/Yokogawa (MEG) |
## Required Files
### Dataset-level (always required)
- `dataset_description.json`
### Dataset-level (recommended)
- `README` or `README.md`
- `CHANGES`
- `participants.tsv` + `participants.json`
- `LICENSE`
### Run-level (recommended)
- `sub-<label>/[ses-<label>/]sub-<label>[_ses-<label>]_scans.tsv` - per-run acquisition metadata
### Modality-specific required files
- **func/bold**: corresponding `_events.tsv` for task data; `TaskName` in JSON sidecar
- **dwi**: `.bvec` and `.bval` files
- **eeg/meg/ieeg**: `_channels.tsv`, `_events.tsv`
- **perf/asl**: `_aslcontext.tsv`
## Directory Structure Rules
1. Subject directories are named `sub-<label>` and sit at dataset root
2. Session directories `ses-<label>` are optional; if used, must be used for ALL subjects
3. Datatype directories (`anat/`, `func/`, etc.) sit inside subject (or session) directories
4. `sourcedata/` stores raw unprocessed data (DICOM, etc.) - not validated
5. `derivatives/` stores processed outputs - each pipeline in its own subdirectory
6. `code/` stores analysis scripts
7. `stimuli/` stores stimulus files used during acquisition
8. `phenotype/` stores questionnaire/behavioral data not tied to specific imaging
## Metadata Inheritance
JSON metadata cascades from higher to lower directories. If the same key appears at multiple levels, the most specific (closest to the data file) wins.
**Resolution order** (highest priority first):
1. File-level sidecar: `sub-01/func/sub-01_task-rest_bold.json`
2. Subject-level sidecar: `sub-01/sub-01_task-rest_bold.json`
3. Dataset-level sidecar: `task-rest_bold.json`
This avoids duplicating metadata that is constant across subjects (e.g., `RepetitionTime`, `TaskName`).
## Standard Template Spaces
Common `space-` values used in derivatives:
| Space Label | Description |
|-------------|-------------|
| `MNI152NLin2009cAsym` | MNI 2009c nonlinear asymmetric (fMRIPrep default) |
| `MNI152NLin6Asym` | MNI 6th-generation nonlinear asymmetric (FSL default) |
| `MNI152Lin` | MNI linear registration |
| `MNIPediatricAsym` | Pediatric MNI templates |
| `T1w` | Individual subject's T1w native space |
| `fsnative` | FreeSurfer individual surface space |
| `fsaverage` | FreeSurfer average surface (164k vertices) |
| `fsaverage5` | FreeSurfer average surface (10k vertices) |
| `fsaverage6` | FreeSurfer average surface (40k vertices) |
| `fsLR` | HCP fs_LR surface space |
| `OASIS30ANTs` | OASIS-30 ANTs template |
| `UNCInfant` | UNC infant templates |
Full list managed by TemplateFlow: https://www.templateflow.org/
## Specification Changelog (Selected)
| Version | Key Changes |
|---------|-------------|
| 1.10.0 | Motion capture modality; refined derivative entity rules |
| 1.9.0 | NIRS modality; Python-based validator reference implementation |
| 1.8.0 | Microscopy modality; `chunk-` entity for large files |
| 1.7.0 | PET modality fully specified |
| 1.6.0 | EEG/MEG/iEEG matured; `_coordsystem.json` |
| 1.5.0 | Genetic descriptors; ASL perfusion |
| 1.4.0 | `dataset_description.json` expanded; derivatives framework |
| 1.0.0 | Initial release: MRI only (anat, func, dwi, fmap) |
## Entity Label Rules
- **Labels** (`<label>`): alphanumeric only, no special characters, no leading zeros (except `run-`)
- **Indices** (`<index>`): non-negative integers, zero-padded to equal width within a dataset (e.g., `run-01`, `run-02`)
- Subject labels: typically numeric (`01`, `02`) but can be alphanumeric (`CON01`, `PAT01`)
- Session labels: descriptive (`pre`, `post`, `baseline`, `followup`) or numeric
- Task labels: brief, descriptive, no spaces (`rest`, `nback`, `faces`, `gonogo`)
skills/bids/references/conversion_tools.md
+475
View File
@@ -0,0 +1,475 @@
# BIDS Conversion Tools Reference
This reference covers detailed workflows for converting DICOM and other raw data formats to BIDS using the three main conversion tools.
## HeuDiConv
HeuDiConv is the most flexible DICOM-to-BIDS converter. It supports three usage modes — from fully automatic turnkey conversion to fully custom heuristics — and handles duplicates, provenance tracking, and sourcedata archiving out of the box.
**Repository**: https://github.com/nipy/heudiconv
**Docs**: https://heudiconv.readthedocs.io/
**Tutorials**: https://heudiconv.readthedocs.io/en/latest/tutorials.html
### Installation
```bash
uv pip install heudiconv
# HeuDiConv wraps dcm2niix for the actual conversion
# dcm2niix is usually installed as a dependency, but can also be installed via:
# conda install -c conda-forge dcm2niix
# or: apt-get install dcm2niix
```
### Mode 1: ReproIn (Turnkey Conversion — Recommended for New Studies)
If scanner protocol names follow the [ReproIn naming convention](https://github.com/repronim/reproin), conversion is fully automatic with no heuristic file to write. ReproIn is a setup for automatic generation of sharable, version-controlled BIDS datasets directly from MR scanners.
```bash
# Turnkey conversion — just point at DICOMs, HeuDiConv does the rest
heudiconv --files dicom/001 -o data -f reproin --bids --minmeta
```
#### ReproIn Protocol Naming Rules
Protocol names encode BIDS entities directly. Format: `<seqtype>[-<suffix>][_<entity>-<label>]...`
| Protocol name at scanner | BIDS output |
|--------------------------|-------------|
| `anat-T1w` or just `anat` | `sub-XX/anat/sub-XX_T1w.nii.gz` |
| `func-bold_task-rest` or `func_task-rest` | `sub-XX/func/sub-XX_task-rest_bold.nii.gz` |
| `dwi_dir-AP` | `sub-XX/dwi/sub-XX_dir-AP_dwi.nii.gz` |
| `fmap_dir-PA` or `fmap-epi_dir-PA` | `sub-XX/fmap/sub-XX_dir-PA_epi.nii.gz` |
| `fmap_acq-4mm` | `sub-XX/fmap/sub-XX_acq-4mm_epi.nii.gz` |
**Key features:**
- **Default suffixes**: `anat` defaults to `T1w`, `func` to `bold`, `fmap` to `epi` — so they can be omitted
- **Subject ID**: extracted automatically from DICOM metadata (Patient ID)
- **Session**: set once on any sequence (e.g., `anat-scout_ses-pre`) and ReproIn propagates it to all sequences in that scanner Program/Patient
- **Duplicate runs**: automatically numbered (`run-01`, `run-02`, ...) when the same protocol is run multiple times
- **Locator hierarchy**: output is nested under Region/Exam from the scanner's Study Description (customizable with `--locator`)
- **sourcedata**: original DICOMs are archived as `.tgz` files under `sourcedata/` for reproducibility
- **Dashes in names**: scanners may strip dashes from protocol names during DICOM export — ReproIn handles this gracefully
#### ReproIn Overview
See also:
- [ReproIn Walkthrough](https://github.com/repronim/reproin#walkthrough) for scanner setup
- [ReproNim Webinar slides and recording](https://github.com/repronim/reproin#presentations) on HeuDiConv + ReproIn
### Mode 2: Custom Heuristic Mapping into ReproIn (For Existing Data)
If you already have collected data with non-ReproIn protocol names (or cannot control scanner naming), you can write a thin heuristic that maps your protocol names into ReproIn conventions. This gives you all ReproIn benefits (automatic entity handling, duplicate management, sourcedata archiving) while accommodating arbitrary scanner naming.
See https://github.com/repronim/reproin/issues/18 for a brief HOWTO on this approach.
The idea is to write a heuristic whose `infotodict` returns keys that follow ReproIn naming patterns, so the ReproIn machinery handles the rest.
### Mode 3: Custom Heuristic (Full Flexibility)
For studies with complex mappings or non-standard requirements, write a full Python heuristic file. This is the most common workflow for retrospective conversion of existing datasets.
#### Step 1: Reconnaissance — Discover DICOM series
```bash
# -f convertall: built-in heuristic that lists all series without converting
# -c none: don't convert, just generate dicominfo.tsv
heudiconv \
--files dicom/219/itbs/*/*.dcm \
-s 219 \
-f convertall \
-c none \
-o Nifti/
```
This creates `.heudiconv/219/info/dicominfo.tsv` containing one row per DICOM series with columns:
- `series_id`, `sequence_name`, `protocol_name`, `series_description`
- `dim1`-`dim4` (image dimensions), `TR`, `TE`, `image_type`
- `is_derived`, `is_motion_corrected` — important for filtering
Review this TSV (open in a spreadsheet) to understand what was acquired and plan the mapping to BIDS names. Step 1 only needs to be done once per project.
#### Step 2: Write a heuristic file
```python
"""HeuDiConv heuristic for a typical fMRI study.
Study design:
- T1w MPRAGE anatomical
- Resting-state BOLD
- Task BOLD (n-back working memory)
- DWI with two phase-encoding directions
- Fieldmap (phase-difference)
"""
def create_key(template, outtype=('nii.gz',), annotation_classes=None):
if template is None or not template:
raise ValueError('Template must be a valid format string')
return template, outtype, annotation_classes
def infotodict(seqinfo):
"""Heuristic evaluator for determining which runs belong where.
Parameters
----------
seqinfo : list of namedtuples
Each namedtuple has fields: .series_id, .sequence_name,
.protocol_name, .series_description, .dim1, .dim2, .dim3, .dim4,
.TR, .TE, .is_derived, .is_motion_corrected, .image_type, etc.
Returns
-------
info : dict
Keys are tuples from create_key(), values are lists of series_id
"""
# Define BIDS output templates
t1w = create_key(
'sub-{subject}/{session}/anat/sub-{subject}_{session}_T1w'
)
rest_bold = create_key(
'sub-{subject}/{session}/func/sub-{subject}_{session}_task-rest_bold'
)
# {item:02d} auto-numbers runs when the same protocol is run multiple times
nback_bold = create_key(
'sub-{subject}/{session}/func/sub-{subject}_{session}_task-nback_run-{item:02d}_bold'
)
dwi_AP = create_key(
'sub-{subject}/{session}/dwi/sub-{subject}_{session}_dir-AP_dwi'
)
dwi_PA = create_key(
'sub-{subject}/{session}/dwi/sub-{subject}_{session}_dir-PA_dwi'
)
fmap_phasediff = create_key(
'sub-{subject}/{session}/fmap/sub-{subject}_{session}_phasediff'
)
fmap_mag1 = create_key(
'sub-{subject}/{session}/fmap/sub-{subject}_{session}_magnitude1'
)
fmap_mag2 = create_key(
'sub-{subject}/{session}/fmap/sub-{subject}_{session}_magnitude2'
)
info = {
t1w: [], rest_bold: [], nback_bold: [],
dwi_AP: [], dwi_PA: [],
fmap_phasediff: [], fmap_mag1: [], fmap_mag2: [],
}
for s in seqinfo:
protocol = s.protocol_name.lower()
series_desc = s.series_description.lower() if s.series_description else ''
# Anatomical — filter by dim3 to exclude localizers
if ('mprage' in protocol or 't1w' in protocol) and s.dim3 > 100:
info[t1w].append(s.series_id)
# Functional — filter by dim4 and exclude MOCO series
elif 'rest' in protocol and s.dim4 > 10 and not s.is_motion_corrected:
info[rest_bold].append(s.series_id)
elif 'nback' in protocol and s.dim4 > 10 and not s.is_motion_corrected:
info[nback_bold].append(s.series_id)
# Diffusion
elif ('dti' in protocol or 'dwi' in protocol) and s.dim4 > 1:
if 'ap' in protocol or 'ap' in series_desc:
info[dwi_AP].append(s.series_id)
elif 'pa' in protocol or 'pa' in series_desc:
info[dwi_PA].append(s.series_id)
# Fieldmaps
elif 'field' in protocol or 'fmap' in protocol:
if 'ph' in s.image_type_text.lower():
info[fmap_phasediff].append(s.series_id)
elif s.series_description and 'e1' in s.series_description.lower():
info[fmap_mag1].append(s.series_id)
elif s.series_description and 'e2' in s.series_description.lower():
info[fmap_mag2].append(s.series_id)
return info
```
#### Step 3: Convert
```bash
# Convert with custom heuristic
heudiconv \
--files dicom/219/itbs/*/*.dcm \
-s 219 \
-ss itbs \
-f Nifti/code/heuristic.py \
-c dcm2niix \
--bids \
--minmeta \
-o Nifti/
# Or using -d template for batch conversion of multiple subjects
heudiconv \
-d /path/to/dicoms/{subject}/*/*/*.dcm \
-s 01 02 03 04 05 \
-f my_heuristic.py \
-c dcm2niix \
--bids \
--minmeta \
-o /path/to/bids_output
# Key flags:
# --files : point to specific DICOM files/directories
# -d : DICOM path template ({subject}, {session} are replaced)
# -s : subject label(s)
# -ss : session label
# -f : heuristic file path, or built-in name (reproin, convertall)
# -c : converter (dcm2niix, none)
# --bids / -b : output BIDS structure (creates JSON sidecars, etc.)
# --minmeta : prevent excess DICOM metadata from overflowing JSON sidecars
# -o : output directory
# --overwrite : re-run conversion overwriting existing files
```
### The .heudiconv Directory
Every conversion creates/updates a `.heudiconv/` hidden directory alongside the output:
- `.heudiconv/<subject>/info/dicominfo.tsv` — DICOM series metadata
- `.heudiconv/<subject>/info/<heuristic>.py` — copy of the heuristic used
- Conversion records for each subject/session
**Important**: If you re-run conversion for a subject/session that was already processed, HeuDiConv silently reuses cached conversion info from `.heudiconv/`. If troubleshooting, delete the subject's entry from `.heudiconv/` (or the whole directory) and re-run.
Keep `.heudiconv/` with your data — together with `code/` it provides valuable provenance information.
### HeuDiConv Tips
1. **Always use `--minmeta`** to prevent excess DICOM metadata from overflowing JSON sidecars — fMRIPrep and MRIQC may crash on bloated JSON files
2. **Use `{item:02d}` in templates** for auto-numbering runs: if multiple series match, they get `run-01`, `run-02`, etc. Without this, later runs silently overwrite earlier ones
3. **Filter by `dim3`/`dim4`** to exclude localizers (small `dim3`) and single-volume scouts (`dim4 == 1`)
4. **Check `s.is_motion_corrected`** to exclude scanner-generated MOCO series (e.g., `if not s.is_motion_corrected`)
5. **Check `s.is_derived`** to skip other derived/processed series
6. **Store heuristic with dataset** under `code/` for reproducibility
7. **Use `--files`** when DICOM organization doesn't follow a clean `{subject}` template pattern
8. **For new studies**: prefer ReproIn protocol naming from the start — it eliminates the need for custom heuristics entirely
9. **For existing data with arbitrary names**: consider the "map into reproin" approach rather than writing a fully custom heuristic — you get duplicate handling, session propagation, and other ReproIn features for free
## dcm2bids (Configuration-File-Based)
dcm2bids uses JSON configuration files instead of Python heuristics. Simpler for straightforward datasets.
**Repository**: https://github.com/UNFmontreal/Dcm2Bids
**Docs**: https://unfmontreal.github.io/Dcm2Bids/
### Installation
```bash
uv pip install dcm2bids
# Also installs dcm2niix
```
### Workflow
#### Step 1: Scaffold a BIDS directory
```bash
dcm2bids_scaffold -o /path/to/bids_output
```
Creates the basic BIDS structure with `dataset_description.json`, `README`, `.bidsignore`, etc.
#### Step 2: Run helper to inspect DICOM metadata
```bash
dcm2bids_helper -d /path/to/dicom_dir -o /path/to/bids_output
```
Creates `tmp_dcm2bids/helper/` with converted NIfTI files and JSON sidecars. Review the JSON files to find distinguishing metadata fields.
#### Step 3: Write configuration file
```json
{
"descriptions": [
{
"id": "id_t1w",
"datatype": "anat",
"suffix": "T1w",
"criteria": {
"SeriesDescription": "*MPRAGE*",
"ImageType": ["ORIGINAL", "PRIMARY", "M", "ND", "NORM"]
}
},
{
"id": "id_bold_rest",
"datatype": "func",
"suffix": "bold",
"custom_entities": "task-rest",
"criteria": {
"SeriesDescription": "*REST*BOLD*",
"ImageType": ["ORIGINAL", "PRIMARY", "M", "ND", "MOSAIC"]
},
"sidecar_changes": {
"TaskName": "rest"
}
},
{
"id": "id_bold_nback",
"datatype": "func",
"suffix": "bold",
"custom_entities": "task-nback",
"criteria": {
"SeriesDescription": "*NBACK*",
"EchoTime": 0.03
},
"sidecar_changes": {
"TaskName": "nback"
}
},
{
"id": "id_dwi",
"datatype": "dwi",
"suffix": "dwi",
"custom_entities": "dir-AP",
"criteria": {
"SeriesDescription": "*DTI*AP*"
}
},
{
"id": "id_fmap_phasediff",
"datatype": "fmap",
"suffix": "phasediff",
"criteria": {
"SeriesDescription": "*field*map*",
"EchoTime1": 0.00492,
"EchoTime2": 0.00738
},
"sidecar_changes": {
"IntendedFor": [
"bids::sub-{subject}/func/sub-{subject}_task-rest_bold.nii.gz",
"bids::sub-{subject}/func/sub-{subject}_task-nback_bold.nii.gz"
]
}
}
]
}
```
**Configuration file fields:**
- `datatype`: BIDS datatype (`anat`, `func`, `dwi`, `fmap`, etc.)
- `suffix`: BIDS suffix (`T1w`, `bold`, `dwi`, etc.)
- `custom_entities`: additional BIDS entities (`task-rest`, `dir-AP`, `acq-highres`, etc.)
- `criteria`: dictionary of DICOM/JSON metadata fields to match (supports wildcards `*`)
- `sidecar_changes`: fields to add/modify in the output JSON sidecar
- `id`: arbitrary identifier for the description (for logging)
#### Step 4: Convert
```bash
# Single subject
dcm2bids -d /path/to/dicom_dir -p 01 -c dcm2bids_config.json -o /path/to/bids_output
# With session
dcm2bids -d /path/to/dicom_dir -p 01 -s pre -c dcm2bids_config.json -o /path/to/bids_output
# Flags:
# -d : DICOM source directory
# -p : participant label
# -s : session label (optional)
# -c : configuration file
# -o : output BIDS directory
# --auto_extract_entities : auto-detect run numbers from DICOM
# --force_dcm2bids : overwrite existing conversions
```
### dcm2bids Tips
1. **Use `dcm2bids_helper` first** to see exactly what metadata dcm2niix extracts
2. **Criteria matching uses wildcards** (`*`) and is case-sensitive
3. **Multiple criteria** are ANDed together; use the most specific combination
4. **`sidecar_changes`** can inject any BIDS metadata (useful for `TaskName`, `IntendedFor`)
5. **Store config file** under `code/dcm2bids_config.json` for reproducibility
## BIDScoin (GUI + YAML Configuration)
BIDScoin provides a graphical interface and YAML-based configuration. Good for users who prefer visual mapping.
**Repository**: https://github.com/Donders-Institute/bidscoin
**Docs**: https://bidscoin.readthedocs.io/
### Installation
```bash
uv pip install bidscoin
# Optional: install with all plugin dependencies
uv pip install "bidscoin[all]"
```
### Workflow
```bash
# Step 1: Create a bidsmap template by scanning DICOMs
bidsmapper /path/to/raw /path/to/bids
# Step 2: Edit the bidsmap (launches GUI)
bidseditor /path/to/bids
# Step 3: Convert using the finalized bidsmap
bidscoiner /path/to/raw /path/to/bids
```
### BIDScoin Tips
1. **GUI-based editing** is BIDScoin's strength - the `bidseditor` shows DICOM metadata alongside BIDS mapping
2. **YAML bidsmap** can be edited manually if preferred
3. **Plugin architecture** supports custom conversion backends beyond dcm2niix
4. **Good for multi-site studies** where protocol names vary - visual mapping makes differences obvious
## Comparison
| Feature | HeuDiConv | dcm2bids | BIDScoin |
|---------|-----------|----------|----------|
| Configuration | Python heuristic | JSON config | YAML + GUI |
| Flexibility | Highest (full Python) | Medium (criteria matching) | Medium (plugin system) |
| Learning curve | Steeper (Python) | Moderate | Gentlest (GUI) |
| Batch processing | Excellent | Good | Good |
| ReproIn support | Built-in | No | No |
| DataLad integration | Built-in | No | No |
| Best for | Complex studies, automation | Simple-to-moderate studies | Visual learners, multi-site |
| Active development | Yes | Yes | Yes |
## Post-Conversion Checklist
After converting DICOM to BIDS with any tool:
1. **Run the BIDS validator**: `bids-validator /path/to/bids_output`
2. **Check JSON sidecars** for critical fields (`RepetitionTime`, `TaskName`, `SliceTiming`, `PhaseEncodingDirection`)
3. **Verify NIfTI headers** match expectations (dimensions, voxel sizes, orientation)
4. **Add missing metadata** that dcm2niix couldn't extract from DICOM
5. **Create `participants.tsv`** with demographic data
6. **Write events files** for task fMRI
7. **Write `README`** describing the dataset
8. **Deface anatomical images** if sharing data
9. **Run `bids-validator` again** after any manual modifications
## Common DICOM-to-BIDS Pitfalls
### Multiband/SMS sequences
- dcm2niix may split slices incorrectly for multiband data
- Check `dim4` (number of volumes) matches expectations
- Verify `SliceTiming` is correct for the multiband factor
### Dual-echo fieldmaps
- Siemens stores both echoes in one series; dcm2niix splits them
- GE/Philips may store them as separate series
- Verify `EchoTime1` < `EchoTime2` in the phasediff sidecar
### Phase encoding direction
- DICOM `InPlanePhaseEncodingDirection` → BIDS `PhaseEncodingDirection`
- Mapping depends on acquisition orientation and NIfTI axis conventions
- **Always verify** by checking the actual distortion pattern in the images
### Multi-run numbering
- Ensure runs are numbered sequentially (`run-01`, `run-02`)
- HeuDiConv: use `{item:02d}` placeholder
- dcm2bids: use `--auto_extract_entities` or manually specify runs
### Derived/processed series
- Scanners may export inline-processed data (e.g., motion-corrected, distortion-corrected)
- These should NOT be converted to BIDS raw data
- Filter by `ImageType` containing `DERIVED` or `is_derived` flag in HeuDiConv
skills/bids/references/metadata_fields.md
+365
View File
@@ -0,0 +1,365 @@
# BIDS Metadata Fields Reference
This reference lists the required and recommended JSON sidecar fields for each BIDS modality.
**Legend:**
- **R** = Required
- **REC** = Recommended
- **OPT** = Optional
## Common MRI Fields (All MRI Modalities)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `MagneticFieldStrength` | REC | number | Field strength in Tesla |
| `Manufacturer` | REC | string | Scanner manufacturer |
| `ManufacturersModelName` | REC | string | Scanner model |
| `DeviceSerialNumber` | REC | string | Scanner serial number |
| `StationName` | REC | string | Scanner station name |
| `SoftwareVersions` | REC | string | Scanner software version |
| `InstitutionName` | REC | string | Name of institution |
| `InstitutionAddress` | REC | string | Address of institution |
| `InstitutionalDepartmentName` | REC | string | Department name |
## Anatomical MRI (anat/)
### T1w, T2w, FLAIR, T2star, PDw
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `RepetitionTime` | REC | number | TR in seconds |
| `EchoTime` | REC | number | TE in seconds |
| `InversionTime` | REC | number | TI in seconds (if applicable) |
| `FlipAngle` | REC | number | Flip angle in degrees |
| `SequenceName` | REC | string | Pulse sequence name |
| `SequenceVariant` | REC | string | Variant of the sequence |
| `ScanningSequence` | REC | string | General description |
| `PulseSequenceType` | REC | string | Type of pulse sequence |
| `NonlinearGradientCorrection` | REC | boolean | Whether applied |
| `ParallelReductionFactorInPlane` | REC | number | iPAT/GRAPPA factor |
| `ContrastBolusIngredient` | REC | string | Active contrast ingredient |
### Quantitative MRI (T1map, T2map, etc.)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `RepetitionTimeExcitation` | R | number | Excitation TR in seconds |
| `RepetitionTimePrepration` | R | number | Preparation TR in seconds |
| `FlipAngle` | R | number/array | Flip angle(s) in degrees |
| `MTState` | R | boolean | Magnetization transfer on/off |
| `SpoilingState` | REC | boolean | Whether RF spoiling applied |
| `SpoilingType` | REC | string | `RF`, `GRADIENT`, or `COMBINED` |
| `SpoilingRFPhaseIncrement` | REC | number | Phase increment in degrees |
## Functional MRI (func/)
### BOLD
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `RepetitionTime` | R | number | TR in seconds (volume acquisition time) |
| `TaskName` | R | string | Name of the task (must match `task-<label>`) |
| `SliceTiming` | REC | array | Time each slice was acquired, in seconds |
| `EchoTime` | REC | number | TE in seconds |
| `FlipAngle` | REC | number | Flip angle in degrees |
| `PhaseEncodingDirection` | REC | string | `i`, `i-`, `j`, `j-`, `k`, `k-` |
| `EffectiveEchoSpacing` | REC | number | Effective echo spacing in seconds |
| `TotalReadoutTime` | REC | number | Total readout time in seconds |
| `MultibandAccelerationFactor` | REC | number | Multiband/SMS factor |
| `NumberOfVolumesDiscardedByScanner` | REC | integer | Dummy scans removed |
| `NumberOfVolumesDiscardedByUser` | REC | integer | Volumes removed post-hoc |
| `TaskDescription` | REC | string | Longer description of the task |
| `CogAtlasID` | REC | string | Cognitive Atlas ID for the task |
| `CogPOID` | REC | string | Cognitive Paradigm Ontology ID |
| `Instructions` | REC | string | Instructions given to participants |
### Multi-echo BOLD
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `EchoTime` | R | number | TE for this echo (each echo in separate file) |
| `EchoTime1`, `EchoTime2` | - | - | NOT used; use `echo-<index>` entity |
### BOLD Timing Details
**SliceTiming** - Array of times (in seconds) at which each slice was acquired relative to the start of volume acquisition. Length must equal the number of slices.
Example for ascending sequential (3 slices, TR=2s):
```json
{"SliceTiming": [0.0, 0.667, 1.333]}
```
Example for interleaved (odd-first, 6 slices, TR=2s):
```json
{"SliceTiming": [0.0, 0.667, 1.333, 0.333, 1.0, 1.667]}
```
**PhaseEncodingDirection** values:
- `i` / `i-` : along first image axis (typically left-right)
- `j` / `j-` : along second image axis (typically anterior-posterior)
- `k` / `k-` : along third image axis (typically inferior-superior)
- The `-` suffix indicates the negative direction along that axis
## Diffusion-Weighted Imaging (dwi/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `PhaseEncodingDirection` | R | string | Phase encoding direction |
| `TotalReadoutTime` | R | number | Total readout time in seconds |
| `EchoTime` | REC | number | TE in seconds |
| `RepetitionTime` | REC | number | TR in seconds |
| `FlipAngle` | REC | number | Flip angle in degrees |
| `EffectiveEchoSpacing` | REC | number | Effective echo spacing in seconds |
| `MultibandAccelerationFactor` | REC | number | SMS/multiband factor |
| `SliceTiming` | REC | array | Slice timing |
### DWI Gradient Files
`.bvec` file (3 rows x N columns, N = number of volumes):
```
0 0.707 -0.707 0 0.577
0 0.707 0.707 0 0.577
0 0 0 1 0.577
```
`.bval` file (1 row x N columns):
```
0 1000 1000 1000 2000
```
- b=0 volumes have zero-vectors in `.bvec`
- Gradient directions are in the image coordinate system
- Values are space-separated (not tab-separated)
- Number of columns must match number of volumes in the NIfTI
## Fieldmaps (fmap/)
### Case 1: Phase-difference map (`_phasediff`)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `EchoTime1` | R | number | TE of the first echo (shorter) |
| `EchoTime2` | R | number | TE of the second echo (longer) |
| `IntendedFor` | R | string/array | BIDS URI(s) of files to correct |
| `B0FieldIdentifier` | REC | string | Identifier for this B0 field |
### Case 2: Two phase maps (`_phase1`, `_phase2`)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `EchoTime` | R | number | TE for this phase image |
| `IntendedFor` | R | string/array | Files to correct |
### Case 3: Direct fieldmap (`_fieldmap`)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `Units` | R | string | Must be `Hz` or `rad/s` |
| `IntendedFor` | R | string/array | Files to correct |
### Case 4: "Pepolar" fieldmaps (`_epi`)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `PhaseEncodingDirection` | R | string | PE direction for this image |
| `TotalReadoutTime` | R | number | Total readout time |
| `IntendedFor` | R | string/array | Files to correct |
| `B0FieldIdentifier` | REC | string | Identifier for this B0 field |
| `B0FieldSource` | REC | string | Which B0 field to use |
### IntendedFor Syntax
**BIDS URI format** (recommended, v1.7+):
```json
{
"IntendedFor": [
"bids::sub-01/func/sub-01_task-rest_bold.nii.gz",
"bids::sub-01/dwi/sub-01_dwi.nii.gz"
]
}
```
**Relative path format** (legacy):
```json
{
"IntendedFor": [
"func/sub-01_task-rest_bold.nii.gz",
"dwi/sub-01_dwi.nii.gz"
]
}
```
**B0FieldIdentifier/B0FieldSource** (preferred in v1.9+):
```json
// In the fieldmap sidecar
{"B0FieldIdentifier": "pepolar_fmap0"}
// In the BOLD sidecar
{"B0FieldSource": "pepolar_fmap0"}
```
## Perfusion Imaging (perf/)
### ASL
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `ArterialSpinLabelingType` | R | string | `CASL`, `PCASL`, or `PASL` |
| `PostLabelingDelay` | R | number/array | PLD in seconds |
| `BackgroundSuppression` | R | boolean | Whether applied |
| `MagneticFieldStrength` | R | number | In Tesla |
| `M0Type` | R | string | `Separate`, `Included`, `Estimate`, `Absent` |
| `RepetitionTimePreparation` | R | number | Time between ASL pulses |
| `LabelingDuration` | R | number | Duration of labeling pulse |
| `BackgroundSuppressionNumberPulses` | REC | integer | Number of suppression pulses |
| `BackgroundSuppressionPulseTime` | REC | array | Timing of suppression pulses |
| `VascularCrushing` | REC | boolean | Whether applied |
| `LabelingOrientation` | REC | string | Orientation of labeling plane |
| `LabelingDistance` | REC | number | Distance from isocenter (mm) |
| `BolusCutOffFlag` | R (PASL) | boolean | Whether QUIPSS applied |
| `BolusCutOffTimingSequence` | R (PASL) | string | QUIPSS sequence type |
| `BolusCutOffDelayTime` | R (PASL) | number | QUIPSS delay time |
### aslcontext.tsv
Required file listing the order of volumes (label/control/m0scan):
```
volume_type
control
label
control
label
m0scan
```
## EEG (eeg/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `TaskName` | R | string | Name of the task |
| `SamplingFrequency` | R | number | In Hz |
| `EEGReference` | R | string | Reference electrode(s) |
| `PowerLineFrequency` | R | number | 50 or 60 Hz (or `n/a`) |
| `SoftwareFilters` | R | object | Online filters applied |
| `EEGPlacementScheme` | REC | string | e.g., `10-20`, `10-10` |
| `CapManufacturer` | REC | string | Cap manufacturer |
| `CapManufacturersModelName` | REC | string | Cap model |
| `EEGChannelCount` | REC | integer | Number of EEG channels |
| `EOGChannelCount` | REC | integer | Number of EOG channels |
| `ECGChannelCount` | REC | integer | Number of ECG channels |
| `EMGChannelCount` | REC | integer | Number of EMG channels |
| `MiscChannelCount` | REC | integer | Number of misc channels |
| `TriggerChannelCount` | REC | integer | Number of trigger channels |
| `RecordingDuration` | REC | number | In seconds |
| `RecordingType` | REC | string | `continuous`, `epoched`, `discontinuous` |
### channels.tsv (EEG)
| Column | Status | Description |
|--------|--------|-------------|
| `name` | R | Channel name |
| `type` | R | `EEG`, `EOG`, `ECG`, `EMG`, `MISC`, `TRIG`, etc. |
| `units` | R | `V`, `mV`, `uV` |
| `sampling_frequency` | OPT | Per-channel if different |
| `low_cutoff` | REC | High-pass filter frequency (Hz) |
| `high_cutoff` | REC | Low-pass filter frequency (Hz) |
| `notch` | REC | Notch filter frequency (Hz) |
| `reference` | REC | Reference electrode name |
| `status` | REC | `good` or `bad` |
| `status_description` | OPT | Reason for bad status |
### electrodes.tsv (EEG)
| Column | Status | Description |
|--------|--------|-------------|
| `name` | R | Electrode name |
| `x` | R | X coordinate |
| `y` | R | Y coordinate |
| `z` | R | Z coordinate |
| `type` | OPT | Electrode type |
| `material` | OPT | Electrode material |
| `impedance` | OPT | Impedance in kOhm |
## MEG (meg/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `TaskName` | R | string | Name of the task |
| `SamplingFrequency` | R | number | In Hz |
| `PowerLineFrequency` | R | number | 50 or 60 Hz |
| `DewarPosition` | R | string | Position of the dewar |
| `SoftwareFilters` | R | object | Online filters |
| `DigitizedLandmarks` | R | boolean | Fiducials digitized |
| `DigitizedHeadPoints` | R | boolean | Head shape digitized |
| `MEGChannelCount` | REC | integer | Number of MEG channels |
| `MEGREFChannelCount` | REC | integer | Reference channels |
| `ContinuousHeadLocalization` | REC | boolean | HPI on |
| `HeadCoilFrequency` | REC | array | HPI coil frequencies |
| `InstitutionName` | REC | string | Institution name |
## PET (pet/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `TracerName` | R | string | Name of the radiotracer |
| `TracerRadionuclide` | R | string | e.g., `C11`, `F18`, `O15` |
| `InjectedRadioactivity` | R | number | In MBq |
| `InjectedRadioactivityUnits` | R | string | Must be `MBq` |
| `InjectedMass` | R | number | Mass of tracer injected |
| `InjectedMassUnits` | R | string | e.g., `ug` |
| `ModeOfAdministration` | R | string | `bolus`, `infusion`, `bolus-infusion` |
| `TimeZero` | R | string | Time of injection (HH:MM:SS) |
| `ScanStart` | R | number | Start time relative to TimeZero |
| `InjectionStart` | R | number | Injection time relative to TimeZero |
| `FrameTimesStart` | R | array | Frame start times in seconds |
| `FrameDuration` | R | array | Frame durations in seconds |
| `Units` | R | string | Unit of voxel values (e.g., `Bq/mL`) |
| `TracerRadLex` | REC | string | RadLex ID for tracer |
| `BodyWeight` | REC | number | In kg |
| `BodyPart` | REC | string | Imaged body part |
| `AttenuationCorrection` | REC | string | Method description |
| `ReconMethodName` | REC | string | Reconstruction method |
| `ReconMethodParameterLabels` | REC | array | Parameter names |
| `ReconMethodParameterValues` | REC | array | Parameter values |
| `ReconFilterType` | REC | string | Post-recon filter type |
| `ReconFilterSize` | REC | number | Filter FWHM in mm |
## Microscopy (micr/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `Manufacturer` | R | string | Microscope manufacturer |
| `ManufacturersModelName` | R | string | Microscope model |
| `PixelSize` | R | array | [X, Y] or [X, Y, Z] in micrometers |
| `PixelSizeUnits` | R | string | `um` (micrometers) |
| `Magnification` | REC | number | Objective magnification |
| `SampleEnvironment` | R | string | `in vivo`, `ex vivo`, `in vitro` |
| `SampleFixation` | REC | string | Fixation method |
| `SampleStaining` | REC | string | Staining protocol |
| `SliceThickness` | REC | number | In micrometers |
| `TissueDeformationScaling` | REC | number | Scaling factor |
## NIRS (nirs/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `TaskName` | R | string | Name of the task |
| `SamplingFrequency` | R | number | In Hz |
| `NIRSSourceOptodeCount` | R | integer | Number of sources |
| `NIRSDetectorOptodeCount` | R | integer | Number of detectors |
| `ACCELChannelCount` | REC | integer | Accelerometer channels |
| `NIRSPlacementScheme` | REC | string | e.g., `10-20` |
## Motion (motion/)
| Field | Status | Type | Description |
|-------|--------|------|-------------|
| `TaskName` | R | string | Name of the task |
| `SamplingFrequency` | R | number | In Hz |
| `TrackingSystemName` | R | string | Name of tracking system |
| `ACCELChannelCount` | REC | integer | Accelerometer channels |
| `GYROChannelCount` | REC | integer | Gyroscope channels |
| `MAGNChannelCount` | REC | integer | Magnetometer channels |
| `RotationOrder` | REC | string | e.g., `XYZ` |
| `RotationRule` | REC | string | `left-hand` or `right-hand` |
| `SpatialAxes` | REC | string | e.g., `ALS` |
skills/bids/scripts/update_schema.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Update BIDS schema JSON and BEPs list from upstream sources.
Downloads:
- bids_schema.json from bids-specification ReadTheDocs (stable release)
- beps.yml from bids-standard/bids-website (current BEP listing)
Usage:
python scripts/update_schema.py
# Fetch schema for a specific spec version or PR preview:
python scripts/update_schema.py --schema-url https://bids-specification.readthedocs.io/en/v1.11.0/schema.json
# Fetch a BEP-specific schema from bids-standard/bids-schema:
python scripts/update_schema.py --schema-url https://raw.githubusercontent.com/bids-standard/bids-schema/main/BEPs/BEP032/schema.json
No external dependencies beyond the Python standard library.
"""
import argparse
import json
import urllib.request
from pathlib import Path
REFERENCES_DIR = Path(__file__).resolve().parent.parent / "references"
SCHEMA_URL = "https://bids-specification.readthedocs.io/en/stable/schema.json"
BEPS_URL = "https://raw.githubusercontent.com/bids-standard/bids-website/main/data/beps/beps.yml"
def fetch(url):
"""Fetch URL content as bytes."""
print(f"Fetching {url} ...")
req = urllib.request.Request(url, headers={"User-Agent": "bids-skill-updater/1.0"})
with urllib.request.urlopen(req) as resp:
return resp.read()
def update_schema(url):
"""Download schema.json and report version info."""
data = fetch(url)
output = REFERENCES_DIR / "bids_schema.json"
# Validate it's proper JSON and extract version
d = json.loads(data)
# Re-serialize with consistent formatting
with open(output, "w") as f:
json.dump(d, f, indent=2)
f.write("\n")
sv = d.get("schema_version", "?")
bv = d.get("bids_version", "?")
print(f" -> {output.name}: schema {sv} / BIDS {bv}")
def update_beps():
"""Download beps.yml."""
data = fetch(BEPS_URL)
output = REFERENCES_DIR / "beps.yml"
output.write_bytes(data)
# Count entries
count = data.count(b"\n- number:")
print(f" -> {output.name}: {count} BEPs")
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"--schema-url",
default=SCHEMA_URL,
help=f"URL for schema.json (default: {SCHEMA_URL})",
)
parser.add_argument(
"--skip-beps",
action="store_true",
help="Skip fetching beps.yml",
)
args = parser.parse_args()
update_schema(args.schema_url)
if not args.skip_beps:
update_beps()
print("Done.")
if __name__ == "__main__":
main()
skills/biopython/SKILL.md
+463
View File
@@ -0,0 +1,463 @@
---
name: biopython
description: Comprehensive molecular biology toolkit. Use for sequence manipulation, file parsing (FASTA/GenBank/PDB), phylogenetics, and programmatic NCBI/PubMed access (Bio.Entrez). Best for batch processing, custom bioinformatics pipelines, BLAST automation. For quick lookups use gget; for multi-service integration use bioservices.
allowed-tools: Read Write Edit Bash
compatibility: Requires Python 3.10+, NumPy, and Biopython. Entrez and web BLAST examples require network access; local BLAST/MUSCLE examples require those command-line tools installed separately.
license: Biopython License Agreement
required_environment_variables: [{"name": "NCBI_EMAIL", "prompt": "Email for NCBI Entrez identification (required by NCBI policy for Entrez calls).", "required_for": "optional features"}, {"name": "NCBI_API_KEY", "prompt": "NCBI API key to raise Entrez rate limits.", "required_for": "optional features"}]
metadata: {"version": "1.2", "skill-author": "K-Dense Inc.", "openclaw": {"envVars": [{"name": "NCBI_EMAIL", "required": false, "description": "Email for NCBI Entrez identification (required by NCBI policy for Entrez calls)."}, {"name": "NCBI_API_KEY", "required": false, "description": "NCBI API key to raise Entrez rate limits."}]}}
---
# Biopython: Computational Molecular Biology in Python
## Overview
Biopython is a comprehensive set of freely available Python tools for biological computation. It provides functionality for sequence manipulation, file I/O, database access, structural bioinformatics, phylogenetics, and many other bioinformatics tasks. The current version is **Biopython 1.87** (released 30 March 2026). It supports **Python 3.10-3.14** and PyPy3.10, and requires NumPy. Biopython 1.87 also addresses **CVE-2025-68463** in `Bio.Entrez.Parser` when parsing untrusted files, so prefer 1.87+ for workflows that parse externally supplied Entrez XML.
## When to Use This Skill
Use this skill when:
- Working with biological sequences (DNA, RNA, or protein)
- Reading, writing, or converting biological file formats (FASTA, GenBank, FASTQ, PDB, mmCIF, etc.)
- Accessing NCBI databases (GenBank, PubMed, Protein, Gene, etc.) via Entrez
- Running BLAST searches or parsing BLAST results
- Performing sequence alignments (pairwise or multiple sequence alignments)
- Analyzing protein structures from PDB files
- Creating, manipulating, or visualizing phylogenetic trees
- Finding sequence motifs or analyzing motif patterns
- Calculating sequence statistics (GC content, molecular weight, melting temperature, etc.)
- Performing structural bioinformatics tasks
- Working with population genetics data
- Any other computational molecular biology task
## Core Capabilities
Biopython is organized into modular sub-packages, each addressing specific bioinformatics domains:
1. **Sequence Handling** - Bio.Seq and Bio.SeqIO for sequence manipulation and file I/O
2. **Alignment Analysis** - Bio.Align and Bio.AlignIO for pairwise and multiple sequence alignments
3. **Database Access** - Bio.Entrez for programmatic access to NCBI databases
4. **BLAST Operations** - Bio.Blast for running and parsing BLAST searches
5. **Structural Bioinformatics** - Bio.PDB for working with 3D protein structures
6. **Phylogenetics** - Bio.Phylo for phylogenetic tree manipulation and visualization
7. **Advanced Features** - Motifs, population genetics, sequence utilities, and more
## Installation and Setup
Install the current stable Biopython release with an explicit version pin for reproducibility:
```bash
uv pip install "biopython==1.87"
```
For NCBI database access, always set your email address (required by NCBI). For reusable software, set a stable `Entrez.tool` value and register the tool/email with NCBI. For higher rate limits (10 req/s instead of 3 req/s), read only `NCBI_API_KEY` from the environment — do not hardcode keys or load unrelated environment variables:
```python
import os
from Bio import Entrez
Entrez.email = "your.email@example.com" # required — use your real email
Entrez.tool = "your_tool_name" # optional but recommended for reusable software
# Optional: register at https://www.ncbi.nlm.nih.gov/account/settings/
if api_key := os.environ.get("NCBI_API_KEY"):
Entrez.api_key = api_key
```
## Using This Skill
This skill provides comprehensive documentation organized by functionality area. When working on a task, consult the relevant reference documentation:
### 1. Sequence Handling (Bio.Seq & Bio.SeqIO)
**Reference:** `references/sequence_io.md`
Use for:
- Creating and manipulating biological sequences
- Reading and writing sequence files (FASTA, GenBank, FASTQ, etc.)
- Converting between file formats
- Extracting sequences from large files
- Sequence translation, transcription, and reverse complement
- Working with SeqRecord objects
**Quick example:**
```python
from Bio import SeqIO
# Read sequences from FASTA file
for record in SeqIO.parse("sequences.fasta", "fasta"):
print(f"{record.id}: {len(record.seq)} bp")
# Convert GenBank to FASTA
SeqIO.convert("input.gb", "genbank", "output.fasta", "fasta")
```
### 2. Alignment Analysis (Bio.Align & Bio.AlignIO)
**Reference:** `references/alignment.md`
Use for:
- Pairwise sequence alignment (global and local)
- Reading and writing multiple sequence alignments
- Using substitution matrices (BLOSUM, PAM)
- Calculating alignment statistics
- Customizing alignment parameters
**Quick example:**
```python
from Bio import Align
# Pairwise alignment
aligner = Align.PairwiseAligner()
aligner.mode = 'global'
alignments = aligner.align("ACCGGT", "ACGGT")
print(alignments[0])
```
### 3. Database Access (Bio.Entrez)
**Reference:** `references/databases.md`
Use for:
- Searching NCBI databases (PubMed, GenBank, Protein, Gene, etc.)
- Downloading sequences and records
- Fetching publication information
- Finding related records across databases
- Batch downloading with proper rate limiting
**Quick example:**
```python
from Bio import Entrez
Entrez.email = "your.email@example.com"
# Search PubMed
handle = Entrez.esearch(db="pubmed", term="biopython", retmax=10)
results = Entrez.read(handle)
handle.close()
print(f"Found {results['Count']} results")
```
### 4. BLAST Operations (Bio.Blast)
**Reference:** `references/blast.md`
Use for:
- Running BLAST searches via NCBI web services
- Running local BLAST searches
- Parsing BLAST XML output
- Filtering results by E-value or identity
- Extracting hit sequences
**Quick example:**
```python
from Bio.Blast import NCBIWWW, NCBIXML
# Run BLAST search
result_handle = NCBIWWW.qblast("blastn", "nt", "ATCGATCGATCG")
blast_record = NCBIXML.read(result_handle)
# Display top hits
for alignment in blast_record.alignments[:5]:
print(f"{alignment.title}: E-value={alignment.hsps[0].expect}")
```
### 5. Structural Bioinformatics (Bio.PDB)
**Reference:** `references/structure.md`
Use for:
- Parsing PDB and mmCIF structure files
- Navigating protein structure hierarchy (SMCRA: Structure/Model/Chain/Residue/Atom)
- Calculating distances, angles, and dihedrals
- Secondary structure assignment (DSSP)
- Structure superimposition and RMSD calculation
- Extracting sequences from structures
**Quick example:**
```python
from Bio.PDB import PDBParser
# Parse structure
parser = PDBParser(QUIET=True)
structure = parser.get_structure("1crn", "1crn.pdb")
# Calculate distance between alpha carbons
chain = structure[0]["A"]
distance = chain[10]["CA"] - chain[20]["CA"]
print(f"Distance: {distance:.2f} Å")
```
### 6. Phylogenetics (Bio.Phylo)
**Reference:** `references/phylogenetics.md`
Use for:
- Reading and writing phylogenetic trees (Newick, NEXUS, phyloXML)
- Building trees from distance matrices or alignments
- Tree manipulation (pruning, rerooting, ladderizing)
- Calculating phylogenetic distances
- Creating consensus trees
- Visualizing trees
**Quick example:**
```python
from Bio import Phylo
# Read and visualize tree
tree = Phylo.read("tree.nwk", "newick")
Phylo.draw_ascii(tree)
# Calculate distance
distance = tree.distance("Species_A", "Species_B")
print(f"Distance: {distance:.3f}")
```
### 7. Advanced Features
**Reference:** `references/advanced.md`
Use for:
- **Sequence motifs** (Bio.motifs) - Finding and analyzing motif patterns
- **Population genetics** (Bio.PopGen) - GenePop files, Fst calculations, Hardy-Weinberg tests
- **Sequence utilities** (Bio.SeqUtils) - GC content, melting temperature, molecular weight, protein analysis
- **Restriction analysis** (Bio.Restriction) - Finding restriction enzyme sites
- **Clustering** (Bio.Cluster) - K-means and hierarchical clustering
- **Genome diagrams** (GenomeDiagram) - Visualizing genomic features
**Quick example:**
```python
from Bio.SeqUtils import gc_fraction, molecular_weight
from Bio.Seq import Seq
seq = Seq("ATCGATCGATCG")
print(f"GC content: {gc_fraction(seq):.2%}")
print(f"Molecular weight: {molecular_weight(seq, seq_type='DNA'):.2f} g/mol")
```
## General Workflow Guidelines
### Reading Documentation
When a user asks about a specific Biopython task:
1. **Identify the relevant module** based on the task description
2. **Read the appropriate reference file** using the Read tool
3. **Extract relevant code patterns** and adapt them to the user's specific needs
4. **Combine multiple modules** when the task requires it
Example search patterns for reference files:
```bash
# Find information about specific functions
rg -n "SeqIO.parse" references/sequence_io.md
# Find examples of specific tasks
rg -n "BLAST" references/blast.md
# Find information about specific concepts
rg -n "alignment" references/alignment.md
```
### Writing Biopython Code
Follow these principles when writing Biopython code:
1. **Import modules explicitly**
```python
from Bio import SeqIO, Entrez
from Bio.Seq import Seq
```
2. **Set Entrez email** when using NCBI databases; load only `NCBI_API_KEY` from the environment if present
```python
import os
from Bio import Entrez
Entrez.email = "your.email@example.com"
Entrez.tool = "your_tool_name"
if api_key := os.environ.get("NCBI_API_KEY"):
Entrez.api_key = api_key
```
3. **Use appropriate file formats** - Check which format best suits the task
```python
# Common formats: "fasta", "genbank", "fastq", "clustal", "phylip"
```
4. **Handle files properly** - Close handles after use or use context managers
```python
with open("file.fasta") as handle:
records = SeqIO.parse(handle, "fasta")
```
5. **Use iterators for large files** - Avoid loading everything into memory
```python
for record in SeqIO.parse("large_file.fasta", "fasta"):
# Process one record at a time
```
6. **Handle errors gracefully** - Network operations and file parsing can fail
```python
from urllib.error import HTTPError
try:
handle = Entrez.efetch(db="nucleotide", id=accession)
except HTTPError as e:
print(f"Error: {e}")
```
## Common Patterns
### Pattern 1: Fetch Sequence from GenBank
```python
from Bio import Entrez, SeqIO
Entrez.email = "your.email@example.com"
# Fetch sequence
handle = Entrez.efetch(db="nucleotide", id="EU490707", rettype="gb", retmode="text")
record = SeqIO.read(handle, "genbank")
handle.close()
print(f"Description: {record.description}")
print(f"Sequence length: {len(record.seq)}")
```
### Pattern 2: Sequence Analysis Pipeline
```python
from Bio import SeqIO
from Bio.SeqUtils import gc_fraction
for record in SeqIO.parse("sequences.fasta", "fasta"):
# Calculate statistics
gc = gc_fraction(record.seq)
length = len(record.seq)
# Find ORFs, translate, etc.
protein = record.seq.translate()
print(f"{record.id}: {length} bp, GC={gc:.2%}")
```
### Pattern 3: BLAST and Fetch Top Hits
```python
from Bio.Blast import NCBIWWW, NCBIXML
from Bio import Entrez, SeqIO
Entrez.email = "your.email@example.com"
# Run BLAST
result_handle = NCBIWWW.qblast("blastn", "nt", sequence)
blast_record = NCBIXML.read(result_handle)
# Get top hit accessions
accessions = [aln.accession for aln in blast_record.alignments[:5]]
# Fetch sequences
for acc in accessions:
handle = Entrez.efetch(db="nucleotide", id=acc, rettype="fasta", retmode="text")
record = SeqIO.read(handle, "fasta")
handle.close()
print(f">{record.description}")
```
### Pattern 4: Build Phylogenetic Tree from Sequences
```python
from Bio import AlignIO, Phylo
from Bio.Phylo.TreeConstruction import DistanceCalculator, DistanceTreeConstructor
# Read alignment
alignment = AlignIO.read("alignment.fasta", "fasta")
# Calculate distances
calculator = DistanceCalculator("identity")
dm = calculator.get_distance(alignment)
# Build tree
constructor = DistanceTreeConstructor()
tree = constructor.nj(dm)
# Visualize
Phylo.draw_ascii(tree)
```
## Best Practices
1. **Always read relevant reference documentation** before writing code
2. **Use grep to search reference files** for specific functions or examples
3. **Validate file formats** before parsing
4. **Handle missing data gracefully** - Not all records have all fields
5. **Cache downloaded data** - Don't repeatedly download the same sequences
6. **Respect NCBI rate limits** - Use API keys, registered tool/email values for reusable software, and Entrez history/batching for large jobs
7. **Test with small datasets** before processing large files
8. **Keep Biopython updated** to get latest features and bug fixes
9. **Use appropriate genetic code tables** for translation
10. **Document analysis parameters** for reproducibility
## Troubleshooting Common Issues
### Issue: "No handlers could be found for logger 'Bio.Entrez'"
**Solution:** This is just a warning. Set Entrez.email to suppress it.
### Issue: "HTTP Error 400" from NCBI
**Solution:** Check that IDs/accessions are valid and properly formatted.
### Issue: "ValueError: EOF" when parsing files
**Solution:** Verify file format matches the specified format string.
### Issue: Alignment fails with "sequences are not the same length"
**Solution:** Ensure sequences are aligned before using AlignIO or MultipleSeqAlignment.
### Issue: BLAST searches are slow
**Solution:** Use local BLAST for large-scale searches, or cache results.
### Issue: PDB parser warnings
**Solution:** Use `PDBParser(QUIET=True)` to suppress warnings, or investigate structure quality.
### Issue: ImportError for Bio.HMM, Bio.MarkovModel, or Bio.Application
**Solution:** These modules were removed in Biopython 1.86. Use [hmmlearn](https://pypi.org/project/hmmlearn/) for HMMs and the standard library `subprocess` module instead of `Bio.Application` CLI wrappers.
### Issue: PairwiseAligner returns fewer alignments after upgrading to 1.86+
**Solution:** The default gap score changed from 0 to -1 in 1.86, eliminating trivial tie alignments. Set `aligner.gap_score = 0` to restore the old behavior if needed (see `references/alignment.md`).
## Additional Resources
- **Official Documentation**: https://biopython.org/docs/latest/
- **Tutorial**: https://biopython.org/docs/latest/Tutorial/
- **Cookbook**: https://biopython.org/docs/latest/Tutorial/ (advanced examples)
- **GitHub**: https://github.com/biopython/biopython
- **Release notes**: https://github.com/biopython/biopython/blob/master/NEWS.rst
- **Deprecated APIs**: https://github.com/biopython/biopython/blob/master/DEPRECATED.rst
- **Mailing List**: biopython@biopython.org
## Quick Reference
To locate information in reference files, use these search patterns:
```bash
# Search for specific functions
rg -n "function_name" references/*.md
# Find examples of specific tasks
rg -n "example" references/sequence_io.md
# Find all occurrences of a module
rg -n "Bio.Seq" references/*.md
```
## Summary
Biopython provides comprehensive tools for computational molecular biology. When using this skill:
1. **Identify the task domain** (sequences, alignments, databases, BLAST, structures, phylogenetics, or advanced)
2. **Consult the appropriate reference file** in the `references/` directory
3. **Adapt code examples** to the specific use case
4. **Combine multiple modules** when needed for complex workflows
5. **Follow best practices** for file handling, error checking, and data management
The modular reference documentation ensures detailed, searchable information for every major Biopython capability.
skills/biopython/references/advanced.md
+580
View File
@@ -0,0 +1,580 @@
# Advanced Biopython Features
## Sequence Motifs with Bio.motifs
### Creating Motifs
```python
from Bio import motifs
from Bio.Seq import Seq
# Create motif from instances
instances = [
Seq("TACAA"),
Seq("TACGC"),
Seq("TACAC"),
Seq("TACCC"),
Seq("AACCC"),
Seq("AATGC"),
Seq("AATGC"),
]
motif = motifs.create(instances)
```
### Motif Consensus and Degenerate Sequences
```python
# Get consensus sequence
print(motif.counts.consensus)
# Get degenerate consensus (IUPAC ambiguity codes)
print(motif.counts.degenerate_consensus)
# Access counts matrix
print(motif.counts)
```
### Position Weight Matrix (PWM)
```python
# Create position weight matrix
pwm = motif.counts.normalize(pseudocounts=0.5)
print(pwm)
# Calculate information content
ic = sum(motif.relative_entropy)
print(f"Information content: {ic:.2f} bits")
```
### Searching for Motifs
```python
from Bio.Seq import Seq
# Search sequence for motif
test_seq = Seq("ATACAGGACAGACATACGCATACAACATTACAC")
# Get Position Specific Scoring Matrix (PSSM)
pssm = pwm.log_odds()
# Search sequence
for position, score in pssm.search(test_seq, threshold=5.0):
print(f"Position {position}: score = {score:.2f}")
```
### Reading Motifs from Files
```python
# Read motif from JASPAR format
with open("motif.jaspar") as handle:
motif = motifs.read(handle, "jaspar")
# Read multiple motifs
with open("motifs.jaspar") as handle:
for m in motifs.parse(handle, "jaspar"):
print(m.name)
# Supported formats: jaspar, meme, transfac, pfm
```
### Writing Motifs
```python
# Write motif in JASPAR format
with open("output.jaspar", "w") as handle:
handle.write(motif.format("jaspar"))
```
## Population Genetics with Bio.PopGen
### Working with GenePop Files
```python
from Bio.PopGen import GenePop
# Read GenePop file
with open("data.gen") as handle:
record = GenePop.read(handle)
# Access populations
print(f"Number of populations: {len(record.populations)}")
print(f"Loci: {record.loci_list}")
# Iterate through populations
for pop_idx, pop in enumerate(record.populations):
print(f"\nPopulation {pop_idx + 1}:")
for individual in pop:
print(f" {individual[0]}: {individual[1]}")
```
### Calculating Population Statistics
```python
from Bio.PopGen.GenePop.Controller import GenePopController
# Create controller
ctrl = GenePopController()
# Calculate basic statistics
result = ctrl.calc_allele_genotype_freqs("data.gen")
# Calculate Fst
fst_result = ctrl.calc_fst_all("data.gen")
print(f"Fst: {fst_result}")
# Test Hardy-Weinberg equilibrium
hw_result = ctrl.test_hw_pop("data.gen", "probability")
```
## Sequence Utilities with Bio.SeqUtils
### GC Content
```python
from Bio.SeqUtils import gc_fraction
from Bio.Seq import Seq
seq = Seq("ATCGATCGATCG")
gc = gc_fraction(seq)
print(f"GC content: {gc:.2%}")
```
### Molecular Weight
```python
from Bio.SeqUtils import molecular_weight
# DNA molecular weight
dna_seq = Seq("ATCG")
mw = molecular_weight(dna_seq, seq_type="DNA")
print(f"DNA MW: {mw:.2f} g/mol")
# Protein molecular weight
protein_seq = Seq("ACDEFGHIKLMNPQRSTVWY")
mw = molecular_weight(protein_seq, seq_type="protein")
print(f"Protein MW: {mw:.2f} Da")
```
### Melting Temperature
```python
from Bio.SeqUtils import MeltingTemp as mt
# Calculate Tm using nearest-neighbor method
seq = Seq("ATCGATCGATCG")
tm = mt.Tm_NN(seq)
print(f"Tm: {tm:.1f}°C")
# Use different salt concentration
tm = mt.Tm_NN(seq, Na=50, Mg=1.5) # 50 mM Na+, 1.5 mM Mg2+
# Wallace rule (for primers)
tm_wallace = mt.Tm_Wallace(seq)
```
### GC Skew
```python
from Bio.SeqUtils import gc_skew
# Calculate GC skew
seq = Seq("ATCGATCGGGCCCAAATTT")
skew = gc_skew(seq, window=100)
print(f"GC skew: {skew}")
```
### ProtParam - Protein Analysis
```python
from Bio.SeqUtils.ProtParam import ProteinAnalysis
protein_seq = "ACDEFGHIKLMNPQRSTVWY"
analyzed_seq = ProteinAnalysis(protein_seq)
# Molecular weight
print(f"MW: {analyzed_seq.molecular_weight():.2f} Da")
# Isoelectric point
print(f"pI: {analyzed_seq.isoelectric_point():.2f}")
# Amino acid composition
print(f"Composition: {analyzed_seq.get_amino_acids_percent()}")
# Instability index
print(f"Instability: {analyzed_seq.instability_index():.2f}")
# Aromaticity
print(f"Aromaticity: {analyzed_seq.aromaticity():.2f}")
# Secondary structure fraction
ss = analyzed_seq.secondary_structure_fraction()
print(f"Helix: {ss[0]:.2%}, Turn: {ss[1]:.2%}, Sheet: {ss[2]:.2%}")
# Extinction coefficient (assumes Cys reduced, no disulfide bonds)
print(f"Extinction coefficient: {analyzed_seq.molar_extinction_coefficient()}")
# Gravy (grand average of hydropathy)
print(f"GRAVY: {analyzed_seq.gravy():.3f}")
```
## Restriction Analysis with Bio.Restriction
```python
from Bio import Restriction
from Bio.Seq import Seq
# Analyze sequence for restriction sites
seq = Seq("GAATTCATCGATCGATGAATTC")
# Use specific enzyme
ecori = Restriction.EcoRI
sites = ecori.search(seq)
print(f"EcoRI sites at: {sites}")
# Use multiple enzymes
rb = Restriction.RestrictionBatch(["EcoRI", "BamHI", "PstI"])
results = rb.search(seq)
for enzyme, sites in results.items():
if sites:
print(f"{enzyme}: {sites}")
# Get all enzymes that cut sequence
all_enzymes = Restriction.Analysis(rb, seq)
print(f"Cutting enzymes: {all_enzymes.with_sites()}")
```
## Sequence Translation Tables
```python
from Bio.Data import CodonTable
# Standard genetic code
standard_table = CodonTable.unambiguous_dna_by_id[1]
print(standard_table)
# Mitochondrial code
mito_table = CodonTable.unambiguous_dna_by_id[2]
# Get specific codon
print(f"ATG codes for: {standard_table.forward_table['ATG']}")
# Get stop codons
print(f"Stop codons: {standard_table.stop_codons}")
# Get start codons
print(f"Start codons: {standard_table.start_codons}")
```
## Cluster Analysis with Bio.Cluster
```python
from Bio.Cluster import kcluster
import numpy as np
# Sample data matrix (genes x conditions)
data = np.array([
[1.2, 0.8, 0.5, 1.5],
[0.9, 1.1, 0.7, 1.3],
[0.2, 0.3, 2.1, 2.5],
[0.1, 0.4, 2.3, 2.2],
])
# Perform k-means clustering
clusterid, error, nfound = kcluster(data, nclusters=2)
print(f"Cluster assignments: {clusterid}")
print(f"Error: {error}")
```
## Genome Diagrams with GenomeDiagram
```python
from Bio.Graphics import GenomeDiagram
from Bio.SeqFeature import SeqFeature, FeatureLocation
from Bio import SeqIO
from reportlab.lib import colors
# Read GenBank file
record = SeqIO.read("sequence.gb", "genbank")
# Create diagram
gd_diagram = GenomeDiagram.Diagram("Genome Diagram")
gd_track = gd_diagram.new_track(1, greytrack=True)
gd_feature_set = gd_track.new_set()
# Add features
for feature in record.features:
if feature.type == "CDS":
color = colors.blue
elif feature.type == "gene":
color = colors.lightblue
else:
color = colors.grey
gd_feature_set.add_feature(
feature,
color=color,
label=True,
label_size=6,
label_angle=45
)
# Draw and save
gd_diagram.draw(format="linear", pagesize="A4", fragments=1)
gd_diagram.write("genome_diagram.pdf", "PDF")
```
## Sequence Comparison with PairwiseAligner
`Bio.pairwise2` is deprecated (since Biopython 1.80). Use `Bio.Align.PairwiseAligner` for new pairwise alignment code (see `alignment.md`). For HMM workflows, `Bio.HMM` and `Bio.MarkovModel` were removed in Biopython 1.86; use [hmmlearn](https://pypi.org/project/hmmlearn/) instead.
```python
from Bio.Align import PairwiseAligner
aligner = PairwiseAligner()
aligner.mode = "global"
aligner.match_score = 1
aligner.mismatch_score = 0
aligner.gap_score = 0 # Optional: mimic the old globalxx scoring style
alignments = aligner.align("ACCGT", "ACGT")
for alignment in alignments[:3]:
print(alignment)
print(f"Score: {alignment.score}")
```
Avoid importing `Bio.pairwise2` in new code. It remains a legacy migration concern only.
## Working with PubChem
```python
from Bio import Entrez
Entrez.email = "your.email@example.com"
# Search PubChem
handle = Entrez.esearch(db="pccompound", term="aspirin")
result = Entrez.read(handle)
handle.close()
compound_id = result["IdList"][0]
# Get compound information
handle = Entrez.efetch(db="pccompound", id=compound_id, retmode="xml")
compound_data = handle.read()
handle.close()
```
## Sequence Features with Bio.SeqFeature
```python
from Bio.SeqFeature import SeqFeature, FeatureLocation
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
# Create a feature
feature = SeqFeature(
location=FeatureLocation(start=10, end=50),
type="CDS",
strand=1,
qualifiers={"gene": ["ABC1"], "product": ["ABC protein"]}
)
# Add feature to record
record = SeqRecord(Seq("ATCG" * 20), id="seq1")
record.features.append(feature)
# Extract feature sequence
feature_seq = feature.extract(record.seq)
print(feature_seq)
```
## Sequence Ambiguity
```python
from Bio.Data import IUPACData
# DNA ambiguity codes
print(IUPACData.ambiguous_dna_letters)
# Protein ambiguity codes
print(IUPACData.ambiguous_protein_letters)
# Resolve ambiguous bases
print(IUPACData.ambiguous_dna_values["N"]) # Any base
print(IUPACData.ambiguous_dna_values["R"]) # A or G
```
## Quality Scores (FASTQ)
```python
from Bio import SeqIO
# Read FASTQ with quality scores
for record in SeqIO.parse("reads.fastq", "fastq"):
print(f"ID: {record.id}")
print(f"Sequence: {record.seq}")
print(f"Quality: {record.letter_annotations['phred_quality']}")
# Calculate average quality
avg_quality = sum(record.letter_annotations['phred_quality']) / len(record)
print(f"Average quality: {avg_quality:.2f}")
# Filter by quality
min_quality = min(record.letter_annotations['phred_quality'])
if min_quality >= 20:
print("High quality read")
```
## Best Practices
1. **Use appropriate modules** - Choose the right tool for your analysis
2. **Handle pseudocounts** - Important for motif analysis
3. **Validate input data** - Check file formats and data quality
4. **Consider performance** - Some operations can be computationally intensive
5. **Cache results** - Store intermediate results for large analyses
6. **Use proper genetic codes** - Select appropriate translation tables
7. **Document parameters** - Record thresholds and settings used
8. **Validate statistical results** - Understand limitations of tests
9. **Handle edge cases** - Check for empty results or invalid input
10. **Combine modules** - Leverage multiple Biopython tools together
## Common Use Cases
### Find ORFs
```python
from Bio import SeqIO
from Bio.SeqUtils import gc_fraction
def find_orfs(seq, min_length=100):
"""Find all ORFs in sequence."""
orfs = []
for strand, nuc in [(+1, seq), (-1, seq.reverse_complement())]:
for frame in range(3):
trans = nuc[frame:].translate()
trans_len = len(trans)
aa_start = 0
while aa_start < trans_len:
aa_end = trans.find("*", aa_start)
if aa_end == -1:
aa_end = trans_len
if aa_end - aa_start >= min_length // 3:
start = frame + aa_start * 3
end = frame + aa_end * 3
orfs.append({
'start': start,
'end': end,
'strand': strand,
'frame': frame,
'length': end - start,
'sequence': nuc[start:end]
})
aa_start = aa_end + 1
return orfs
# Use it
record = SeqIO.read("sequence.fasta", "fasta")
orfs = find_orfs(record.seq, min_length=300)
for orf in orfs:
print(f"ORF: {orf['start']}-{orf['end']}, strand={orf['strand']}, length={orf['length']}")
```
### Analyze Codon Usage
```python
from Bio import SeqIO
def analyze_codon_usage(fasta_file):
"""Analyze codon usage in coding sequences."""
codon_counts = {}
for record in SeqIO.parse(fasta_file, "fasta"):
# Ensure sequence is multiple of 3
seq = record.seq[:len(record.seq) - len(record.seq) % 3]
# Count codons
for i in range(0, len(seq), 3):
codon = str(seq[i:i+3])
codon_counts[codon] = codon_counts.get(codon, 0) + 1
# Calculate frequencies
total = sum(codon_counts.values())
codon_freq = {k: v/total for k, v in codon_counts.items()}
return codon_freq
```
### Calculate Sequence Complexity
```python
def sequence_complexity(seq, k=2):
"""Calculate k-mer complexity (Shannon entropy)."""
import math
from collections import Counter
# Generate k-mers
kmers = [str(seq[i:i+k]) for i in range(len(seq) - k + 1)]
# Count k-mers
counts = Counter(kmers)
total = len(kmers)
# Calculate entropy
entropy = 0
for count in counts.values():
freq = count / total
entropy -= freq * math.log2(freq)
# Normalize by maximum possible entropy
max_entropy = math.log2(4 ** k) # For DNA
return entropy / max_entropy if max_entropy > 0 else 0
# Use it
from Bio.Seq import Seq
seq = Seq("ATCGATCGATCGATCG")
complexity = sequence_complexity(seq, k=2)
print(f"Sequence complexity: {complexity:.3f}")
```
### Extract Promoter Regions
```python
def extract_promoters(genbank_file, upstream=500):
"""Extract promoter regions upstream of genes."""
from Bio import SeqIO
record = SeqIO.read(genbank_file, "genbank")
promoters = []
for feature in record.features:
if feature.type == "gene":
if feature.strand == 1:
# Forward strand
start = max(0, feature.location.start - upstream)
end = feature.location.start
else:
# Reverse strand
start = feature.location.end
end = min(len(record.seq), feature.location.end + upstream)
promoter_seq = record.seq[start:end]
if feature.strand == -1:
promoter_seq = promoter_seq.reverse_complement()
promoters.append({
'gene': feature.qualifiers.get('gene', ['Unknown'])[0],
'sequence': promoter_seq,
'start': start,
'end': end
})
return promoters
```
skills/biopython/references/alignment.md
+377
View File
@@ -0,0 +1,377 @@
# Sequence Alignments with Bio.Align and Bio.AlignIO
## Overview
Bio.Align provides tools for pairwise sequence alignment using various algorithms, while Bio.AlignIO handles reading and writing multiple sequence alignment files in various formats.
## Pairwise Alignment with Bio.Align
### The PairwiseAligner Class
The `PairwiseAligner` class performs pairwise sequence alignments using Needleman-Wunsch (global), Smith-Waterman (local), Gotoh (three-state), and Waterman-Smith-Beyer algorithms. The appropriate algorithm is automatically selected based on gap score parameters.
### Creating an Aligner
```python
from Bio import Align
# Create aligner with default parameters
aligner = Align.PairwiseAligner()
# Default scores (as of Biopython 1.86+):
# - Match score: +1.0
# - Mismatch score: 0.0
# - All gap scores: -1.0 (changed from 0 in 1.86 to avoid trivial tie alignments)
```
**Note (1.86+):** The default gap score changed from 0 to -1. Previously, mismatches and gap combinations could score 0, producing many logically equivalent alignments. To restore pre-1.86 behavior:
```python
aligner.gap_score = 0
```
### Customizing Alignment Parameters
```python
# Set scoring parameters
aligner.match_score = 2.0
aligner.mismatch_score = -1.0
aligner.gap_score = -0.5
# Or use separate gap opening/extension penalties
aligner.open_gap_score = -2.0
aligner.extend_gap_score = -0.5
# Set internal gap scores separately
aligner.internal_open_gap_score = -2.0
aligner.internal_extend_gap_score = -0.5
# Set end gap scores (for semi-global alignment)
aligner.left_open_gap_score = 0.0
aligner.left_extend_gap_score = 0.0
aligner.right_open_gap_score = 0.0
aligner.right_extend_gap_score = 0.0
```
### Alignment Modes
```python
# Global alignment (default)
aligner.mode = 'global'
# Local alignment
aligner.mode = 'local'
```
### Performing Alignments
```python
from Bio.Seq import Seq
seq1 = Seq("ACCGGT")
seq2 = Seq("ACGGT")
# Get all optimal alignments
alignments = aligner.align(seq1, seq2)
# Iterate through alignments
for alignment in alignments:
print(alignment)
print(f"Score: {alignment.score}")
# Get just the score
score = aligner.score(seq1, seq2)
```
### Using Substitution Matrices
```python
from Bio.Align import substitution_matrices
# Load a substitution matrix
matrix = substitution_matrices.load("BLOSUM62")
aligner.substitution_matrix = matrix
# Align protein sequences
protein1 = Seq("KEVLA")
protein2 = Seq("KSVLA")
alignments = aligner.align(protein1, protein2)
```
### Available Substitution Matrices
Common matrices include:
- **BLOSUM** series (BLOSUM45, BLOSUM50, BLOSUM62, BLOSUM80, BLOSUM90)
- **PAM** series (PAM30, PAM70, PAM250)
- **MATCH** - Simple match/mismatch matrix
```python
# List available matrices
available = substitution_matrices.load()
print(available)
```
## Multiple Sequence Alignments with Bio.AlignIO
### Reading Alignments
Bio.AlignIO provides similar API to Bio.SeqIO but for alignment files:
```python
from Bio import AlignIO
# Read a single alignment
alignment = AlignIO.read("alignment.aln", "clustal")
# Parse multiple alignments from a file
for alignment in AlignIO.parse("alignments.aln", "clustal"):
print(f"Alignment with {len(alignment)} sequences")
print(f"Alignment length: {alignment.get_alignment_length()}")
```
### Supported Alignment Formats
Common formats include:
- **clustal** - Clustal format
- **phylip** - PHYLIP format
- **phylip-relaxed** - Relaxed PHYLIP (longer names)
- **stockholm** - Stockholm format
- **fasta** - FASTA format (aligned)
- **nexus** - NEXUS format
- **emboss** - EMBOSS alignment format
- **msf** - MSF format
- **maf** - Multiple Alignment Format
### Writing Alignments
```python
# Write alignment to file
AlignIO.write(alignment, "output.aln", "clustal")
# Convert between formats
count = AlignIO.convert("input.aln", "clustal", "output.phy", "phylip")
```
### Working with Alignment Objects
```python
from Bio import AlignIO
alignment = AlignIO.read("alignment.aln", "clustal")
# Get alignment properties
print(f"Number of sequences: {len(alignment)}")
print(f"Alignment length: {alignment.get_alignment_length()}")
# Access individual sequences
for record in alignment:
print(f"{record.id}: {record.seq}")
# Get alignment column
column = alignment[:, 0] # First column
# Get alignment slice
sub_alignment = alignment[:, 10:20] # Positions 10-20
# Get specific sequence
seq_record = alignment[0] # First sequence
```
### Alignment Analysis
```python
# Calculate alignment statistics with current Biopython APIs.
# Avoid Bio.AlignInfo.SummaryInfo: it is deprecated in 1.86 and several
# methods were removed in 1.85/1.86.
from Bio import AlignIO
from Bio.motifs import Motif
msa = AlignIO.read("alignment.aln", "clustal")
alignment = msa.alignment # New-style Bio.Align.Alignment
# Build a motif from a DNA alignment to inspect per-column counts
motif = Motif("ACGT", alignment)
counts = motif.counts
consensus = counts.consensus
# Information content replacement for deprecated SummaryInfo methods
information_content = sum(motif.relative_entropy)
# Replacement dictionary from the new-style Alignment object
substitutions = alignment.substitutions
```
## Creating Alignments Programmatically
### From SeqRecord Objects
```python
from Bio.Align import MultipleSeqAlignment
from Bio.SeqRecord import SeqRecord
from Bio.Seq import Seq
# Create records
records = [
SeqRecord(Seq("ACTGCTAGCTAG"), id="seq1"),
SeqRecord(Seq("ACT-CTAGCTAG"), id="seq2"),
SeqRecord(Seq("ACTGCTA-CTAG"), id="seq3"),
]
# Create alignment
alignment = MultipleSeqAlignment(records)
```
### Adding Sequences to Alignments
```python
# Start with empty alignment
alignment = MultipleSeqAlignment([])
# Add sequences (must have same length)
alignment.append(SeqRecord(Seq("ACTG"), id="seq1"))
alignment.append(SeqRecord(Seq("ACTG"), id="seq2"))
# Extend with another alignment
alignment.extend(other_alignment)
```
## Advanced Alignment Operations
### Removing Gaps
```python
# Remove all gap-only columns
no_gaps = []
for i in range(alignment.get_alignment_length()):
column = alignment[:, i]
if set(column) != {'-'}: # Not all gaps
no_gaps.append(column)
```
### Alignment Sorting
```python
# Sort by sequence ID
sorted_alignment = sorted(alignment, key=lambda x: x.id)
alignment = MultipleSeqAlignment(sorted_alignment)
```
### Computing Pairwise Identities
```python
def pairwise_identity(seq1, seq2):
"""Calculate percent identity between two sequences."""
matches = sum(a == b for a, b in zip(seq1, seq2) if a != '-' and b != '-')
length = sum(1 for a, b in zip(seq1, seq2) if a != '-' and b != '-')
return matches / length if length > 0 else 0
# Calculate all pairwise identities
for i, record1 in enumerate(alignment):
for record2 in alignment[i+1:]:
identity = pairwise_identity(record1.seq, record2.seq)
print(f"{record1.id} vs {record2.id}: {identity:.2%}")
```
## Running External Alignment Tools
Biopython 1.86 removed `Bio.Application` and all command-line wrapper modules, including `Bio.Align.Applications`. Use Python's standard `subprocess` module with argument lists. Keep executable names and flags explicit, and do not construct command arguments from unsanitized user input.
### Clustal Omega (via subprocess)
```python
import subprocess
from Bio import AlignIO
cmd = [
"clustalo",
"-i", "sequences.fasta",
"-o", "alignment.aln",
"--outfmt", "clu",
"--force",
"--auto",
]
subprocess.run(cmd, check=True)
# Read result
alignment = AlignIO.read("alignment.aln", "clustal")
```
### MUSCLE (via subprocess)
```python
import subprocess
from Bio import AlignIO
cmd = [
"muscle",
"-align", "sequences.fasta",
"-output", "alignment.fasta",
]
subprocess.run(cmd, check=True)
alignment = AlignIO.read("alignment.fasta", "fasta")
```
## Best Practices
1. **Choose appropriate scoring schemes** - Use BLOSUM62 for proteins, custom scores for DNA
2. **Consider alignment mode** - Global for similar-length sequences, local for finding conserved regions
3. **Set gap penalties carefully** - Higher penalties create fewer, longer gaps
4. **Use appropriate formats** - FASTA for simple alignments, Stockholm for rich annotation
5. **Validate alignment quality** - Check for conserved regions and percent identity
6. **Handle large alignments carefully** - Use slicing and iteration for memory efficiency
7. **Preserve metadata** - Maintain SeqRecord IDs and annotations through alignment operations
## Common Use Cases
### Find Best Local Alignment
```python
from Bio.Align import PairwiseAligner
from Bio.Seq import Seq
aligner = PairwiseAligner()
aligner.mode = 'local'
aligner.match_score = 2
aligner.mismatch_score = -1
seq1 = Seq("AGCTTAGCTAGCTAGC")
seq2 = Seq("CTAGCTAGC")
alignments = aligner.align(seq1, seq2)
print(alignments[0])
```
### Protein Sequence Alignment
```python
from Bio.Align import PairwiseAligner, substitution_matrices
aligner = PairwiseAligner()
aligner.substitution_matrix = substitution_matrices.load("BLOSUM62")
aligner.open_gap_score = -10
aligner.extend_gap_score = -0.5
protein1 = Seq("KEVLA")
protein2 = Seq("KEVLAEQP")
alignments = aligner.align(protein1, protein2)
```
### Extract Conserved Regions
```python
from Bio import AlignIO
alignment = AlignIO.read("alignment.aln", "clustal")
# Find columns with >80% identity
conserved_positions = []
for i in range(alignment.get_alignment_length()):
column = alignment[:, i]
most_common = max(set(column), key=column.count)
if column.count(most_common) / len(column) > 0.8:
conserved_positions.append(i)
print(f"Conserved positions: {conserved_positions}")
```
skills/biopython/references/blast.md
+463
View File
@@ -0,0 +1,463 @@
# BLAST Operations with Bio.Blast
## Overview
Bio.Blast provides tools for running BLAST searches (both locally and via NCBI web services) and parsing BLAST results in various formats. The module handles the complexity of submitting queries and parsing outputs.
## Running BLAST via NCBI Web Services
### Bio.Blast.NCBIWWW
The `qblast()` function submits sequences to NCBI's online BLAST service:
```python
from Bio.Blast import NCBIWWW
from Bio import SeqIO
# Read sequence from file
record = SeqIO.read("sequence.fasta", "fasta")
# Run BLAST search
result_handle = NCBIWWW.qblast(
program="blastn", # BLAST program
database="nt", # Database to search
sequence=str(record.seq) # Query sequence
)
# Save results
with open("blast_results.xml", "w") as out_file:
out_file.write(result_handle.read())
result_handle.close()
```
### BLAST Programs Available
- **blastn** - Nucleotide vs nucleotide
- **blastp** - Protein vs protein
- **blastx** - Translated nucleotide vs protein
- **tblastn** - Protein vs translated nucleotide
- **tblastx** - Translated nucleotide vs translated nucleotide
### Common Databases
**Nucleotide databases:**
- `nt` - All GenBank+EMBL+DDBJ+PDB sequences
- `refseq_rna` - RefSeq RNA sequences
**Protein databases:**
- `nr` - All non-redundant GenBank CDS translations
- `refseq_protein` - RefSeq protein sequences
- `pdb` - Protein Data Bank sequences
- `swissprot` - Curated UniProtKB/Swiss-Prot
### Advanced qblast Parameters
```python
result_handle = NCBIWWW.qblast(
program="blastn",
database="nt",
sequence=str(record.seq),
expect=0.001, # E-value threshold
hitlist_size=50, # Number of hits to return
alignments=25, # Number of alignments to show
word_size=11, # Word size for initial match
gapcosts="5 2", # Gap costs (open extend)
format_type="XML" # Output format (default)
)
```
### Using Sequence Files or IDs
```python
# Use FASTA format string
fasta_string = open("sequence.fasta").read()
result_handle = NCBIWWW.qblast("blastn", "nt", fasta_string)
# Use GenBank ID
result_handle = NCBIWWW.qblast("blastn", "nt", "EU490707")
# Use GI number
result_handle = NCBIWWW.qblast("blastn", "nt", "160418")
```
## Parsing BLAST Results
### Bio.Blast.NCBIXML
NCBIXML provides parsers for BLAST XML output (the recommended format):
```python
from Bio.Blast import NCBIXML
# Parse single BLAST result
with open("blast_results.xml") as result_handle:
blast_record = NCBIXML.read(result_handle)
```
### Accessing BLAST Record Data
```python
# Query information
print(f"Query: {blast_record.query}")
print(f"Query length: {blast_record.query_length}")
print(f"Database: {blast_record.database}")
print(f"Number of sequences in database: {blast_record.database_sequences}")
# Iterate through alignments (hits)
for alignment in blast_record.alignments:
print(f"\nHit: {alignment.title}")
print(f"Length: {alignment.length}")
print(f"Accession: {alignment.accession}")
# Each alignment can have multiple HSPs (high-scoring pairs)
for hsp in alignment.hsps:
print(f" E-value: {hsp.expect}")
print(f" Score: {hsp.score}")
print(f" Bits: {hsp.bits}")
print(f" Identities: {hsp.identities}/{hsp.align_length}")
print(f" Gaps: {hsp.gaps}")
print(f" Query: {hsp.query}")
print(f" Match: {hsp.match}")
print(f" Subject: {hsp.sbjct}")
```
### Filtering Results
```python
# Only show hits with E-value < 0.001
E_VALUE_THRESH = 0.001
for alignment in blast_record.alignments:
for hsp in alignment.hsps:
if hsp.expect < E_VALUE_THRESH:
print(f"Hit: {alignment.title}")
print(f"E-value: {hsp.expect}")
print(f"Identities: {hsp.identities}/{hsp.align_length}")
print()
```
### Multiple BLAST Results
For files containing multiple BLAST results (e.g., from batch searches):
```python
from Bio.Blast import NCBIXML
with open("batch_blast_results.xml") as result_handle:
blast_records = NCBIXML.parse(result_handle)
for blast_record in blast_records:
print(f"\nQuery: {blast_record.query}")
print(f"Hits: {len(blast_record.alignments)}")
if blast_record.alignments:
# Get best hit
best_alignment = blast_record.alignments[0]
best_hsp = best_alignment.hsps[0]
print(f"Best hit: {best_alignment.title}")
print(f"E-value: {best_hsp.expect}")
```
## Running Local BLAST
### Prerequisites
Local BLAST requires:
1. BLAST+ command-line tools installed
2. BLAST databases downloaded locally
### Running BLAST+ via subprocess
Biopython 1.86 removed `Bio.Application` and the BLAST command-line wrappers in `Bio.Blast.Applications`. Use the standard library `subprocess` module with explicit argument lists. Keep command names and flags fixed, validate file paths before running, and do not interpolate unsanitized user input into shell commands.
```python
import subprocess
from Bio.Blast import NCBIXML
cmd = [
"blastn",
"-query", "input.fasta",
"-db", "local_database",
"-evalue", "0.001",
"-outfmt", "5", # XML format for NCBIXML
"-out", "results.xml",
]
subprocess.run(cmd, check=True)
# Parse results
with open("results.xml") as result_handle:
blast_record = NCBIXML.read(result_handle)
```
### Common BLAST+ commands
- `blastn` - nucleotide vs nucleotide
- `blastp` - protein vs protein
- `blastx` - translated nucleotide vs protein
- `tblastn` - protein vs translated nucleotide
- `tblastx` - translated nucleotide vs translated nucleotide
- `makeblastdb` - create local BLAST databases
### Creating BLAST Databases
```python
import subprocess
cmd = [
"makeblastdb",
"-in", "sequences.fasta",
"-dbtype", "nucl",
"-out", "my_database",
]
subprocess.run(cmd, check=True)
```
## Analyzing BLAST Results
### Extract Best Hits
```python
def get_best_hits(blast_record, num_hits=10, e_value_thresh=0.001):
"""Extract best hits from BLAST record."""
hits = []
for alignment in blast_record.alignments[:num_hits]:
for hsp in alignment.hsps:
if hsp.expect < e_value_thresh:
hits.append({
'title': alignment.title,
'accession': alignment.accession,
'length': alignment.length,
'e_value': hsp.expect,
'score': hsp.score,
'identities': hsp.identities,
'align_length': hsp.align_length,
'query_start': hsp.query_start,
'query_end': hsp.query_end,
'sbjct_start': hsp.sbjct_start,
'sbjct_end': hsp.sbjct_end
})
break # Only take best HSP per alignment
return hits
```
### Calculate Percent Identity
```python
def calculate_percent_identity(hsp):
"""Calculate percent identity for an HSP."""
return (hsp.identities / hsp.align_length) * 100
# Use it
for alignment in blast_record.alignments:
for hsp in alignment.hsps:
if hsp.expect < 0.001:
identity = calculate_percent_identity(hsp)
print(f"{alignment.title}: {identity:.2f}% identity")
```
### Extract Hit Sequences
```python
from Bio import Entrez, SeqIO
Entrez.email = "your.email@example.com"
def fetch_hit_sequences(blast_record, num_sequences=5):
"""Fetch sequences for top BLAST hits."""
sequences = []
for alignment in blast_record.alignments[:num_sequences]:
accession = alignment.accession
# Fetch sequence from GenBank
handle = Entrez.efetch(
db="nucleotide",
id=accession,
rettype="fasta",
retmode="text"
)
record = SeqIO.read(handle, "fasta")
handle.close()
sequences.append(record)
return sequences
```
## Parsing Other BLAST Formats
### Tab-Delimited Output (outfmt 6/7)
```python
import subprocess
cmd = [
"blastn",
"-query", "input.fasta",
"-db", "database",
"-outfmt", "6",
"-out", "results.txt",
]
subprocess.run(cmd, check=True)
# Parse tabular results
with open("results.txt") as f:
for line in f:
fields = line.strip().split('\t')
query_id = fields[0]
subject_id = fields[1]
percent_identity = float(fields[2])
align_length = int(fields[3])
e_value = float(fields[10])
bit_score = float(fields[11])
print(f"{query_id} -> {subject_id}: {percent_identity}% identity, E={e_value}")
```
### Custom Output Formats
```python
import subprocess
# Specify custom columns (outfmt 6 with custom fields)
cmd = [
"blastn",
"-query", "input.fasta",
"-db", "database",
"-outfmt", "6 qseqid sseqid pident length evalue bitscore qseq sseq",
"-out", "results.txt",
]
subprocess.run(cmd, check=True)
```
## Best Practices
1. **Use XML format** for parsing (outfmt 5) - most reliable and complete
2. **Save BLAST results** - Don't re-run searches unnecessarily
3. **Set appropriate E-value thresholds** - Default is 10, but 0.001-0.01 is often better
4. **Handle rate limits** - NCBI limits request frequency
5. **Use local BLAST** for large-scale searches or repeated queries
6. **Cache results** - Save parsed data to avoid re-parsing
7. **Check for empty results** - Handle cases with no hits gracefully
8. **Consider alternatives** - For large datasets, consider DIAMOND or other fast aligners
9. **Batch searches** - Submit multiple sequences together when possible
10. **Filter by identity** - E-value alone may not be sufficient
## Common Use Cases
### Basic BLAST Search and Parse
```python
from Bio.Blast import NCBIWWW, NCBIXML
from Bio import SeqIO
# Read query sequence
record = SeqIO.read("query.fasta", "fasta")
# Run BLAST
print("Running BLAST search...")
result_handle = NCBIWWW.qblast("blastn", "nt", str(record.seq))
# Parse results
blast_record = NCBIXML.read(result_handle)
# Display top 5 hits
print(f"\nTop 5 hits for {blast_record.query}:")
for i, alignment in enumerate(blast_record.alignments[:5], 1):
hsp = alignment.hsps[0]
identity = (hsp.identities / hsp.align_length) * 100
print(f"{i}. {alignment.title}")
print(f" E-value: {hsp.expect}, Identity: {identity:.1f}%")
```
### Find Orthologs
```python
from Bio.Blast import NCBIWWW, NCBIXML
from Bio import Entrez, SeqIO
Entrez.email = "your.email@example.com"
# Query gene sequence
query_record = SeqIO.read("gene.fasta", "fasta")
# BLAST against specific organism
result_handle = NCBIWWW.qblast(
"blastn",
"nt",
str(query_record.seq),
entrez_query="Mus musculus[Organism]" # Restrict to mouse
)
blast_record = NCBIXML.read(result_handle)
# Find best hit
if blast_record.alignments:
best_hit = blast_record.alignments[0]
print(f"Potential ortholog: {best_hit.title}")
print(f"Accession: {best_hit.accession}")
```
### Batch BLAST Multiple Sequences
```python
from Bio.Blast import NCBIWWW, NCBIXML
from Bio import SeqIO
# Read multiple sequences
sequences = list(SeqIO.parse("queries.fasta", "fasta"))
# Create batch results file
with open("batch_results.xml", "w") as out_file:
for seq_record in sequences:
print(f"Searching for {seq_record.id}...")
result_handle = NCBIWWW.qblast("blastn", "nt", str(seq_record.seq))
out_file.write(result_handle.read())
result_handle.close()
# Parse batch results
with open("batch_results.xml") as result_handle:
for blast_record in NCBIXML.parse(result_handle):
print(f"\n{blast_record.query}: {len(blast_record.alignments)} hits")
```
### Reciprocal Best Hits
```python
def reciprocal_best_hit(seq1_id, seq2_id, database="nr", program="blastp"):
"""Check if two sequences are reciprocal best hits."""
from Bio.Blast import NCBIWWW, NCBIXML
from Bio import Entrez
Entrez.email = "your.email@example.com"
# Forward BLAST
result1 = NCBIWWW.qblast(program, database, seq1_id)
record1 = NCBIXML.read(result1)
best_hit1 = record1.alignments[0].accession if record1.alignments else None
# Reverse BLAST
result2 = NCBIWWW.qblast(program, database, seq2_id)
record2 = NCBIXML.read(result2)
best_hit2 = record2.alignments[0].accession if record2.alignments else None
# Check reciprocity
return best_hit1 == seq2_id and best_hit2 == seq1_id
```
## Error Handling
```python
from Bio.Blast import NCBIWWW, NCBIXML
from urllib.error import HTTPError
try:
result_handle = NCBIWWW.qblast("blastn", "nt", "ATCGATCGATCG")
blast_record = NCBIXML.read(result_handle)
result_handle.close()
except HTTPError as e:
print(f"HTTP Error: {e.code}")
except Exception as e:
print(f"Error running BLAST: {e}")
```

Some files were not shown because too many files have changed in this diff Show More