wehub/mvanhorn--last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Security / Dependency review (push) Has been skipped
Details
Scorecard / Scorecard analysis (push) Failing after 0s
Details
Validate / eval (push) Failing after 0s
Details
Security / Dependency audit (push) Failing after 1s
Details
Security / Secret scan (push) Failing after 1s
Details
Validate / tests (push) Failing after 0s
Details
Validate / mcp-tests (push) Failing after 1s
Details
GitHub Actions Security Analysis with zizmor 🌈 / zizmor (push) Failing after 1s
Details
Security / SAST scan (push) Failing after 13m52s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:05:33 +08:00
commit 2860fb5d18
392 changed files with 102831 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/plugins/marketplace.json
+20
View File
@@ -0,0 +1,20 @@
{
"name": "last30days-skill",
"interface": {
"displayName": "last30days"
},
"plugins": [
{
"name": "last30days",
"source": {
"source": "url",
"url": "https://github.com/mvanhorn/last30days-skill.git"
},
"policy": {
"installation": "AVAILABLE",
"authentication": "ON_INSTALL"
},
"category": "Research"
}
]
}
.claude-plugin/marketplace.json
+24
View File
@@ -0,0 +1,24 @@
{
"name": "last30days-skill",
"owner": {
"name": "Matt Van Horn",
"url": "https://github.com/mvanhorn"
},
"metadata": {
"description": "Marketplace hosting the last30days research plugin."
},
"plugins": [
{
"name": "last30days",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"version": "3.13.0",
"author": {
"name": "Matt Van Horn",
"url": "https://github.com/mvanhorn"
},
"source": "./",
"category": "productivity",
"homepage": "https://github.com/mvanhorn/last30days-skill"
}
]
}
.claude-plugin/plugin.json
+30
View File
@@ -0,0 +1,30 @@
{
"name": "last30days",
"version": "3.13.0",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"author": {
"name": "Matt Van Horn",
"email": "mvanhorn@gmail.com",
"url": "https://github.com/mvanhorn"
},
"homepage": "https://github.com/mvanhorn/last30days-skill",
"repository": "https://github.com/mvanhorn/last30days-skill",
"license": "MIT",
"keywords": [
"competitor research",
"research",
"reddit",
"twitter",
"youtube",
"tiktok",
"instagram",
"trends",
"prompts",
"polymarket",
"github",
"perplexity",
"threads",
"pinterest",
"hacker-news"
]
}
.clawhubignore
+71
View File
@@ -0,0 +1,71 @@
# ClawHub/Hermes packaging exclusions for repository-root scans.
# Mirrors .skillignore so non-runtime docs/dev artifacts stay out of the
# public bundle and install-time skill security scan.
# VCS, local envs, caches, and generated outputs
.git/
.venv/
__pycache__/
*.pyc
*.log
*.jsonl
*.mp3
*.jpeg
*.jpg
*.png
*.gif
assets/
skills/last30days/assets/
.DS_Store
.coverage
htmlcov/
dist/
work/
print/
# Repo/dev automation and host-specific package metadata
.github/
.agents/
.claude-plugin/
hooks/
mcp/
gemini-extension.json
greptile.json
pyproject.toml
# Non-runtime docs, plans, release notes, fixtures, and tests
docs/
fixtures/
tests/
plans/
agents/
variants/
media/
README.md
CHANGELOG.md
AGENTS.md
CLAUDE.md
CONCEPTS.md
CONFIGURATION.md
CONTRIBUTORS.md
HERMES_SETUP.md
release-notes.md
SKILL-original.md
SPEC.md
TASKS.md
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
skills/last30days/scripts/build-skill.sh
skills/last30days/scripts/compare.sh
skills/last30days/scripts/evaluate_search_quality.py
skills/last30days/scripts/setup-keychain.sh
skills/last30days/scripts/setup-pass.sh
skills/last30days/scripts/test_device_auth.py
skills/last30days/scripts/test-v1-vs-v2.sh
skills/last30days/scripts/verify_v3.py
# Keep visible: optional runtime watchlist/store/briefing feature scripts
# (`watchlist.py`, `store.py`, and `briefing.py`).
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
skills/last30days/scripts/lib/vendor/
.codex-plugin/plugin.json
+50
View File
@@ -0,0 +1,50 @@
{
"name": "last30days",
"version": "3.13.0",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
"author": {
"name": "Matt Van Horn",
"email": "mvanhorn@gmail.com",
"url": "https://github.com/mvanhorn"
},
"homepage": "https://github.com/mvanhorn/last30days-skill",
"repository": "https://github.com/mvanhorn/last30days-skill",
"license": "MIT",
"keywords": [
"competitor research",
"research",
"reddit",
"twitter",
"youtube",
"tiktok",
"instagram",
"trends",
"prompts",
"polymarket",
"github",
"perplexity",
"threads",
"pinterest",
"hacker-news"
],
"skills": "./skills/",
"interface": {
"displayName": "last30days",
"shortDescription": "Research what people are saying about a topic now.",
"longDescription": "last30days adds a Codex skill for researching any topic based on recent discussion and engagement signals across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
"developerName": "Matt Van Horn",
"category": "Research",
"capabilities": [
"Interactive",
"Read",
"Write"
],
"websiteURL": "https://github.com/mvanhorn/last30days-skill",
"defaultPrompt": [
"TikTok shop trends",
"Codex vs Cursor",
"best travel credit cards"
],
"brandColor": "#6F42C1"
}
}
.gitattributes
+43
View File
@@ -0,0 +1,43 @@
# Exclude non-runtime files from `git archive` output.
# Used by skills/last30days/scripts/build-skill.sh to produce a
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
# Anthropic canonical skill-packaging excludes
# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py)
__pycache__/ export-ignore
node_modules/ export-ignore
*.pyc export-ignore
.DS_Store export-ignore
evals/ export-ignore
# Dev, docs, test, and media - not needed at skill runtime
tests/ export-ignore
docs/ export-ignore
fixtures/ export-ignore
assets/ export-ignore
# NOTE: skills/ and .claude-plugin/ are NOT export-ignored here because
# Claude Code's /plugin install fetches this same git archive tarball.
# Removing those from the archive (as v3.0.1 did) silently breaks installs.
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
# Historical + repo-only manifests
SPEC.md export-ignore
TASKS.md export-ignore
CONTRIBUTORS.md export-ignore
HERMES_SETUP.md export-ignore
CHANGELOG.md export-ignore
uv.lock export-ignore
# Platform adapters are kept in git archives because Claude Code and Codex
# plugin installs use the same repository archive as their source payload.
.hermes-plugin/ export-ignore
# CI workflows - repo-only, not needed at skill runtime
.github/ export-ignore
# Build config itself
.clawhubignore export-ignore
.gitignore export-ignore
.gitattributes export-ignore
.github/ISSUE_TEMPLATE/bug_report.yml
+53
View File
@@ -0,0 +1,53 @@
name: Bug Report
description: Report a bug or unexpected behavior
labels: [bug]
body:
- type: textarea
id: summary
attributes:
label: Summary
description: What happened?
placeholder: Describe the bug in 1-2 sentences.
validations:
required: true
- type: textarea
id: repro
attributes:
label: Steps to Reproduce
description: How can we reproduce this?
placeholder: |
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
2. ...
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected Behavior
description: What should have happened?
validations:
required: true
- type: textarea
id: traceback
attributes:
label: Error / Traceback
description: Paste the full traceback or error output.
render: text
- type: dropdown
id: install
attributes:
label: Install Method
options:
- Claude Code plugin
- Gemini CLI extension
- Codex plugin
- Hermes skill
- Manual (git clone)
- Other
validations:
required: true
- type: input
id: os
attributes:
label: OS
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
.github/ISSUE_TEMPLATE/feature_request.yml
+24
View File
@@ -0,0 +1,24 @@
name: Feature Request
description: Suggest a new feature or improvement
labels: [enhancement]
body:
- type: textarea
id: problem
attributes:
label: Problem
description: What problem does this solve?
placeholder: When I try to ..., I can't ...
validations:
required: true
- type: textarea
id: solution
attributes:
label: Proposed Solution
description: How should this work?
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives Considered
description: Other approaches you thought of (optional).
.github/PULL_REQUEST_TEMPLATE.md
+19
View File
@@ -0,0 +1,19 @@
## Summary
<!-- What does this PR do? 1-3 sentences. -->
## Changes
<!-- Bullet list of what changed. Reference files if helpful. -->
-
## Testing
<!-- How did you verify this works? -->
- [ ] Ran `uv run python -m pytest -q --tb=short`
## Related Issues
<!-- Link issues: Fixes #123 or Relates to #456 -->
.github/copilot-instructions.md
+37
View File
@@ -0,0 +1,37 @@
This file contains Copilot-specific additions. See AGENTS.md for the shared cross-tool governance layer.
# Copilot-specific guidance
## Test generation
- Prefer unittest.TestCase for generated tests to match the existing test suite.
- Mock external calls with unittest.mock.patch.
## Pull request reminders
Before suggesting a pull request:
- Confirm that pytest passes.
- If changes were made anywhere under skills/last30days/, confirm the install copy has been refreshed with:
npx skills add . -g -y
## Vendor exclusion zone
- Never suggest changes to skills/last30days/scripts/lib/vendor/.
- Treat skills/last30days/scripts/lib/vendor/ as a no-touch zone.
## CI expectations
GitHub CI runs:
- pytest
- ruff
Generated changes should pass both before review is requested.
## CLI examples
When suggesting CLI usage examples for safe local testing, default to:
--emit=compact --mock
.github/dependabot.yml
+22
View File
@@ -0,0 +1,22 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
cooldown:
default-days: 7
- package-ecosystem: uv
directory: /
schedule:
interval: weekly
cooldown:
default-days: 7
- package-ecosystem: gomod
directory: /mcp
schedule:
interval: weekly
cooldown:
default-days: 7
.github/workflows/osv-scanner.yml
+31
View File
@@ -0,0 +1,31 @@
name: OSV-Scanner
# Scheduled OSV-Scanner workflow for vulnerability drift detection.
# Scans the repository lockfiles (uv.lock, mcp/go.sum) on a weekly schedule
# and uploads results to GitHub code scanning, so newly disclosed CVEs in
# the dependency tree are visible even between PRs.
#
# Advisory-first: fail-on-vuln is false until maintainers confirm a clean
# baseline, matching the pattern in security.yml.
#
# Separate from the pip-audit job in security.yml (which runs on every PR
# and push) and from the dependency-review gate (which blocks on new
# vulnerable deps at PR time). This workflow fills the scheduled-drift gap.
on:
schedule:
# Weekly, Mondays at 12:30 UTC.
- cron: "30 12 * * 1"
workflow_dispatch:
permissions: {}
jobs:
scan-scheduled:
uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
permissions:
contents: read
security-events: write
with:
# Advisory-first: surface results in code scanning without blocking.
fail-on-vuln: false
.github/workflows/release.yml
+168
View File
@@ -0,0 +1,168 @@
name: Release
on:
push:
tags:
- "v*"
permissions: {}
jobs:
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
# surface). Unchanged from prior versions; just isolated into its own job
# so the .mcpb matrix can run in parallel.
build-skill:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
attestations: write
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Build .skill artifact
run: |
bash skills/last30days/scripts/build-skill.sh
test -f dist/last30days.skill
- name: Attest .skill artifact provenance
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
with:
subject-path: dist/last30days.skill
- name: Upload skill artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: last30days-skill
path: dist/last30days.skill
# Cross-compile the Go MCP server for each Claude Desktop platform and
# package each as a .mcpb. printing-press bundle handles the manifest +
# zip layout; we only supply the pre-built binary via --skip-build.
build-mcpb:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
attestations: write
env:
MCPB_OUTPUT: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
MCPB_PLATFORM: ${{ matrix.platform }}
strategy:
fail-fast: false
matrix:
include:
- goos: darwin
goarch: arm64
platform: darwin/arm64
- goos: darwin
goarch: amd64
platform: darwin/amd64
- goos: linux
goarch: amd64
platform: linux/amd64
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
# printing-press v4.8.0 declares `go >= 1.26.3`, newer than the
# engine's own floor in mcp/go.mod. Install a 1.26.x toolchain so the
# PP `go install` below is satisfied without a runtime toolchain
# download (which GOSUMDB=off would block). Building the MCP binary
# with a newer toolchain than mcp/go.mod declares is backward-safe.
go-version: "1.26"
cache: false
- name: Install printing-press
# Pin to a known-good PP release so the bundle command's behavior
# is deterministic across our tags. Bump deliberately when adopting
# a newer PP version. GOSUMDB=off skips the sumdb 404 some
# private-namespaced go install calls hit even when the repo is
# public; harmless here because the module path is fully qualified.
env:
GOPRIVATE: github.com/mvanhorn/*
GOSUMDB: "off"
run: go install github.com/mvanhorn/cli-printing-press/v4/cmd/printing-press@v4.8.0
- name: Sync engine into vendored/
run: bash mcp/scripts/sync-engine.sh
- name: Build MCP binary
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: "0"
RELEASE_VERSION: ${{ github.ref_name }}
run: |
mkdir -p mcp/build
go -C mcp build \
-ldflags "-X main.Version=${RELEASE_VERSION}" \
-o build/last30days-pp-mcp \
./cmd/last30days-pp-mcp
- name: Bundle .mcpb
# printing-press bundle reads manifest.json from the cli dir and
# rewrites the binary into bin/<entry_point> inside the zip. The
# --platform tag drives the output filename suffix; the binary
# itself is whatever we just cross-compiled.
run: |
printing-press bundle mcp \
--skip-build \
--binary mcp/build/last30days-pp-mcp \
--platform "${MCPB_PLATFORM}" \
--output "${MCPB_OUTPUT}"
- name: Attest .mcpb artifact provenance
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
with:
subject-path: ${{ env.MCPB_OUTPUT }}
- name: Upload .mcpb artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
path: ${{ env.MCPB_OUTPUT }}
# Gather every platform artifact and attach to one GitHub release.
release:
needs: [build-skill, build-mcpb]
runs-on: ubuntu-latest
permissions:
actions: read
contents: write
steps:
# gh release create --verify-tag shells out to git, so the job needs a
# checkout with the tag present; without it the step fails with
# "fatal: not a git repository".
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Download all artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: dist
merge-multiple: true
- name: Create GitHub release
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ github.ref_name }}
run: |
gh release create "${RELEASE_TAG}" \
dist/last30days.skill \
dist/last30days-pp-mcp-*.mcpb \
--generate-notes \
--verify-tag
.github/workflows/scorecard.yml
+69
View File
@@ -0,0 +1,69 @@
name: Scorecard
# OpenSSF Scorecard tracks broader repo security-health drift (branch
# protection, token permissions, pinned actions, dangerous workflows, CI
# tests, maintenance signals) on a schedule, complementing the per-diff
# dependency-audit and secret-scan jobs in security.yml.
#
# Advisory-first: this workflow only measures and publishes a score, and it
# never blocks merges. It runs on the default branch (Scorecard needs repo-level
# data and a token, so it is not meaningful on PR forks) plus a weekly schedule
# so regressions in security health surface even when no code changes.
on:
branch_protection_rule:
schedule:
# Weekly, Mondays at 07:00 UTC.
- cron: '0 7 * * 1'
push:
branches:
- main
workflow_dispatch:
# Top-level token is read-only; the analysis job widens only what it needs.
permissions: read-all
jobs:
analysis:
name: Scorecard analysis
runs-on: ubuntu-latest
# Job-level permissions fully replace the top-level block (unlisted scopes
# default to none), so the reads checkout and Scorecard need are explicit.
permissions:
# Needed by actions/checkout to clone the repo, and by Scorecard to read
# workflow files for its Dangerous-Workflow / Token-Permissions checks.
contents: read
actions: read
# Needed to upload the SARIF results to the code-scanning dashboard.
security-events: write
# Needed to publish results and obtain a badge (uses OIDC, no secrets).
id-token: write
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Run OpenSSF Scorecard
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
with:
results_file: scorecard.sarif
results_format: sarif
# Publishes results to the OpenSSF REST API for the public badge and
# trend tracking. Set to false if maintainers prefer to keep the
# score private (the SARIF upload below still works either way).
publish_results: true
# Retain the raw SARIF as a build artifact for offline inspection.
- name: Upload artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: scorecard-sarif
path: scorecard.sarif
retention-days: 5
- name: Upload SARIF to code-scanning
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
sarif_file: scorecard.sarif
.github/workflows/security.yml
+89
View File
@@ -0,0 +1,89 @@
name: Security
on:
pull_request:
push:
branches:
- main
workflow_dispatch:
permissions: {}
jobs:
dependency-audit:
name: Dependency audit
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
# Block known vulnerabilities in the locked Python dependency graph.
- name: Run uv audit against locked dependencies
run: uv audit --locked
dependency-review:
name: Dependency review
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Review dependency changes
uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0
secret-scan:
name: Secret scan
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout full history for diff-aware scanning
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
# The action derives the commit range from the GitHub event and fails on
# verified secrets. Keep output limited to verified findings to avoid noisy
# unverified annotations.
- name: Run TruffleHog OSS secret scan
uses: trufflesecurity/trufflehog@30d5bb91af1a771378349dbbb0c82129392acf70 # v3.95.6
with:
version: 3.95.5
extra_args: --results=verified
sast-scan:
name: SAST scan
runs-on: ubuntu-latest
permissions:
contents: read
container:
image: semgrep/semgrep@sha256:06938c1f365d3f67b8cedd8bc117607ae64253f88a0e768e9da9408548927dd6 # v1.167.0
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
# Advisory-first: visibility before enforcement. Semgrep CE scans
# the repository with the community rule set (Python, shell, YAML,
# JavaScript, Go) to catch source-level security bugs before they
# reach production. Set continue-on-error: false once a clean baseline
# is confirmed.
- name: Run Semgrep SAST scan
continue-on-error: true
env:
SEMGREP_SEND_METRICS: off
run: semgrep scan --config=auto
.github/workflows/validate.yml
+68
View File
@@ -0,0 +1,68 @@
name: Validate
on:
pull_request:
push:
branches:
- main
permissions: {}
jobs:
tests:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
- name: Set up Python
run: uv python install 3.12
- name: Run test suite
run: uv run pytest --cov --cov-report=term-missing
eval:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
- name: Set up Python
run: uv python install 3.12
- name: Score research quality
run: uv run pytest tests/eval -x -s
mcp-tests:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: "1.25.5"
cache-dependency-path: mcp/go.sum
- name: Run MCP Go tests
run: go test -race ./...
working-directory: mcp
.github/workflows/zizmor.yml
+23
View File
@@ -0,0 +1,23 @@
name: GitHub Actions Security Analysis with zizmor 🌈
on:
push:
branches: ["main"]
pull_request:
branches: ["**"]
permissions: {}
jobs:
zizmor:
runs-on: ubuntu-latest
permissions:
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Run zizmor 🌈
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
.gitignore
+51
View File
@@ -0,0 +1,51 @@
# Private benchmark / evaluation artifacts — never push to upstream
docs/comparison-results/
test-run.log
scripts/evaluate-synthesis.py
scripts/generate-synthesis-inputs.py
fixtures/polymarket_sample.json
docs/v2.1-tweets.md
docs/30-day-anniversary-thread.md
docs/30-day-anniversary-tweets.md
variants/open/references/research.md
docs/investigations/
# OS / tool files
.DS_Store
.claude/
.entire/
__pycache__/
*.pyc
mise.toml
.memsearch/
.venv/
.coverage
htmlcov/
# Local secrets/config. Keep tracked examples if added later.
.env
.env.*
!.env.example
# Root vendor/ is accidental - real vendored client lives at scripts/lib/vendor/bird-search/
/vendor/
# build artifact from scripts/build-skill.sh
/dist/
# Go MCP bundle build outputs - source of truth for vendored/ stays under
# skills/last30days/scripts/; build/ holds cross-compiled binaries + .mcpb files.
# vendored/ lives inside the engine package because //go:embed cannot reach
# outside its own package directory; the .gitkeep anchor stays tracked so
# the embed pattern always finds a match even before sync-engine runs.
/mcp/internal/engine/vendored/*
!/mcp/internal/engine/vendored/.gitkeep
/mcp/build/
# Internal planning docs (ce:plan output) — keep local, don't publish
docs/plans/
docs/brainstorms/
.context/
/work
/print
.grok-plugin/marketplace.json
+25
View File
@@ -0,0 +1,25 @@
{
"name": "last30days-skill",
"owner": {
"name": "Matt Van Horn",
"url": "https://github.com/mvanhorn"
},
"description": "Marketplace for the last30days research plugin",
"plugins": [
{
"name": "last30days",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"version": "3.13.0",
"category": "productivity",
"source": {
"source": "url",
"url": "https://github.com/mvanhorn/last30days-skill.git"
},
"homepage": "https://github.com/mvanhorn/last30days-skill",
"keywords": [
"last30days",
"last 30 days"
]
}
]
}
.grok-plugin/plugin.json
+18
View File
@@ -0,0 +1,18 @@
{
"name": "last30days",
"version": "3.13.0",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"author": {
"name": "Matt Van Horn",
"email": "mvanhorn@gmail.com",
"url": "https://github.com/mvanhorn"
},
"homepage": "https://github.com/mvanhorn/last30days-skill",
"repository": "https://github.com/mvanhorn/last30days-skill",
"license": "MIT",
"keywords": [
"last30days",
"last 30 days"
],
"skills": "./skills/"
}
.skillignore
+70
View File
@@ -0,0 +1,70 @@
# Hermes install-time scanner/package exclusions for repository-root scans.
# Keep the public bundle focused on the runtime skill under skills/last30days/.
# VCS, local envs, caches, and generated outputs
.git/
.venv/
__pycache__/
*.pyc
*.log
*.jsonl
*.mp3
*.jpeg
*.jpg
*.png
*.gif
assets/
skills/last30days/assets/
.DS_Store
.coverage
htmlcov/
dist/
work/
print/
# Repo/dev automation and host-specific package metadata
.github/
.agents/
.claude-plugin/
hooks/
mcp/
gemini-extension.json
greptile.json
pyproject.toml
# Non-runtime docs, plans, release notes, fixtures, and tests
docs/
fixtures/
tests/
plans/
agents/
variants/
media/
README.md
CHANGELOG.md
AGENTS.md
CLAUDE.md
CONCEPTS.md
CONFIGURATION.md
CONTRIBUTORS.md
HERMES_SETUP.md
release-notes.md
SKILL-original.md
SPEC.md
TASKS.md
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
skills/last30days/scripts/build-skill.sh
skills/last30days/scripts/compare.sh
skills/last30days/scripts/evaluate_search_quality.py
skills/last30days/scripts/setup-keychain.sh
skills/last30days/scripts/setup-pass.sh
skills/last30days/scripts/test_device_auth.py
skills/last30days/scripts/test-v1-vs-v2.sh
skills/last30days/scripts/verify_v3.py
# Keep visible: optional runtime watchlist/store/briefing feature scripts
# (`watchlist.py`, `store.py`, and `briefing.py`).
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
skills/last30days/scripts/lib/vendor/
AGENTS.md
+112
View File
@@ -0,0 +1,112 @@
# last30days Skill
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, Grok (xAI), and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
## Structure
- `skills/last30days/SKILL.md` — canonical skill definition / runtime spec the model reads when the slash command fires
- `skills/last30days/scripts/last30days.py` — main research engine
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
- `CONFIGURATION.md` — user-facing knobs (env vars, flags, per-host install patterns); keep in sync per the rules below
- `CHANGELOG.md` — structured release history (launch copy lives in GitHub Releases)
- `HERMES_SETUP.md` — install instructions for the Hermes harness specifically
## Orientation
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
## Commands
```bash
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only).
# Saves to $LAST30DAYS_MEMORY_DIR when set in shell or ~/.config/last30days/.env;
# add --save-dir <path> for a one-off override. Mirrors LAST30DAYS_STORE convention.
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
npx skills add . -g -y # copies skill into ~/.agents/skills/<name>/ (frozen at install time); re-run to sync working-tree edits — see Rules below
# Tests (pytest, ~89 files under tests/, configured in pyproject.toml)
uv run pytest # full suite
uv run pytest tests/test_dedupe_v3.py # single file
uv run pytest tests/test_dedupe_v3.py -k some_case # single case
uv run pytest --cov # with coverage (skips lib/vendor/)
```
Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
## Rules
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
- One-time setup: `npx skills add . -g -y` copies the skill into `~/.agents/skills/<name>/` (real directory) and, for harnesses that support symlinked skill dirs, drops a per-host symlink pointing at that copy. **Working-tree edits do NOT propagate automatically** — the `~/.agents/skills/<name>/` copy is frozen at install time. To sync after edits, re-run `npx skills add . -g -y`. For live-edit on a dev machine, replace the install copy with a symlink to the working tree: `ln -sfn "$PWD/skills/last30days" ~/.agents/skills/last30days` (run from the repo root).
- Git remote: origin = public (`mvanhorn/last30days-skill`)
- Do not reduce `fail_under` in `pyproject.toml` (`[tool.coverage.report]`) without documenting why in the PR. The coverage gate is a floor meant to rise over time, not to be relaxed when new code is under-tested.
- Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`.
- **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`.
- **First-run onboarding is consent-driven, model-led, and host-split.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth, and emitting the engine-owned welcome via `--welcome`) — it cannot prompt, so consent lives in `SKILL.md` Step 0. Two flows avoid model-authored prose that Claude Code folds or the model skips: in the **Modal Flow** the welcome pitch is embedded in the setup modal's question (the AskUserQuestion modal is the only always-fully-visible surface — a separate welcome message or `--welcome` Bash run gets buried behind "ctrl+o to expand"); the **Non-Modal Prose Flow** still uses `last30days.py --welcome` (relayed verbatim) since it has no modal. The GitHub device code is surfaced by a two-command split — `setup --github-start` returns the code fast (foreground, copies to clipboard) and `setup --github-poll` waits for authorization (`setup --github` still chains both for back-compat). Step 0 has TWO branches: a **Claude Code Modal Flow** (the restored v3.0.0 `AskUserQuestion`-driven NUX — welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, `INCLUDE_SOURCES` opt-in, first-topic picker) for hosts with modals, and a **Non-Modal Prose Flow** for hosts without (OpenClaw, Codex, Cursor, Gemini CLI, Grok). Both ask before reading cookies, surface the macOS Full Disk Access fix on permission-denied, and offer the ScrapeCreators GitHub signup (10,000 free calls) on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do NOT collapse the modal flow back into a bare silent `setup` call or flatten it to prose-only — the guided modals are the feature (they eroded once and were restored). The onboarding contract is locked by `tests/test_onboarding_contract.py`. The Step 5 source opt-in is two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything** (also Threads + Pinterest). Comments are on by default (posts on → comments on for all three platforms); **Threads and Pinterest are the only opt-in extras**, appearing only in the Step 5 Everything option, never in the welcome or the Step 4 offer. Instagram comments are fetched via ScrapeCreators (`/v2/instagram/post/comments`, ranked by `comment_like_count`) with full vote-weighting parity to YouTube/TikTok (a dedicated `_instagram_engagement` carve-out, the `_VOTE_LOG_REFERENCE`/label/threshold entries). The cross-platform "Top Community Comments" list (`render._render_top_comments`) selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) so a viral platform can't crowd out a smaller one, and drops the per-platform absolute floor so a less-watched video's killer low-vote comment still surfaces.
## Security hygiene
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
## Maintaining CONFIGURATION.md
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
Update `CONFIGURATION.md` when:
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Grok, Hermes…)
- adding a new optional source that requires its own credential
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
## Plugin manifests (Grok)
The repo doubles as a native Grok Build plugin via `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json`. Grok also reads `.claude-plugin/*` for compatibility; the native pair is the first-class lane and what an official xAI marketplace listing points at. The self-hosted catalog uses a bare Git URL source (`{"source":"url","url":"https://github.com/mvanhorn/last30days-skill.git"}`) so `grok plugin marketplace add mvanhorn/last30days-skill` tracks HEAD — not a self-referential local `path: "."` (Grok does not enumerate those). Version lockstep with Claude/Codex/Gemini manifests is enforced by `tests/test_plugin_contract.py`. Validate with `grok plugin validate .`.
## Submitting to the xAI plugin marketplace
Getting last30days into xAI's official catalog (`xai-org/plugin-marketplace`) is an outbound PR to *their* repo — an index that only points at our source, so nothing of last30days is vendored there. Do this **after** the change you want to ship has merged to `main`: the entry pins a commit that must already exist.
1. Fork `xai-org/plugin-marketplace` and branch from `main`.
2. Get the commit to pin — a full 40-char lowercase SHA; a branch, tag, or short SHA is rejected by their validator:
```bash
git ls-remote https://github.com/mvanhorn/last30days-skill.git HEAD
```
3. Add one entry to their `.grok-plugin/marketplace.json` under `plugins[]`, a remote source pinned to that SHA:
```json
{
"name": "last30days",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"category": "productivity",
"source": {
"source": "url",
"url": "https://github.com/mvanhorn/last30days-skill.git",
"sha": "<full-40-char-sha-from-step-2>"
},
"homepage": "https://github.com/mvanhorn/last30days-skill",
"keywords": ["last30days", "last 30 days"]
}
```
4. Regenerate their component index (never hand-edit it) and validate exactly as their CI does:
```bash
python3 scripts/generate-plugin-index.py
python3 scripts/validate-catalog.py
python3 scripts/generate-plugin-index.py --check
```
5. Open the PR, fill in their template, and wait for code-owner review.
To roll out a later update in their catalog, bump the pinned `sha` in the existing entry — never open a second, parallel entry.
Do not confuse this with our own `.grok-plugin/marketplace.json`: that file makes this repo directly addable as a Grok marketplace (`grok plugin marketplace add mvanhorn/last30days-skill`) and uses a **bare URL** source (no SHA) so it tracks HEAD; the xAI entry above lives in *their* repo and uses a **remote** source pinned to a SHA.
## Beta channel
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
CHANGELOG.md
+840
View File
@@ -0,0 +1,840 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
## [3.13.0] - 2026-07-12
### Added
- Xiaohongshu (RED) documented as a first-class requested-only source, with auto-detection of a logged-in local browser-session service: last30days probes `http://localhost:18060` then `http://host.docker.internal:18060` when the source is opted in; `XIAOHONGSHU_API_BASE` remains the explicit override. Zero probing and zero behavior change for users who have not opted in. ([#766](https://github.com/mvanhorn/last30days-skill/pull/766), thanks @yuzhiyang1)
- DripStack as an opt-in source: premium financial newsletter and analyst-writeup search (free public API, no key), complementing StockTwits retail sentiment and Polymarket odds with professional analyst signal. Ships default-off; requests route through the shared HTTP layer and honor the 30-day window. ([#791](https://github.com/mvanhorn/last30days-skill/pull/791), thanks @zimoo354)
- Persistent opt-in for both new sources via `INCLUDE_SOURCES=xiaohongshu` / `INCLUDE_SOURCES=dripstack` in `.env`, matching the LinkedIn/Perplexity pattern; per-run `--search` still works. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
### Fixed
- Whitespace in comma-separated `INCLUDE_SOURCES` values no longer silently breaks any source's persisted opt-in. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
- DripStack article bodies (subtitle/lede) now reach ranking and synthesis instead of only the capped snippet; the Xiaohongshu doctor prescription no longer recommends an env pin that disables auto-probing. ([#811](https://github.com/mvanhorn/last30days-skill/pull/811))
- Release hygiene: SKILL.md body header and uv.lock are regenerated with the version bump (both were missed in the 3.12.0 cut and hotfixed on main).
## [3.12.0] - 2026-07-12
### Added
- Typed per-run source outcomes: every run records what actually happened per source (`ok`, `no-results`, `partial`, `rate-limited`, `auth-failed`, `unreachable`, `timeout`, `schema-drift`, `skipped-unconfigured`, `error`) in `source_status`, with doctor-aligned states and fix hints - silence is never mistaken for coverage. ([#797](https://github.com/mvanhorn/last30days-skill/pull/797))
- Versioned agent JSON export profile: `--emit=json --json-profile=agent` returns a stable machine contract (`schema_version` 1.2) with `source_status`, clusters, ranked results with joinable `candidate_id`, and freshness verdicts; `--json-profile=raw` keeps the legacy dump byte-identical. ([#798](https://github.com/mvanhorn/last30days-skill/pull/798), [#810](https://github.com/mvanhorn/last30days-skill/pull/810))
- Research-quality eval harness: recorded-fixture regression suite scoring runs on citation grounding, recency compliance, cluster coherence, coverage, and determinism against per-fixture floors, in CI. ([#799](https://github.com/mvanhorn/last30days-skill/pull/799))
- `--drill`: re-research one cluster of the cached report in depth without a full re-run. ([#800](https://github.com/mvanhorn/last30days-skill/pull/800))
- `--discover`: topic-less trending sweeps over listing feeds with velocity-ranked story clusters and ready-to-run research commands. ([#801](https://github.com/mvanhorn/last30days-skill/pull/801))
- `library feed`: renders every saved brief into a browsable HTML library with a topic-grouped index and a subscribable Atom feed; hand-written pages are preserved with backups. ([#802](https://github.com/mvanhorn/last30days-skill/pull/802))
- `library search`: SQLite FTS5 full-text search across saved briefs and store sightings, plus a passive "From your library" section when new runs overlap past research; scoped `--save-dir` libraries stay fully isolated from the shared store. ([#803](https://github.com/mvanhorn/last30days-skill/pull/803))
- `--register` audience templates: `exec`, `dev`, and `creator` presets reshape section order and budgets for the reader; `eli5` is unified into the same mechanism. ([#804](https://github.com/mvanhorn/last30days-skill/pull/804))
- `--verify-freshness`: typed per-claim act-time verdicts (`current` / `stale` / `contradicted` / `unsupported`) with point re-fetch of Polymarket lines, GitHub stars, and StockTwits sentiment, inline or post-hoc over the cached report; closes the recency-promise audit gap. ([#805](https://github.com/mvanhorn/last30days-skill/pull/805), closes [#769](https://github.com/mvanhorn/last30days-skill/issues/769))
- `--corpus`: register local directories as a private, offline, deterministic source; matching notes rank alongside social evidence under a LOCAL ONLY badge and are excluded from hosted publishing and agent JSON by default. ([#808](https://github.com/mvanhorn/last30days-skill/pull/808))
- Native Grok Build (xAI) plugin and marketplace lane: `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json` so `grok plugin install mvanhorn/last30days-skill` and `grok plugin marketplace add mvanhorn/last30days-skill` work as first-class install paths. The self-hosted catalog uses a bare Git URL source (tracks HEAD); submitting to the official `xai-org/plugin-marketplace` remains a post-merge SHA-pinned outbound PR documented in `AGENTS.md`.
### Fixed
- Session-start hook no longer deadlocks under Homebrew bash 5.3: removed every heredoc from `check-config.sh` (bash 5.3 can block forever in `heredoc_write` inside command substitution). ([#809](https://github.com/mvanhorn/last30days-skill/pull/809))
- Trustpilot transient-error retries keep their domain parameters. ([#794](https://github.com/mvanhorn/last30days-skill/pull/794))
- Hosted same-day saves no longer overwrite earlier reports, and `save_output` never silently overwrites date-stamped files. ([#784](https://github.com/mvanhorn/last30days-skill/pull/784), [#785](https://github.com/mvanhorn/last30days-skill/pull/785))
- `.env` reads as UTF-8 (with BOM tolerance and locale fallback) on Windows. ([#780](https://github.com/mvanhorn/last30days-skill/pull/780), [#715](https://github.com/mvanhorn/last30days-skill/pull/715))
- `FUN_LEVEL` and `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` are registered in `env.py` so `.env` values are no longer silently ignored; doctor detects `GITHUB_TOKEN` from the process environment. ([#708](https://github.com/mvanhorn/last30days-skill/pull/708), [#732](https://github.com/mvanhorn/last30days-skill/pull/732), [#782](https://github.com/mvanhorn/last30days-skill/pull/782))
- File descriptors close promptly across the engine (`open()` wrapped in `with`). ([#775](https://github.com/mvanhorn/last30days-skill/pull/775))
## [3.11.0] - 2026-07-05
### Added
- `last30days doctor`: a unified health command that aggregates every source's probe state into a single grouped report with copy-pasteable fix prescriptions. Layered design: dependency probes (missing/broken/timeout detection), backend-chain descriptors (predict-then-report, never a network call), a centralized prescription registry shared by doctor and quality nudges, and an aggregator with grouped rendering. Replaces the fragmented health knowledge previously spread across `--diagnose`, `--preflight`, `lib/health.py`, and post-run nudges. ([#753](https://github.com/mvanhorn/last30days-skill/pull/753))
### Fixed
- Techmeme: `search` results are now windowed to each record's own ISO date instead of stamping every record with today's date, so years-old archive headlines can no longer surface as current news. Dated in-window records take result-cap slots first; undated records (old `techmeme-pp-cli` binary or upstream markup change) degrade gracefully with a logged upgrade hint. The sync machinery is removed because `search` never read the local cache. ([#752](https://github.com/mvanhorn/last30days-skill/pull/752))
- LinkedIn now renders in the emoji-tree footer (👔 with likes/comments), the `## Stats` engagement summary, and with the correct "LinkedIn" label. Previously LinkedIn items were counted in `## Stats` but silently dropped from the footer because `_FOOTER_SOURCES`, `ENGAGEMENT_DISPLAY`, and `SOURCE_LABELS` all omitted the source - an 8-item LinkedIn run looked like the source never ran. ([#758](https://github.com/mvanhorn/last30days-skill/pull/758))
## [3.10.0] - 2026-07-04
### Added
- Instagram comments as a first-class ScrapeCreators source: `instagram.enrich_with_comments` fetches top comments via `GET /v2/instagram/post/comments` (ranked by `comment_like_count`), gated by `SCRAPECREATORS_API_KEY` + `instagram_comments` in `INCLUDE_SOURCES`. Full vote-weighting parity with YouTube/TikTok - a dedicated `_instagram_engagement` gives IG posts the same top-comment ranking carve-out, and IG comments render with a "likes" label. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
- Comments are now on by default: the first-run Step 5 Recommended tier enables top comments for TikTok, Instagram, and YouTube (`INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`); the Everything tier adds Threads + Pinterest. Comments were previously an opt-in "Everything" feature. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
### Changed
- The cross-platform "Top Community Comments" list now selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) instead of a global vote-magnitude sort, so the top-3-of-each-platform outranks the 4th-of-any and each platform's #1 is guaranteed a slot - a viral platform can no longer sweep the list. The list also drops the per-platform absolute vote floor so a less-watched video's high-signal low-vote comment still surfaces (the per-candidate card keeps its floor). ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
### Fixed
- First-run wizard: the welcome pitch is embedded directly in the setup modal (the only always-visible surface) instead of a separate `--welcome` message that Claude Code folds behind "ctrl+o to expand"; the cookie-consent and ScrapeCreators-offer copy now name every installed CLI (yt-dlp, Digg, arXiv, Techmeme) and describe the key's real reach (auto Reddit enrichment + YouTube search backstop), with the GitHub device code auto-copied to the clipboard. ([#750](https://github.com/mvanhorn/last30days-skill/pull/750))
## [3.9.4] - 2026-07-04
### Fixed
- First-run wizard: the welcome message and the ScrapeCreators GitHub device code are now engine-driven instead of model-authored, because a real cold run showed the model skipping the welcome and never surfacing the device code no matter how forceful the SKILL.md prose. The welcome is printed by a new `last30days.py --welcome` command that Step 1 relays verbatim (single source of truth; it can't be skipped or drift), and the GitHub device flow is split into `setup --github-start` (submits, copies the code to the clipboard, prints it to stdout, opens the browser, returns immediately) and `setup --github-poll` (waits for authorization and persists the key). The one-shot `setup --github` still chains both. The code now always appears in the command output, and the "on your clipboard" claim is only made when the copy actually succeeded. ([#748](https://github.com/mvanhorn/last30days-skill/pull/748))
## [3.9.3] - 2026-07-04
### Added
- Optional remote research API backend (env-driven). When both `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` are set in the process environment (never read from `.env`), a search runs through the configured remote endpoint (submit -> poll with stderr progress -> render) instead of local sources; with either unset, behavior is byte-identical to local-only. Opt-in and inert by default (no built-in endpoint); the key is confined to the `Authorization` header and never logged or persisted. Handles the clarify gate and 401/402/429 paths. ([#747](https://github.com/mvanhorn/last30days-skill/pull/747))
### Fixed
- First-run wizard: the welcome message is now mandated before the setup modal (it was being skipped), the Auto-setup option lists every installed CLI (yt-dlp, Digg, arXiv, Techmeme, not just two), and the ScrapeCreators GitHub signup reliably surfaces the device code with an "it's on your clipboard, just paste" hint as a required step instead of leaving the user staring at a spinner. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
- ScrapeCreators GitHub signup: an already-linked account whose `.env` is cold no longer fails with the misleading "GitHub auth didn't complete." The `Authorized but failed to fetch API key` case now gets an honest branch (auth worked; the account is likely already linked -- get your key from scrapecreators.com and paste it), and `fetch_api_key` logs the `/profile` response field names (never values) so a full auto-fetch can follow. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
## [3.9.2] - 2026-07-03
### Fixed
- Trustpilot source returned 0 items on company topics: the engine passed raw topic names to a domain-keyed CLI (`info ThriftBooks` -> HTTP 404) and parallel subqueries raced concurrent Chrome WAF-cookie harvests. Company names now resolve to their Trustpilot review-page domain via the CLI's search (per-topic cache; name-match mandatory, ambiguous cases fall back rather than misattributing another company's reviews), a new `--trustpilot-domain` flag pins the domain explicitly (verbatim, bypasses the brand-shape gate, per-entity `trustpilot_domain` in `--competitors-plan`), the WAF session warms once per 240s window behind a lock at first fetch, Trustpilot is capped to one fetch per run and excluded from the thin-source retry, and headless `--auto-resolve` fills a verified domain hint. SKILL.md Step 0.5d documents the resolution flow. ([#745](https://github.com/mvanhorn/last30days-skill/pull/745))
## [3.9.1] - 2026-07-03
### Fixed
- First-run setup wizard: the browser-cookie scan now tries Chrome/Chromium first (Keychain, no Full Disk Access) before Safari, so macOS users logged into X in Chrome authenticate in ~2s instead of hitting the Safari Full Disk Access dead-end. The winning browser is pinned for later runs only when it is Firefox/Safari, so Chrome never re-triggers the Keychain prompt. Consent copy leads with Chrome and the one-time "Always Allow" cue. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
- ScrapeCreators GitHub signup now surfaces the device code immediately (emitted to stdout so a backgrounded caller shows it at once, instead of a spinner until the process exits), validates the `XXXX-XXXX` code shape before copying/labeling it, short-circuits an already-registered account without a fresh device dance, and masks the API key on every status (not just success). Removed the false "GitHub CLI ~2 seconds, no browser" promise. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
- ScrapeCreators source opt-in is now two real tiers. The Step 5 choices were previously identical — a key auto-ran TikTok, Instagram, Threads, and YouTube comments regardless of `INCLUDE_SOURCES`, and Pinterest's opt-in silently ignored a persisted `INCLUDE_SOURCES`. Threads, YouTube comments, and Pinterest are now genuine `INCLUDE_SOURCES` opt-ins: **Recommended** = TikTok + Instagram + the rate-limit backups; **Everything** = also Threads, Pinterest, and YouTube/TikTok/Instagram comments. "ScrapeCreators backups" is now defined inline (keeps Reddit/YouTube working at rate limits). ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
## [3.9.0] - 2026-07-03
### Added
- StockTwits as a source, gated to ticker/crypto topics only. Surfaces a retail sentiment ratio (self-reported Bullish/Bearish tags) and message volume on a resolved symbol. Inert on non-financial topics: an unambiguous finance-vocabulary gate (cashtags, "stock", "earnings", "dividend", "crypto", named coins) keeps it from injecting stock chatter into general runs, and it degrades to an empty lane if the public API fails without touching other sources. ([#658](https://github.com/mvanhorn/last30days-skill/pull/658), thanks @wtiwana)
- LinkedIn as a source via ScrapeCreators, surfacing articles as high-signal results with date-range filtering, gated behind `INCLUDE_SOURCES`. ([#702](https://github.com/mvanhorn/last30days-skill/pull/702))
- arXiv and Techmeme sources (default-on) plus Trustpilot (opt-in). ([#709](https://github.com/mvanhorn/last30days-skill/pull/709))
### Fixed
- Runtime preflight now auto-provisions a uv-managed CPython 3.12 on hosts that have `uv` but no system Python 3.12+ (most agent sandboxes), instead of hard-failing the version gate. The install is bounded by a 30s HTTP timeout, matches an existing managed `>=3.12` interpreter before downloading, and announces the one-time ~28MB download on stderr rather than installing silently; hosts without `uv` still get the original clear error. Setup invocations now honor `LAST30DAYS_PYTHON` so first-run setup works on the same hosts. ([#738](https://github.com/mvanhorn/last30days-skill/pull/738), thanks @buntysomroy; setup-interpreter fix adapted from #699 by @SeanGearin)
- Setup wizard summary now displays the install status of the arXiv/Techmeme pp_sources CLIs, so users can see whether they landed on PATH. ([#741](https://github.com/mvanhorn/last30days-skill/pull/741), thanks @23241a6749)
- `--diagnose` / `--preflight` no longer falsely reports X as unreachable when X auth comes from `FROM_BROWSER` browser cookies. These modes run in `plan_only` and skip cookie extraction for privacy (no Keychain access), so X was dropped from `available_sources` even though a real run authenticates fine. A new side-effect-free `env.x_pending_browser_auth` predicate now reports X as available-pending-browser-auth (and surfaces an `x_pending_browser_auth` flag in `--diagnose`) by keying only on the already-resolved browser list — no cookie is read. Covers every configured browser, including Chrome. ([#692](https://github.com/mvanhorn/last30days-skill/issues/692); first reported and fixed by @23241a6749 in #700)
### Internal
- Tightened Hermes `.skillignore` regression coverage: the test now fails if an ignored path is deleted without updating the ignore list, or if a runtime-contract file is accidentally ignored. ([#739](https://github.com/mvanhorn/last30days-skill/pull/739), thanks @SyntaxSawdust)
## [3.8.3] - 2026-06-25
### Added
- Free Reddit gets dedicated-subreddit lanes: entity-home subs (e.g. r/Kanye for "Kanye West", via the new `--dedicated-subreddits` flag) are pulled in full from top+hot+new listings and exempt from the relevance floor, since the whole sub is the topic. Fixes the over-aggressive floor that dropped on-topic posts whose titles lacked the entity name.
- `reddit_arctic` resolves upvote counts for threads found only via RSS search (which carries no score) using the free, keyless arctic-shift archive — batched, paced, cached, and graceful-degrading. Reddit now gets headlines-with-points and best-comments-with-points entirely for free, at parity with ScrapeCreators.
- `LAST30DAYS_REDDIT_SC_MIN_ITEMS` (default 0 = unchanged empty-only behavior): set above 0 to let the ScrapeCreators backup backfill a thin free Reddit run instead of sitting idle. Backfilled items merge deduped by post id.
### Removed
- The permanently-403 `search.json` Tier 0 is gone from the keyless Reddit path; discovery is RSS breadth + shreddit listing partials (real scores) + the dedicated-sub lanes, with no wasted 403 calls.
## [3.8.2] - 2026-06-25
### Added
- Advisory Semgrep SAST scan runs on every push/PR as part of the Security workflow, catching source-level security bugs using Semgrep CE community rules ([#563](https://github.com/mvanhorn/last30days-skill/issues/563))
- Scheduled OSV-Scanner vulnerability-drift workflow scans repository lockfiles weekly and uploads SARIF results to GitHub code scanning, catching newly disclosed CVEs in the dependency tree even between PRs ([#571](https://github.com/mvanhorn/last30days-skill/issues/571))
- `LAST30DAYS_REDDIT_BACKEND=scrapecreators` makes ScrapeCreators the primary Reddit backend with the public path as fallback. Users with a ScrapeCreators key who were getting shallow public data will now get full nested comment trees by setting this flag ([#589](https://github.com/mvanhorn/last30days-skill/issues/589))
- MCP Go tests (`mcp/`) now run in CI on every push/PR alongside the Python test suite, so MCP server regressions are caught before merge ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
- PR dependency review gate blocks merges that introduce new vulnerable dependencies ([#551](https://github.com/mvanhorn/last30days-skill/issues/551))
### Changed
- Citations are now renderer-aware (LAW 8). On hidden-link hosts (Claude Code) every citation stays an inline `[name](url)` link as before; on visible-URL hosts (Codex, Cursor, Gemini CLI, raw CLI) citations render as plain source labels so the narrative no longer turns into `label (https://...)` URL soup. The host is detected deterministically from the `CLAUDECODE` environment variable, and full URLs remain reachable through the engine footer and the saved raw file.
### Fixed
- The query-plan invocation guidance now warns against wrapping the heredoc in `bash -lc '...'` / `zsh -lc '...'`, whose single quotes terminate at the first apostrophe in a ranking string and abort the engine run with `unmatched "` on Codex. The quoted `<<'PLAN_EOF'` heredoc is already apostrophe-safe; the `-lc` wrapper was the hazard.
- Firefox profile detection on Linux now checks `$XDG_CONFIG_HOME/mozilla/firefox` (or its default `~/.config/mozilla/firefox`) in addition to `~/.mozilla/firefox`, fixing cookie extraction on distros that honour the XDG Base Directory Specification ([#667](https://github.com/mvanhorn/last30days-skill/issues/667))
## [3.8.1] - 2026-06-22
### Added
- **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
- **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659).
### Fixed
- **First-run setup no longer runs silently.** The prior Step 0 told the model to run `setup` and "follow the wizard's prompts end-to-end", but the wizard has no prompts — so onboarding extracted cookies, installed tools, and wrote `SETUP_COMPLETE` with zero interaction and never offered the ScrapeCreators signup. Reproduced 2026-06-22 (Fredy Montero, fresh macOS).
## [3.8.0] - 2026-06-21
### Added
- **Single X source with backend failover.** X is now one source backed by an ordered chain of interchangeable backends (xai, bird, xurl, xquik) with runtime failover, rather than separate sources. The key-based xquik backend reaches parity with bird, gaining the X-quality ranking and FROM/ABOUT handle lanes, so hosts that cannot supply browser cookies (OpenClaw, CI/cron, headless harnesses) get real X coverage from an xquik key alone. Handle lanes run via the first handle-capable backend in the chain even when a non-capable backend (xai/xurl) is primary. (#622)
## [3.7.1] - 2026-06-21
### Fixed
- GitHub repo stars are no longer mislabeled as "reactions" in the report footer. Repo cards use a distinct `stars` engagement key, velocity cards use `merged_prs`, and genuine issue/PR reaction counts keep their own `reactions` key. (#645, closes #642)
- Hacker News returned zero stories on every run: the Algolia query sent `points>2`, which the HN index no longer accepts as a filterable attribute, so every request 400'd. Dropped the server-side `points` filter; low-engagement demotion still happens at parse time. (#639)
- Polymarket surfaced off-topic markets and rendered a mangled footer. The relevance filter was fed the per-subquery string instead of the stable topic, and market labels were truncated mid-article into fragments like "an Anthropic Claude model score at: an 19%". Now filters on the stable topic and cleans the labels. (#640)
## [3.7.0] - 2026-06-20
### Added
- **Direct Perplexity API support.** When `PERPLEXITY_API_KEY` is set it is preferred over OpenRouter for the Perplexity source, unlocking first-party Search API results and async Deep Research. Adds `LAST30DAYS_PERPLEXITY_MODE=sonar|search|both` plus model, search-context, domain/language/country, recency, and reasoning-effort knobs. OpenRouter stays the Sonar compatibility fallback when no direct key is set. Async Deep Research preserves request id, status, idempotency key, poll count, lifecycle timestamps, and failure metadata in raw artifacts. (#629, by @sk-holmes)
### Changed
- `check-config.sh` now parses env files in pure bash (no `sed` / `tr`), which also fixes the YouTube-availability hint breaking in minimal environments that lack those tools. (#629)
## [3.6.1] - 2026-06-20
### Added
- **ScrapeCreators transcript fallback.** When `SCRAPECREATORS_API_KEY` is set, YouTube transcripts fall back to the ScrapeCreators transcript endpoint after the keyless yt-dlp cascade fails (fetched server-side, so no 429 / cookies / PO tokens). yt-dlp stays primary and a credit is only spent on a genuine failure, never on success and never on a video proven to have no captions. With a key, yt-dlp also fails over fast (one short-timeout attempt) so a 429 hands off to ScrapeCreators in roughly 17s instead of roughly 90s. (#637, idea from #595)
- **YouTube comments default-on.** Comment enrichment now activates whenever a ScrapeCreators key is present (bounded to the top ~3 videos by engagement, ~3 credits per run) instead of requiring `INCLUDE_SOURCES=youtube_comments`. Suppress with `EXCLUDE_SOURCES=youtube_comments`. TikTok/Instagram comments remain `INCLUDE_SOURCES` opt-ins. (#637)
### Fixed
- **Salvage partial YouTube transcripts on non-zero yt-dlp exit.** With the default `en,es,pt` languages an English video wrote `en.vtt` then 429'd on `es`/`pt`, and the already-written transcript was discarded and retried back into the rate limit. Any VTT on disk is now read before the failure is classified, which fixes the dominant `0/N transcripts` case. (#636)
- **Windows transcript crash on subprocess timeout.** Guarded the SIGKILL escalation path in `run_with_timeout` against `os.killpg` / `os.getpgid` raising `AttributeError` on Windows (they are POSIX-only), mirroring the primary path's guard. (#638, reported in #588)
## [3.6.0] - 2026-06-18
### Added
- **First-party X posts are no longer buried.** A post authored by one of the run's resolved handles (`--x-handle`, `--x-related`, the GitHub user) is now treated as first-class evidence: it is exempt from the entity-miss demotion (a post never repeats its own author's name, so the body-text grounding check used to zero out the subject's own highest-signal posts) and gets a small authorship credit. Third-party collision-noise suppression is unchanged.
- **Engagement rescue for on-topic X posts.** A high-engagement X post that is first-party or entity-grounded gets a `final_score` floor scaled by its engagement percentile within the run's X pool, so a viral on-topic post can't sit at ~0. Off-topic name-collision posts are explicitly excluded.
- **First-party interaction signal.** A first-party post directed at another account (a reply / leading @mention) is floated into the visible band regardless of like-count and tagged `interaction:→@handle` in the EVIDENCE block, so the synthesis reads it as a relationship signal rather than low-engagement noise. New **LAW 10** in SKILL.md teaches the model to surface first-party posts and read the interaction tag.
### Changed
- The X FROM lane (the subject's own timeline) now pulls up to 8 posts per handle (was 3); the about/related lanes stay modest.
### Fixed
- Secrets `.env` and its parent config directory are now auto-tightened to `0o600`/`0o700` after creation, and `check-config.sh`'s `check_perms` now auto-fixes loose permissions with `chmod 600` instead of warning only ([#573](https://github.com/mvanhorn/last30days-skill/issues/573))
## [3.5.0] - 2026-06-18
### Added
- **X surfaces tweets FROM and ABOUT a person, both engagement-weighted.** The handle search now pulls the person's real timeline (`from:handle since:`, topic used for ranking only — never AND'd into the query, which previously matched only tweets where they wrote their own name and returned ~0), and a new mention lane (`@handle since:`) surfaces what others say to/about them, excluding their own tweets and deduping against the FROM lane ([#610](https://github.com/mvanhorn/last30days-skill/pull/610)).
- **`## Top Community Comments` block.** The engine now surfaces vote-ranked community comments across all candidates (not just the top-cluster representatives), per-platform-normalized, into the EVIDENCE-for-synthesis block, so the funniest/sharpest crowd reactions reach the synthesizing model even when no LLM fun-scorer is available. Paired with a new SKILL.md **LAW 9** that requires weaving ≥2 verbatim attributed comments, copying URLs verbatim, and never narrating the tooling in the deliverable ([#608](https://github.com/mvanhorn/last30days-skill/pull/608)).
### Fixed
- **`--diagnose` honesty.** X status now reflects a real 1-tweet probe (downgrades from green when X is effectively dead; fail-open on a transient timeout) and reports the true auth lane (browser / env / keychain) instead of a hardcoded `env AUTH_TOKEN`. Handle/mention searches log query + result count on success, not only on failure ([#609](https://github.com/mvanhorn/last30days-skill/pull/609)).
- **X column de-pollution.** The last-chance keyword retry no longer collapses a multi-word subquery to a bare generic token (e.g. `compound`); it keeps an entity anchor ([#607](https://github.com/mvanhorn/last30days-skill/pull/607)).
- **Mandatory person-aware subquery disambiguation.** Collision-prone person names (Kevin Rose vs Kevin Warsh, Lan Xuezhao vs Lanzhou) must anchor every subquery with the resolved company/role/domain context ([#611](https://github.com/mvanhorn/last30days-skill/pull/611)).
## [3.4.0] - 2026-06-18
### Added
- **Crowd-vote weighting in the fun judge (Best Takes).** The fun judge now factors how many upvotes/likes each top comment earned. Comment vote counts are fed into the LLM prompt (as traction, not funniness), and Best-Takes selection ranks by an effective score — `fun_score` plus a bounded, per-platform-normalized, relevance-confidence-scaled crowd nudge — so genuinely funny, crowd-loved, on-topic comments surface while off-topic virality and high-voted-but-unfunny rants are excluded. `FUN_LEVEL=medium` stays the default and applies the signal as a meaningful factor ([#592](https://github.com/mvanhorn/last30days-skill/pull/592)).
- **Digg added to first-run setup.** The free, keyless `digg-pp-cli` is now auto-installed during the first-run wizard (best-effort via the Printing Press installer, with a recommend-only fallback), so the already-built Digg AI-news source activates automatically for new users instead of silently never appearing ([#590](https://github.com/mvanhorn/last30days-skill/pull/590)).
- **`LAST30DAYS_YOUTUBE_SSH_HOST` transcript routing** — yt-dlp transcript fetch runs on the remote SSH host via a mktemp + cat pipeline ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
- Browser-cookie auth for X/Twitter now covers the full Chromium family on macOS - Brave, Microsoft Edge, Vivaldi, Opera, Arc, and Chromium - alongside the existing Chrome, Firefox, and Safari. They all share Chrome's v10 AES-128-CBC decryption, differing only in profile path and Keychain service name, so they run through one shared decryption core. The profile finder probes both the modern `Default/Network/Cookies` layout (Chromium >= 96) and the legacy flat `Default/Cookies`, and Chrome now resolves through that same finder so it picks up the modern layout too. Set `FROM_BROWSER=auto` to try every browser, or `FROM_BROWSER=<name>` (e.g. `brave`, `edge`, `arc`) to target one. Verified end-to-end on real Brave and Edge installs ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
- **First-party positioning research + pitch-vs-pulse synthesis (company / product / service topics).** A new mandatory research step captures each entity's current stated positioning from first-party sources (homepage, docs, pricing) rather than from memory. The fetched pitch grounds `What it is` descriptions (entities described as they pitch themselves today), helps reject unrelated brand-name noise, and feeds an evidence-triggered prose beat: when the month's conversation directly supports a specific claim, cuts against one, or is squarely about the pitched ground, the synthesis says so anchored to the top thread — and stays silent when the pulse is orthogonal to the pitch, because a manufactured connection is worse than omission. Claims are tested at matched altitude (specific claims against specific threads; broad taglines are never graded against individual items), and statements stay windowed to the 30 days — no trend verdicts. Scoped to entities with an identifiable first party: people are always excluded (even founders whose companies qualify), as are events, abstract concepts, and ownerless topics like Bitcoin; the beat requires positioning fetched during the run, never from memory.
### Changed
- Updated "Unlock X" promo message to mention Chrome/macOS support and Windows Firefox-only limitation instead of generic "Firefox or Safari" ([#387](https://github.com/mvanhorn/last30days-skill/issues/387))
### Fixed
- **SSH routing failures no longer present as "0 results"** — `search_youtube` surfaces non-zero SSH exit codes as an explicit `error` field ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
- `extract_browser_credentials()` silently ignored Brave even though the lower-level `cookie_extract` layer already supported it: `FROM_BROWSER=brave` fell back to Firefox/Safari and `FROM_BROWSER=auto` never tried Brave. The env wiring now passes Brave - and the rest of the Chromium family - through to the extractor ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
- Chromium cookie extraction now fetches the macOS Keychain key lazily - only when an encrypted cookie actually needs decrypting. Previously the key was fetched as soon as the cookie DB existed, so `FROM_BROWSER=auto` could trigger a Keychain prompt for every installed Chromium browser. Now only the browser that actually holds the requested cookie prompts ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
- YouTube transcript budget prioritises recent videos (by a combination of views and recency) instead of views alone, preventing transcript slots from being consumed by old high-view-count videos that would be discarded by strict_recent freshness pruning ([#531](https://github.com/mvanhorn/last30days-skill/issues/531))
- YouTube items with successfully extracted transcripts are no longer pruned by title-only relevance scoring; the transcript content proves substantive topical coverage even when the video title has low lexical overlap with the query ([#468](https://github.com/mvanhorn/last30days-skill/issues/468))
- First-run setup wizard in SKILL.md now references the existing Python setup wizard (`last30days.py setup`) instead of the missing `nux-wizard.md` file, so first-run setup actually runs on new installs. ([#574](https://github.com/mvanhorn/last30days-skill/issues/574))
- `check-config.sh` no longer exits 1 on the ScrapeCreators-configured path when no prior run exists (empty `LAST_RUN_LINE`) — swapped `&&` guard for an `if` block that always exits cleanly ([#463](https://github.com/mvanhorn/last30days-skill/issues/463))
- `check-config.sh` no longer exits 1 when a `.env` value contains an unbalanced quote — replaced `xargs` (which interprets quotes) with `sed` for whitespace trimming in `load_env_vars` ([#506](https://github.com/mvanhorn/last30days-skill/issues/506))
- X/Twitter `.env` template now includes `CT0` alongside `AUTH_TOKEN` in the example skeleton ([CONFIGURATION.md](CONFIGURATION.md)), and the just-in-time unlock wizard offers AUTH_TOKEN/CT0 cookie entry ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
- `check-config.sh` no longer counts X as an active source when only `AUTH_TOKEN` is set without `CT0` — both cookies are now required to credit X in the source count ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
- Firefox cookie extraction now falls back to scanning non-default profiles when the default profile has no matching X cookies, fixing multi-profile setups where login lives on a non-default profile ([#498](https://github.com/mvanhorn/last30days-skill/issues/498))
- `subproc.py` `run_with_timeout()` now guards `os.killpg` / `os.getpgid` with `hasattr`, preventing an uncaught `AttributeError` crash when a subprocess times out on Windows where these functions don't exist ([#527](https://github.com/mvanhorn/last30days-skill/issues/527))
- Entity-grounding rerank demotion now keys on the head token of the primary entity instead of requiring the full multi-word phrase as a contiguous substring. A high-engagement on-entity item (e.g. a 323-pt HN thread titled "Stripe is friendly to 'friendly fraud'") is no longer demoted to score 0 on a `Stripe payments` query just because it lacks the trailing search-hint word. The intended demotion still fires for items that never name the brand at all. The keyless Reddit comment-enrichment slot selection (`_slot_priority`), which mirrors this signal, was updated to the same head-token grounding so the two paths stay consistent.
- `--plan` / `--competitors-plan` file reads now specify `encoding="utf-8"` and catch `UnicodeDecodeError`, preventing crashes on non-ASCII content like accented entity names on Windows (cp1252). `check_perms()` in `check-config.sh` now skips the POSIX 600-permission check on MSYS/MinGW/Cygwin where `stat` runs in noacl mode. `skill_meta.py` `read_skill_version()` now passes `encoding="utf-8"` so SKILL.md emoji doesn't break version detection on Windows. ([#549](https://github.com/mvanhorn/last30days-skill/issues/549))
## [3.3.2] - 2026-06-06
### Fixed
- YouTube transcript extraction now falls back through `en,es,pt` (configurable via `LAST30DAYS_YT_SUB_LANGS`) instead of English-only, so non-English videos with auto-captions in any of those three languages now contribute transcripts to the brief ([#469](https://github.com/mvanhorn/last30days-skill/issues/469))
- Keyless Reddit comment enrichment now spends its limited slots on entity-matching posts first (mirroring rerank's entity-miss demotion signal) instead of raw upvote order, so off-topic high-upvote threads from broad subreddits no longer consume the comment budget only to be demoted afterward ([#484](https://github.com/mvanhorn/last30days-skill/pull/484))
## [3.3.1] - 2026-05-30
### Fixed
- Removed the redundant `commands/last30days.md` wrapper so the plugin exposes only the skill ([#461](https://github.com/mvanhorn/last30days-skill/issues/461)). Previously the plugin shipped both a command wrapper and the skill under the same name, so `/last30` surfaced two `last30days` entries with two different descriptions. The skill already carries its own `argument-hint`, so the `/last30days <topic>` picker UX is unchanged.
- Corrected the README install note that claimed Claude Code dedupes the slash command across install methods; it does not, so having both the marketplace plugin and the `npx skills` copy active shows two entries.
## [3.3.0] - 2026-05-17
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
### Added
**Emit modes and sources**
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
**Configuration knobs**
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
**Pipeline and storage**
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
**Tests and CI**
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
**Docs**
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
### Changed
**Install story modernized**
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
**Skill semantics and multi-harness reframe**
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
**Dependencies and tooling**
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
### Removed
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
### Fixed
**Reddit**
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
**Authentication and credentials**
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
**Windows compatibility**
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
**xAI / X / xurl**
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
**YouTube**
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
**bird_x / HTTP**
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
**Planner and sources**
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
**Render, version display, hosting paths**
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
**Hooks**
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
**Sync and version metadata**
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
### Contributors
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
- Dave Morin — portable test-harness paths
- Alex Key — `removeprefix("r/")` for subreddit names
- Eric Oberhofer — multi-key rotation restored
- gujishh — Windows process cleanup
- Franco Carballar — Reddit browser-like headers
- Jonathan Oppenheim — Reddit 402 fallback chain
- Kaustav Mishra — xAI error surfacing
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
## [3.2.0] - 2026-05-09
### Added
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
## [3.1.1] - 2026-04-24
### Fixed
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
## [3.1.0] - 2026-04-22
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
### Added
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
### Fixed
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
## [3.0.14] - 2026-04-22
### Changed
- **Comparison-mode title attribution.** The synthesis title for vs-mode and `--competitors` outputs changes from `What the Community Says (Last 30 Days)` to `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range. Three SKILL.md occurrences updated; pure documentation change.
## [3.0.13] - 2026-04-22
### Changed
- **vs mode runs N full passes in parallel, one per entity.** Architectural revert of the 3-pass → 1-pass latency optimization from an earlier version. `/last30days "OpenAI vs Anthropic vs xAI"` now runs three full `pipeline.run()` calls in parallel via the same fanout `--competitors` uses, producing three `*-raw.md` save files plus a merged comparison output. Each entity gets its own Step 0.55-grade targeting, own primary X handle weight, own subreddit scoping — apples-to-apples depth instead of the one-pool merged retrieval the single-pass path produced. Parallel execution keeps wall clock ≈ single pass.
- **`--competitors` is now a SKILL.md-level shortcut for vs-mode with auto-discovery.** The hosting reasoning model (Claude Code, Codex, Hermes, Gemini, any agent with WebSearch) performs discovery and Step 0.55 per entity via its own WebSearch tool, then invokes the engine with a vs-topic and `--competitors-plan` JSON. The engine flag remains for headless/cron use with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys (engine-internal `auto_resolve` stays as fallback).
- **LAW 7-style stderr for `--competitors` with no backend** now leads with the hosting-model path (WebSearch + Step 0.55 + `--competitors-plan`) instead of `BRAVE_API_KEY`. API-key framing moved to a secondary "headless" section.
### Added
- **`--competitors-plan` JSON flag** for per-entity Step 0.55 targeting. Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`. Accepts inline JSON or a file path (matches `--plan`). When present for an entity, skips engine-internal `auto_resolve` and uses the provided values; missing fields fall back to `auto_resolve` (if backend) or planner defaults. Case-insensitive entity matching. The `subrun_kwargs_for` helper is the single source of truth for per-entity kwargs — no closure-default fallthrough from main scope.
- **Per-entity save files** when `--save-dir` is set on a vs-mode or `--competitors` run. Each entity's sub-run produces its own `{slug}-raw.md` with a single-row Resolved Entities block — matches historical vs-mode behavior (N passes → N save files).
- **`--polymarket-keywords "kw1,kw2"`** to filter Polymarket matches for ambiguous single-token topics (e.g., "Warriors" → `nba,gsw,golden-state` kills Glasgow Warriors rugby and Honor of Kings Rogue Warriors noise).
### Fixed
- **BRAVE/SERPER footer nudge suppressed** when `--plan` or `--competitors-plan` is present. The nudge told Claude Code users to set an API key when they already have WebSearch via the hosting model. Nudge still fires for true headless runs (no `--plan`, no backend) where the advice is correct.
- **Override-leak regression testing.** 3.0.12 already fixed the main-topic `--subreddits` / `--x-handle` / `--github-*` from leaking into peer sub-runs via explicit per-entity kwargs scrubbing. This release adds a 4-test regression suite (`test_competitor_subrun_isolation.py`) locking in the invariant.
## [3.0.12] - 2026-04-22
### Fixed
- **Per-entity Step 0.55 resolution for competitor sub-runs.** In 3.0.11, only the main topic got X handle / subreddit / GitHub resolution; competitor sub-runs ran with planner defaults and produced visibly thinner evidence (Reddit 403 fallbacks, single-word queries). Each competitor sub-run now calls `resolve.auto_resolve()` inside `fanout.run_competitor_fanout` when a web backend is available, mirroring the main topic's pre-flight resolution. Per-entity X handle, subreddit list, GitHub user/repos, and news context are threaded into each sub-run's `pipeline.run()` call. Deep-copied config per sub-run prevents `_auto_resolve_context` cross-leak. Surfaces in a new `## Resolved Entities` output block so the resolution coverage is visible without reading stderr.
- **LAW 7 false-positive on internal fan-out sub-runs.** Each competitor sub-run was emitting the `[Planner] No --plan passed... YOU ARE the planner` stderr warning. LAW 7 targets the hosting-reasoning-model path, not engine-internal fan-out. New `internal_subrun=True` keyword on `planner.plan_query` and `pipeline.run` suppresses the warning for sub-runs only; the default path is unchanged.
- **Marketplace-stale SKILL.md trap.** Added a STEP 0 canonical-path self-check at the top of SKILL.md. Two of three 2026-04-22 test runs loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (Claude-Code-managed git clone pinned to origin/main, lagging the versioned cache), then ran `--help` against the same stale path, did not see `--competitors`, and fell back to a manual comparison plan. The STEP 0 block forces any reader to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` and re-read from the versioned cache if not.
### Changed
- **Default `--competitors` count is now 2 (3-way total: original + 2 peers).** Previously 3. `--competitors=N` still customizes (range 1..6). Matches the feature description's canonical example (`Kanye vs Drake vs Kendrick`).
### Added
- **`## Resolved Entities` block** in `render_comparison_multi` output. Shows per-entity X handle, subreddits, GitHub user/repos, and truncated context for every entity in the comparison. Block is omitted entirely when no entity has a resolved payload (mock mode, no backend).
## [3.0.11] - 2026-04-22
### Added
- **`--competitors` flag for auto-discovered comparison fan-out.** Pass `--competitors` on a single-entity topic and the engine discovers 2-6 peer entities via web search, then runs the full pipeline on each in parallel and emits one N-way comparison. `last30days Kanye West --competitors` resolves Drake, Kendrick Lamar, and one more peer. `last30days OpenAI --competitors` resolves Anthropic, xAI, Google Gemini. `--competitors=N` controls count, `--competitors-list="A,B,C"` skips discovery and uses the explicit list. Discovery mirrors the `auto_resolve` pattern (Brave / Exa / Serper / Parallel) with deterministic text extraction - no internal LLM call. Sub-runs inherit the main `--quick`/`--deep`/`--days`, run in a `ThreadPoolExecutor`, and degrade gracefully when at least 2 entities survive. Output reuses the existing 9-axis `## Head-to-Head` scaffold.
## [3.0.10] - 2026-04-21
### Added
- **Commenter handles on evidence lines.** Top-comment rendering now includes the commenter's handle - `u/author` for Reddit, `@handle` for TikTok/YouTube/Instagram/Bluesky/X/Threads. The enrichment adapters already captured `author`; the render layer just was not using it. Evidence lines change from `- Comment (6822 upvotes): Finally, John Apple` to `- u/Cyrisaurus (6822 upvotes): Finally, John Apple`. Person-level citations make synthesis-side inline markdown links per LAW 8 much more natural. Both the compact and full render paths are covered.
### Fixed
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
### Behavior fallback
- When an author is empty, `[deleted]`, or `[removed]`, the render falls back to the legacy `Comment (...)` shape - no `u/` or `@` prefix with an empty handle is ever emitted.
## [3.0.9] - 2026-04-18 - The Self-Debug Release
### Highlights
v3.0.9 adds the engine-side Class 1 keyword-trap refuse-gate ("birthday gift for 40 year old" now gets a clarifying question, not 5 minutes of junk), promotes TikTok and YouTube top comments to the same first-class rendering Reddit's got, lands Hermes AI Agent as a first-class deploy target, and moves the SKILL.md formatting contract from line 1094 to the top of the file.
"The Self-Debug Release" refers to how the fixes in 3.0.6-3.0.9 were written: 5 separate Opus 4.7 instances each debugged their own failed outputs. Three converged on "SKILL.md is too big and the LAWs are too deep." Two converged on "the engine should refuse demographic-shopping queries." I shipped exactly what they said. Validation: 5/5 canonical compliance.
### Added
- **Engine Class 1 keyword-trap refuse-gate** (`scripts/lib/preflight.py`, new). Pattern-matches demographic-shopping queries at main() front-door. Exit code 2 with structured REFUSE message. Escape hatch: `LAST30DAYS_SKIP_PREFLIGHT=1`. 29 tests in `tests/test_preflight.py`.
- **TikTok + YouTube top comments** rendered with same `💬 Top comment` prominence as Reddit's. Shipped in [#260](https://github.com/mvanhorn/last30days-skill/pull/260); enrichment fixed in [#265](https://github.com/mvanhorn/last30days-skill/pull/265).
- **Hermes AI Agent as a deploy target** - thanks @stephenmcconnachie ([#228](https://github.com/mvanhorn/last30days-skill/pull/228)). `scripts/sync.sh` detects `~/.hermes/skills/research` and deploys automatically.
- **Multi-key SCRAPECREATORS_API_KEY rotation** - thanks @zaydiscold ([#268](https://github.com/mvanhorn/last30days-skill/pull/268)). Set `SCRAPECREATORS_API_KEY_1`, `_2`, etc. Engine rotates on rate-limit.
- **Offline quality evaluation fixture** - thanks @j-sperling ([#233](https://github.com/mvanhorn/last30days-skill/pull/233)). `eval_topics.json` lets contributors run quality regressions without burning live API credits.
- **END-OF-CANONICAL-OUTPUT boundary** in `render_compact()`. Engine now emits an explicit pass-through instruction so re-synthesis requires actively ignoring a visible boundary.
- **LAW 1 verbatim-pattern override.** LAW 1 now quotes the exact WebSearch tool-result reminder ("CRITICAL REQUIREMENT: MUST include Sources: section") and declares it OVERRIDDEN inside last30days output.
### Changed
- **SKILL.md restructure.** VOICE CONTRACT LAWs and BADGE MANDATORY block moved from line 1094 to lines 75-150. Grounded in 3 separate Opus 4.7 self-debugs.
- **Engine emits the badge as stdout.** `🌐 last30days v3.0.9 · synced YYYY-MM-DD` is the first line of every compact emit. Pass-through is now the default-correct behavior.
- **Reddit client HTTP consolidation** - thanks @iliaal ([#207](https://github.com/mvanhorn/last30days-skill/pull/207)). Migrated to `http.get(params=...)` helper.
- **ScrapeCreators header consolidation** - thanks @iliaal ([#209](https://github.com/mvanhorn/last30days-skill/pull/209)). `_sc_headers` refactored into `http.scrapecreators_headers`.
- **Simpler Hermes sync.** `scripts/sync.sh` Hermes branch now always uses main SKILL.md (previously had a `.hermes-plugin/SKILL.md` fallback that created a wrong-file-capture hazard).
### Fixed
- **Peter Steinberger trailing Sources leak.** 2026-04-18 validation failure where the model appended a TechCrunch / TED / Fortune / Wikipedia Sources list after the invitation. Now structurally prevented at three layers: engine emits the canonical body, LAW 1 quotes the exact WebSearch reminder, closing boundary names the anti-pattern.
- **Wrong-file SKILL.md capture.** Deleted `.agents/skills/last30days/SKILL.md` (1382 lines, April 13 snapshot) and `.hermes-plugin/SKILL.md` (269 lines). One SKILL.md per plugin now, at the plugin root.
- **GitHub date parsing garbage** - thanks @iliaal ([#208](https://github.com/mvanhorn/last30days-skill/pull/208)). `_parse_date` now rejects invalid input cleanly.
- **Windows Bird X stability** - thanks @Chelebii ([#227](https://github.com/mvanhorn/last30days-skill/pull/227)).
- **Linux `check_perms` false-warn** - thanks @george231224 ([#216](https://github.com/mvanhorn/last30days-skill/pull/216)). Uses GNU stat first.
- **UTF-8 saved output** - thanks @Gujiassh ([#225](https://github.com/mvanhorn/last30days-skill/pull/225)).
- **Version metadata alignment** - thanks @Gujiassh ([#217](https://github.com/mvanhorn/last30days-skill/pull/217)) and @shalomma ([#229](https://github.com/mvanhorn/last30days-skill/pull/229)).
- **`--days` alias backcompat** - thanks @BryanTegomoh ([#230](https://github.com/mvanhorn/last30days-skill/pull/230)).
- **`INCLUDE_SOURCES` env default** - thanks @hnshah ([#223](https://github.com/mvanhorn/last30days-skill/pull/223)).
- **Bird X all-None engagement** - thanks @j-sperling ([#234](https://github.com/mvanhorn/last30days-skill/pull/234)).
### Contributors
@j-sperling, @stephenmcconnachie, @zaydiscold, @iliaal, @Chelebii, @Gujiassh, @hnshah, @george231224, @shalomma, @BryanTegomoh for PRs since v3.0.0. @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, @Jah-yee, @dannyshmueli, @Cody-Coyote for issues and PRs that shaped the v3 roadmap.
### Recovery
```
/plugin update last30days
/reload-plugins
```
Verify: `cat ~/.claude/plugins/cache/last30days-skill/last30days/*/.claude-plugin/plugin.json | grep version` returns `"version": "3.0.9"`.
Smoke test: `/last30days birthday gift for 40 year old` should ask a clarifying question before running.
## [3.0.5] - 2026-04-15
### Added
- **`/last30days` slash command for plugin users.** New `commands/last30days.md` registers a Claude Code slash command. Users type `/last30days <topic>` and Claude Code's autocomplete prefix-matches it to the canonical `/last30days:last30days` form (the same way `/ce:plan` resolves to `/compound-engineering:ce-plan`). The command delegates to the existing `last30days` skill body — no skill behavior changes.
### Removed
- **`skills/last30days-nux/`** — byte-identical duplicate of root `SKILL.md` that created confusing `/last30days:last30days-nux` autocomplete entries via Claude Code's plugin namespacing. The root `SKILL.md` remains the canonical skill source.
### Recovery
```
/plugin update last30days
/reload-plugins
```
Then type `/last30days <topic>` to invoke the skill via slash command. Natural-language invocation ("search the last 30 days for X") continues to work unchanged.
## [3.0.4] - 2026-04-15
### Fixed
- **Cleared `/doctor` path-escape error on Claude Code v2.1.109+.** `.claude-plugin/plugin.json` previously declared `"skills": ["./"]`. That value shipped unchanged from v2.1.0 through v3.0.3 and worked on older Claude Code, but current versions reject `./` with `Path escapes plugin directory: ./ (skills)`. The `"skills"` key is now omitted entirely, matching the pattern used by every other plugin in the Claude Code marketplace ecosystem. Claude Code auto-discovers `skills/*/SKILL.md` when the key is absent.
### Recovery
If `/doctor` reports a path-escape error for last30days, run `/plugin update last30days` then `/reload-plugins`. If errors persist, uninstall and reinstall the plugin.
## [3.0.3] - 2026-04-15
### Fixed
- **Restored `skills/` and `.claude-plugin/` to the plugin install tarball.** v3.0.1 added `.gitattributes` rules that excluded both directories from `git archive` output to shrink the claude.ai `.skill` bundle. Claude Code's `/plugin install` fetches the same archive, so users installing v3.0.1 or v3.0.2 received a tarball with no plugin manifest and no skill files. `git archive v3.0.0` contained 8 files under those paths; `v3.0.1` and `v3.0.2` contained 0. This release reverts those `.gitattributes` lines.
- **Reverted `plugin.json` `"skills"` field to `["./"]`.** v3.0.2 changed this to `["skills"]` based on a misdiagnosis — the manifest change had no effect because the manifest wasn't in the tarball at all. The historical `["./"]` value shipped in every release from v2.1.0 through v3.0.0 without issues and is restored here.
### Recovery
Users on v3.0.1 or v3.0.2: run `/plugin update last30days` then `/reload-plugins`. If autoUpdate is enabled, the next session start will pull v3.0.3 automatically. Users on cached v3.0.0 or earlier installs were unaffected.
### Notes
- The claude.ai `.skill` bundle built by `scripts/build-skill.sh` still works — the archive grew from 89 to 97 files, well under the 200-file cap.
- claude.ai-specific exclusions (avoiding duplicate `SKILL.md` files in the bundle) should move into `scripts/build-skill.sh` rather than `.gitattributes` in a future release, since `.gitattributes` cannot distinguish between the two distribution channels.
## [3.0.2] - 2026-04-15
### Fixed
- **`/last30days` slash command now registers on Claude Code v2.1.105+.** `.claude-plugin/plugin.json` declared `"skills": ["./"]`, which newer Claude Code rejects with `Path escapes plugin directory: ./ (skills)`. The skill silently failed to register, so `/last30days <query>` returned "Unknown command" even though `/plugin list` showed the plugin as installed. Fix: `"skills": ["skills"]` so the loader scans the real skill subdirectory.
- **Version drift between manifests.** `.claude-plugin/marketplace.json` was pinned to `3.0.0` while `.claude-plugin/plugin.json` advertised `3.0.1`. The `/plugin` resolver used the marketplace version and could install stale cached metadata alongside the correct build. Both manifests now agree on `3.0.2`.
### Recovery
If `/last30days` stopped working for you, run `/plugin update last30days` then `/reload-plugins`. If `/doctor` still reports errors, uninstall and reinstall the plugin from the marketplace.
## [3.0.1] - 2026-04-14
### Fixed
- **Skill upload packaging** - `scripts/build-skill.sh` produces a claude.ai-upload-ready `.skill` file that fits under the 200-file cap. Previously, zipping the repo hit 406 files and the "Upload skill" UI rejected it outright.
- **SKILL.md description length** - trimmed from 228 to 167 chars (Anthropic caps descriptions at 200).
### Removed
- Unused root `vendor/` directory (215 files from an accidental commit in PR #48 - the real vendored X client lives at `scripts/lib/vendor/bird-search/`).
- Legacy top-level `plans/` directory (superseded by `docs/plans/`; both plans described work that was already shipped in v3).
### Added
- `.gitattributes` with `export-ignore` entries so `git archive` drops tests, docs, fixtures, assets, historical manifests, and internal skill subdirs. Mirrors Anthropic's canonical `package_skill.py` exclusions.
- `scripts/build-skill.sh` - one-command path to produce `dist/last30days.skill` with a single top-level `last30days/` folder, defensive `=200` file check, and dirty-tree refusal.
- `README.md` section documenting the claude.ai skill upload workflow.
## [3.0.0] - 2026-04-11
### Highlights
Intelligent search, fun judge, cross-source cluster merging, single-pass comparisons, and OpenClaw as a first-class citizen. The v3 engine doesn't just search for your topic -- it figures out *where* to search before the search begins. Engine architecture by @j-sperling.
### Added
- **Intelligent pre-research** -- Resolves X handles, subreddits, TikTok hashtags, and YouTube channels via a new Python brain before any API calls fire. Bidirectional: person to company, product to founder.
- **Fun judge / Best Takes** -- Second parallel LLM judge scores humor, cleverness, and virality. Surfaces the best reactions in a dedicated output section.
- **Cross-source cluster merging** -- Entity-based overlap detection merges the same story across Reddit, X, YouTube into one cluster instead of three separate items.
- **Single-pass comparisons** -- "X vs Y" runs one pass with entity-aware subqueries instead of three serial passes. 3 minutes instead of 12+.
- **GitHub as a source** -- Stars, reactions, and comments from repos and issues.
- **OpenClaw first-class citizen** -- Auto-resolve for engine-side pre-research. Device auth for frictionless ScrapeCreators signup.
- **Per-author cap** -- Max 3 items per author prevents single-voice dominance.
- **Entity disambiguation** -- Synthesis trusts resolved handles over keyword matches.
- **Perplexity Sonar Pro as additive source** -- AI-synthesized research with citations via OpenRouter. Opt-in via `INCLUDE_SOURCES=perplexity`. Returns structured narratives that complement social data.
- **Perplexity Deep Research** -- `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query). Premium opt-in for serious investigation.
- **OpenRouter as reasoning provider** -- One OPENROUTER_API_KEY powers planning, reranking, and Perplexity search. Auto-detected after Gemini/OpenAI/xAI.
- **Parallel AI grounding backend** -- `--web-backend parallel` or auto-detected via PARALLEL_API_KEY.
- **Grounding in planner** -- Grounding source properly registered in SOURCE_CAPABILITIES instead of force-injected.
### Changed
- YouTube transcript candidate pool widened 3x past music videos to reach talk/review content with captions
- Reddit comment enrichment sorted by total engagement (upvotes + comments), not just upvotes
- Polymarket display shows % odds only; dollar volumes removed
- 852 tests passing
### Fixed
- Marketplace validation: duplicate `name: last30days` collision in `skills/last30days/SKILL.md` caused strict validators to reject the plugin. Resolved by renaming the internal v3 architecture spec to `last30days-v3-spec` with `user-invocable: false`. Fixed in #214 (reported by @Cody-Coyote in #204).
- Stale README link to the deleted `skills/last30days-v3/` path from the v3 directory rename. Fixed in #214.
- OpenAI Codex CLI discoverability: added `.agents/skills/last30days/SKILL.md` as a real file (Codex's loader skips symlinked files) plus `.codex-plugin/plugin.json` as the namespace marker. The skill now registers as `last30days:last30days` when Codex runs in a checkout of the repo. Fixed in #219 (inspired by @Jah-yee in #153 and @dannyshmueli on X).
### Contributors
- @j-sperling -- v3 engine architecture, Python pre-research brain
- @hnshah -- Watchlist features
- @Cody-Coyote -- Marketplace validation bug report (#204)
- @Jah-yee -- Codex CLI integration inspiration (#153)
## [2.9.4] - 2026-03-06
### Changed
- Move save into Python script via `--save-dir` flag - raw research data saved during the existing script Bash call, zero extra tool calls after invitation
- Remove entire "Save Research to Documents" section from SKILL.md (~45 lines removed)
- No more `📎` footer, no Bash heredoc, no `(No output)`, no multi-minute cogitation after research
## [2.9.3] - 2026-03-06
### Fixed
- **Critical:** Switch save from `run_in_background` to foreground Bash - background callbacks caused model to re-engage, hallucinate fake user messages, and generate unsolicited multi-paragraph responses
- Save uses foreground `cat >` heredoc (executes sub-second, no callback, no delayed notification)
## [2.9.2] - 2026-03-06
### Fixed
- Save research silently using background Bash heredoc instead of Write tool (eliminates "Wrote N lines..." clutter)
- Suppress follow-up text after background save completes (no more "Research briefing saved..." noise)
- Add `📎` footer line for save path instead of verbose confirmation
## [2.9.1] - 2026-03-05
### Highlights
Auto-save research briefings to the default memory directory as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
### Added
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to the default memory directory after every run
- Kebab-case filename generation from topic (e.g., "Claude Code skills" -> `claude-code-skills.md`)
- Duplicate topic handling: appends date suffix instead of overwriting (e.g., `claude-code-skills-2026-03-05.md`)
- Agent mode (`--agent`) also saves research files
- Brief confirmation after save with the saved file path
### Credits
- [@devin_explores](https://x.com/devin_explores) -- Inspired this feature by sharing their workflow of saving every last30days run into organized .md files ([PR #51](https://github.com/mvanhorn/last30days-skill/pull/51))
## [2.9.0] - 2026-03-05
### Highlights
ScrapeCreators Reddit as the default backend (one `SCRAPECREATORS_API_KEY` covers Reddit + TikTok + Instagram), smart subreddit discovery with relevance-weighted scoring, and top comments elevated with 10% scoring weight and prominent display.
### Added
- ScrapeCreators Reddit backend (`scripts/lib/reddit.py`) — keyword search, subreddit discovery, comment enrichment, all via `api.scrapecreators.com`
- Smart subreddit discovery with relevance-weighted scoring: frequency × recency × topic-word match, replacing pure frequency count
- `UTILITY_SUBS` blocklist to filter noise subreddits (r/tipofmytongue, r/whatisthisthing, etc.) from discovery results
- Top comment scoring: 10% weight in engagement formula via `log1p(top_comment_score)`
- Top comment rendering: `💬 Top comment` lines with upvote counts in compact and full report output
- Comment excerpt length increased from 300 → 400 chars; `comment_insights` limit raised from 7 → 10
### Changed
- `primaryEnv` switched from `OPENAI_API_KEY` to `SCRAPECREATORS_API_KEY` — one key now powers Reddit, TikTok, and Instagram
- Reddit engagement scoring formula: `0.55/0.40/0.05` (score/comments/ratio) → `0.50/0.35/0.05/0.10` (score/comments/ratio/top-comment)
- SKILL.md synthesis instructions updated to emphasize quoting top comments
### Fixed
- Utility subreddit noise in discovery (e.g., r/tipofmytongue appearing for unrelated topics)
- Reddit search no longer requires `OPENAI_API_KEY` — ScrapeCreators API handles search directly
## [2.8.0] - 2026-03-04
### Highlights
Instagram Reels as the 8th signal source, TikTok migrated from Apify to ScrapeCreators API, and SKILL.md quality improvements. One API key (`SCRAPECREATORS_API_KEY`) now covers both TikTok and Instagram.
### Added
- Instagram Reels as 8th research source via ScrapeCreators API — keyword search, engagement metrics (views, likes, comments), spoken-word transcript extraction (`scripts/lib/instagram.py`)
- `InstagramItem` dataclass, normalization, scoring (45% relevance / 25% recency / 30% engagement), deduplication, cross-source linking, and rendering
- Instagram in SKILL.md: stats template (`📸 Instagram:`), citation priority, item format description, output footer
- URL-to-name extraction examples in SKILL.md for cleaner web source display
- `--search=instagram` flag support
### Changed
- TikTok backend migrated from Apify to ScrapeCreators API (`api.scrapecreators.com`)
- `APIFY_API_TOKEN` replaced by `SCRAPECREATORS_API_KEY` in config
- SKILL.md version bumped to v2.8
- WebSearch citation instruction strengthened to prevent trailing Sources: blocks
- Security section updated: Apify → ScrapeCreators references
### Fixed
- Web stats line showing full URLs instead of plain domain names
- Trailing "Sources:" block appearing after skill invitation (WebSearch tool mandate conflict)
- Instagram/TikTok not running in web-only mode when `--search=instagram` used without Reddit/X
- `$ARGUMENTS` quoting in SKILL.md for correct flag forwarding
## [2.1.0] - 2026-02-15
### Highlights
Three headline features: watchlists for always-on bots, YouTube transcripts as a 4th source, and Codex CLI compatibility. Plus bundled X search with no external CLI needed.
### Added
- Open-class skill with watchlists, briefings, and history modes (SQLite-backed, FTS5 full-text search, WAL mode) (`feat(open)`)
- YouTube as a 4th research source via yt-dlp -- search, view counts, and auto-generated transcript extraction (`feat: Add YouTube`)
- OpenAI Codex CLI compatibility -- install to `~/.agents/skills/last30days`, invoke with `$last30days` (`feat: Add Codex CLI`)
- Bundled X search -- vendored subset of Bird's Twitter GraphQL client (MIT, originally by @steipete), no external CLI needed (`v2.1: Bundle Bird X search`)
- Native web search backends: Parallel AI, Brave Search, OpenRouter/Perplexity Sonar Pro (`feat(engine)`)
- `--diagnose` flag for checking available sources and authentication status
- `--store` flag for SQLite accumulation (open variant)
- Conversational first-run experience (NUX) with dynamic source status (`feat(nux)`)
### Changed
- Smarter query construction -- strips noise words, auto-retries with shorter queries when X returns 0 results
- Two-phase search architecture -- Phase 1 discovers entities (@handles, r/subreddits), Phase 2 drills into them
- Reddit JSON enrichment -- real upvotes, comments, and upvote ratio from reddit.com/.json endpoint
- Engagement-weighted scoring: relevance 45%, recency 25%, engagement 30% (log1p dampening)
- Model auto-selection with 7-day cache and fallback chain (gpt-4.1 -> gpt-4o -> gpt-4o-mini)
- `--days=N` configurable lookback flag (thanks @jonthebeef, [#18](https://github.com/mvanhorn/last30days-skill/pull/18))
- Model fallback for unverified orgs (thanks @levineam, [#16](https://github.com/mvanhorn/last30days-skill/pull/16))
- Marketplace plugin support via `.claude-plugin/plugin.json` (inspired by @galligan, [#1](https://github.com/mvanhorn/last30days-skill/pull/1))
### Fixed
- YouTube timeout increased to 90s, Reddit 429 rate limit fail-fast
- YouTube soft date filter -- keeps evergreen content instead of filtering to 0 results
- Eager import crash in `__init__.py` that broke Codex environments
- Reddit future timeout (same pattern as YouTube timeout bug)
- Process cleanup on timeout/kill -- tracks child PIDs for clean shutdown
- Windows Unicode fix for cp1252 emoji crash (thanks @JosephOIbrahim, [#17](https://github.com/mvanhorn/last30days-skill/pull/17))
- X search returning 0 results on popular topics due to over-specific queries
### New Contributors
- @JosephOIbrahim -- Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
- @levineam -- Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
- @jonthebeef -- `--days=N` configurable lookback ([#18](https://github.com/mvanhorn/last30days-skill/pull/18))
### Credits
- @galligan -- Marketplace plugin inspiration
- @hutchins -- Pushed for YouTube feature
## [1.0.0] - 2026-01-15
Initial public release. Reddit + X search via OpenAI Responses API and xAI API.
[3.0.9]: https://github.com/mvanhorn/last30days-skill/compare/v3.0.5...v3.0.9
[2.9.1]: https://github.com/mvanhorn/last30days-skill/compare/v2.9.0...v2.9.1
[2.9.0]: https://github.com/mvanhorn/last30days-skill/compare/v2.8.0...v2.9.0
[2.8.0]: https://github.com/mvanhorn/last30days-skill/compare/v2.6.0...v2.8.0
[2.1.0]: https://github.com/mvanhorn/last30days-skill/compare/v1.0.0...v2.1.0
[1.0.0]: https://github.com/mvanhorn/last30days-skill/releases/tag/v1.0.0
CLAUDE.md
+1
View File
@@ -0,0 +1 @@
@AGENTS.md
CONCEPTS.md
+41
View File
@@ -0,0 +1,41 @@
# Concepts
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
## The package
### Skill
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
### Engine
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
### Harness
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
## Research pipeline
### Primary entity
The brand or proper-noun core of a research topic — the topic with its Intent modifier stripped. It is what the research is *about*, as distinct from how the user phrased the search.
### Intent modifier
A trailing word or phrase in a topic that expresses what the user wants to know rather than what the topic is ("review", "use cases", "pricing"). Stripped when deriving the Primary entity.
### Entity grounding
The check that a candidate item plausibly mentions the Primary entity before final ranking. Grounding keys on the head token (first word) of the Primary entity rather than the full phrase — trailing words are usually search descriptors, so requiring them falsely demotes on-entity items.
An item that fails grounding receives a decisive entity-miss demotion, designed so engagement cannot rescue off-entity content. Because the demotion is decisive, the grounding bar is deliberately conservative: its failure modes degrade toward "no penalty," never toward burying on-entity signal.
### Keyless path
The research flow available with no API keys: source data is gathered by scraping and RSS rather than authenticated APIs, and ranking falls back to local scoring instead of LLM-based reranking. This is the free tier of the Skill; lexical quality safeguards like Entity grounding matter most here, because no LLM is available to judge relevance semantically.
### Comment-enrichment slots
The small, depth-dependent budget of Reddit posts whose comments get fetched in the Keyless path. Slot selection is relevance-aware: posts that pass Entity grounding claim slots first, so the budget is not spent on high-engagement posts that final ranking will demote anyway.
CONFIGURATION.md
+565
View File
@@ -0,0 +1,565 @@
# Configuration
Everything you can tune in `/last30days` without editing the engine source.
Three layers, in order of how often you'll touch them:
1. **Per-run flags** - what you pass on the command line.
2. **Environment variables and `.env`** - what's enabled across all runs.
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
Per-client patterns and the experimental beta channel are at the bottom.
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
## Why this document exists
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
---
## Where output is saved
| Platform | Default path | Override |
|---|---|---|
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
Each run produces one file per topic, slug-named:
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
### Recommended `.env` entry
`.env` files don't travel between machines or harnesses, so set `LAST30DAYS_MEMORY_DIR` explicitly in `~/.config/last30days/.env` once per host. The `/last30days` slash command works without it (the SKILL.md wrapper has its own default), but **bare engine invocations** — `python3 scripts/last30days.py ...` from cron jobs, scripts, or agents that bypass the wrapper — silently no-op the file save unless the engine sees the env var. Mirrors the `LAST30DAYS_STORE` env-or-flag convention.
```bash
# ~/.config/last30days/.env (pick ONE — uncomment the line that matches your OS)
LAST30DAYS_MEMORY_DIR=~/Documents/Last30Days # POSIX — defaults to this path when unset
# LAST30DAYS_MEMORY_DIR=C:\Users\<user>\Documents\Last30Days # Windows
# LAST30DAYS_LIBRARY_OWNER=Your Name # Optional Atom feed author
# LAST30DAYS_LIBRARY_CONTEXT=off # Disable prior-run context (default: on)
```
The engine's `.env` reader doesn't expand `$HOME` — only the tilde, via `Path().expanduser()` downstream. Use `~/...` or an absolute path; **don't** write the literal string `$HOME/...` into your `.env` (it gets stored verbatim and breaks path resolution).
**Per-run overrides:**
- `--save-dir <path>` - one-off output location. **Flag wins over env var.** If neither flag nor env var is set, the engine does not write a file (DB persistence is independent — see `LAST30DAYS_STORE` below).
- `--output <file>` - write the rendered output to an exact file path, using the format selected by `--emit`.
- `--json-profile {agent,raw}` - select the research JSON shape used with `--emit=json`. `agent` is the default, versioned workflow contract; `raw` preserves the full internal `Report` dump for debugging and power users. See the [JSON export reference](docs/reference/json-export.md).
- `--corpus <dir>` - add a local `.md`/`.txt` directory as a private ranked source; repeat the flag for multiple directories. PDFs are extracted only when `pdftotext` is on PATH and otherwise skip with a note. File modification time supplies recency, so the normal research window applies.
- `--corpus-all-time` - include relevant registered files whose modification time is older than the current research window. Without this flag, a 30-day run includes only files modified in those 30 days.
- `--register {default,exec,dev,creator,eli5}` - shape a standard single-topic Markdown or HTML research brief for its audience. `exec` is decisions-first with five core findings and numbers up top; `dev` gives GitHub, code, and technical signals more room; `creator` leads with hooks, Best Takes, community reactions, and virality metrics; `eli5` keeps the established evidence layout and asks the synthesizing agent for accessible language. Registers do not change retrieval, JSON exports, discovery, drill, library feed/search, or comparison output.
- `--discover <domain>` - topic-less trending discovery. Sweeps rising/top-week Reddit listings (category-mapped communities, with r/all as the uncategorized fallback), Hacker News front/best stories, Digg AI 1000 clusters when `digg-pp-cli` is on PATH, and broad X activity when an X backend is authenticated, then returns 5-10 engagement-velocity-ranked topics. Run without a positional topic; it is mutually exclusive with `--drill`. `--emit=json` uses the separate versioned discovery contract documented in the [JSON export reference](docs/reference/json-export.md).
- `--drill <target>` - deep follow-up over the fresh `~/.config/last30days/last-report.json` cache. Accepts a 1-based index (`--drill "cluster 3"` or `--drill "3"`) or a fuzzy cluster title/entity description. It re-fetches only sources that contributed to the matched cluster, enables their deep comment/transcript enrichment paths, merges/dedupes the evidence, and replaces the cache so drills can chain. Run it without a positional topic; if the cache is absent or expired, run a normal research pass first.
- `--verify-freshness` - opt into an act-time verification pass for conservatively extracted, source-grounded claims (Polymarket odds/end dates, GitHub stars, StockTwits sentiment ratios, and explicit status assertions). With a topic, verification runs after research; without a topic, it re-verifies the fresh `last-report.json` cache without repeating research. Verdicts are `current`, `stale`, `contradicted`, or `unsupported` and include evidence timestamps. Set `LAST30DAYS_VERIFY_FRESHNESS=on` in `.env` to make the pass default for normal research runs.
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
- `--no-browser-cookies` - hard-disable browser-cookie extraction for this run, even when `FROM_BROWSER` is configured. MCP and folder-mode hosts use this for safe defaults.
- `--publish-html` - with `--emit=html`, publish the rendered HTML to `ht-ml.app` after local output/save-dir writes. This is explicit opt-in only; pages are public by default.
- `library feed` - scan `LAST30DAYS_MEMORY_DIR` plus `~/.local/share/last30days/briefs/`, then write a self-contained `index.html`, valid Atom `feed.xml`, and browser-ready pages under `briefs/`. The index is reverse-chronological and grouped by topic. For direct engine use: `python3 skills/last30days/scripts/last30days.py library feed`; use `--save-dir <path>` to scan and write another library directory.
- `library feed --publish` - publish each rendered brief and the HTML index through `ht-ml.app`. The generated `feed.xml` remains a first-class local artifact because this HTML host does not serve Atom with an XML content type. Host the output directory on any static host (for example, GitHub Pages) to make `feed.xml` subscribable. Publishing is explicit opt-in and pages are public by default; public pages may be crawled or indexed.
- `library search "<query>"` - incrementally sync `LAST30DAYS_MEMORY_DIR` and `~/.local/share/last30days/briefs/` through the shared library scanner, then run offline SQLite FTS5 across those briefs plus dated per-run sightings in `~/.local/share/last30days/research.db`. Results are grouped by topic run. The sibling search index lives at `~/.local/share/last30days/library.db`; hand edits, renames, and deletes are picked up on sync, and a corrupt index is rebuilt automatically.
- `LAST30DAYS_LIBRARY_OWNER=<name>` - optional feed-level Atom author. Defaults to `last30days research library`.
- `LAST30DAYS_LIBRARY_CONTEXT=on|off` - controls passive prior-run context on fresh research reports. It defaults to `on`; matching saved research appears in a short `From your library` section. Set `off` to skip the local index read and leave reports unchanged. Mock runs, eval replays, and internal fan-out subruns do not load library context, keeping fixtures deterministic.
- `--publish-password <password>` - optional shared password for `--publish-html` or `library feed --publish`. Prefer `LAST30DAYS_PUBLISH_PASSWORD=<password>` instead so the password is not visible in the process list or shell history. Use a unique non-personal password; never reuse the user's own password. The provider's update key is treated as secret and is not written to stdout, HTML, raw output, or `.publish.json` metadata.
- `--preflight` - print a human-readable permission preflight. It reports config source, project config trust/ignore state, browser-cookie plan, planned writes, optional commands, source availability, and endpoint overrides without reading browser cookies, writing setup/config/report files, or running research. Add `--emit=json` for the separate machine-readable preflight contract (`--json-profile` does not change it); use `--diagnose` when you need the full source diagnostic JSON.
- `--welcome` - print the first-run welcome text (engine-owned; the skill relays it verbatim on first run). Safe: prints and exits, no reads or writes.
- `--record-fixtures <dir>` - developer-only, hidden flag that records scrubbed source responses for the offline research-quality eval harness. It writes `<dir>/http.json`; see the [eval reference](docs/reference/eval.md) before recording or committing fixtures.
- `setup --github-start` / `setup --github-poll` - the two-command ScrapeCreators GitHub device-auth split. `--github-start` submits the device flow, copies the code to the clipboard, opens the browser, and returns the code immediately (foreground); `--github-poll` waits for you to authorize and persists the key. `setup --github` still runs both in one shot for back-compat.
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
Every completed research pass writes a structured `last-report.json` cache beside `last-run.json`. HTML follow-up renders use it so `--emit=html --synthesis-file` can reuse report metadata/footer without fetching sources again; `--drill <target>` uses it as the grounded starting point for targeted re-research; bare `--verify-freshness` updates only the cached report's claim verdicts. Reuse is intentionally short-lived: `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` defaults to `3600` (one hour). Set it to another integer number of seconds to tune the window, or `0` to disable report-cache reuse and post-run follow-ups.
---
## First-run onboarding
On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It takes one of two forms depending on the host:
- **Claude Code Modal Flow** - the restored v3.0.0 guided NUX, used on hosts with `AskUserQuestion` (Claude Code). A welcome message, then modals for Auto/Manual/Skip setup, cookie consent, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker.
- **Non-Modal Prose Flow** - the same work done conversationally on hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI, Grok, raw CLI).
Both share the same consent points:
1. **Browser cookies** - the model asks before reading anything. On yes it runs `setup --allow-browser-cookies`, which extracts Firefox/Safari cookies (never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser is explicitly configured) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup without `--allow-browser-cookies` (or with `FROM_BROWSER=off`), which skips all cookie reads and still installs the tools.
2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry.
3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, and the SC Reddit/YouTube backups activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. The Step 5 opt-in has two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything**, which also adds Threads + Pinterest. Comments are on by default; Threads and Pinterest are the only opt-in extras.
Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation and both host flows are specified in `skills/last30days/SKILL.md` Step 0. The original v3.0.0 wizard is captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
---
## API keys (`.env`)
The skill reads keys from a `.env` file. Two locations are supported:
1. **`~/.config/last30days/.env`** at the user level (global default) - loaded by default.
2. **`.claude/last30days.env`** in the current project directory (project-scoped) - loaded only when trusted by setting `LAST30DAYS_TRUST_PROJECT_CONFIG=1` in the process environment or global config.
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
The project-scoped file is useful for **intentional per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), then opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or `~/.config/last30days/.env`. Folder-mode hosts such as Codex desktop do not trust hidden project config by default, and discovery stops at the git root so unrelated parent folders cannot silently influence runs.
**`LAST30DAYS_API_KEY`** + **`LAST30DAYS_API_BASE`** - optional remote-API backend. Set BOTH to route research through a remote API endpoint instead of running the local sources: `LAST30DAYS_API_BASE` is the endpoint (there is no built-in default), and `LAST30DAYS_API_KEY` is the bearer key for it. When both are set (and `--mock` is not passed), the engine submits the topic to that endpoint, polls with progress on stderr, and prints the server's report; none of the per-source keys below are used for that run. A configured local corpus is the privacy exception: the engine bypasses the hosted backend and runs locally rather than forwarding file-derived input. Non-default `--register` selections are forwarded with the request so server-side synthesis uses the same audience preset. Leave either unset to run local sources exactly as normal. Unlike the other keys here, these two are read only from the **process environment** (export them in your shell or host config) - they are deliberately not loaded from the `.env` files above, so a project-scoped `.env` can never silently redirect research to a remote endpoint. The remote endpoint does not return the local `Report` needed for the versioned agent JSON profile; use `--emit=json --json-profile=raw` for its existing server-response JSON contract.
### Local corpus (your files)
Register persistent directories with `LAST30DAYS_CORPUS_DIRS`. Separate paths with `:` on macOS/Linux (the platform path separator is `;` on Windows):
```bash
# ~/.config/last30days/.env
LAST30DAYS_CORPUS_DIRS=~/notes:~/meeting-transcripts
# LAST30DAYS_CORPUS_IN_EXPORT=1 # explicit agent-JSON opt-in; off by default
```
The slash-command experience remains primary: ask `/last30days` to include your registered notes. For direct engine scripting or development, the equivalent one-off invocation is:
```bash
python3 skills/last30days/scripts/last30days.py "MCP servers" \
--corpus ~/notes --corpus ~/meeting-transcripts
```
**Privacy:** corpus files are read locally, never sent through a source HTTP client, never forwarded to `LAST30DAYS_API_BASE`, never included in remote reranker/fun-scoring prompts, and do not consume network-source concurrency or retry budget. Matches appear in a badged **From your files** section. Corpus candidates are removed from `--publish-html`, `library feed --publish`, and the versioned agent JSON export by default, including corpus-derived cluster titles and source outcomes. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only when you intentionally want corpus results in the agent JSON written to local stdout/files. The unversioned `--json-profile=raw` debug dump remains a full local report and can contain corpus text; do not redirect it to an external system unless that is intentional. Extracted text is cached by file mtime in `~/.config/last30days/corpus-cache.json` with mode `0600`; a corpus-bearing `last-report.json` cache is also tightened to `0600`. Delete either cache at any time to clear it.
**Source-by-source** - what each key unlocks:
| Source | Key(s) | Required for | Free tier |
|---|---|---|---|
| Local corpus | `--corpus <dir>` or `LAST30DAYS_CORPUS_DIRS` | private `.md`/`.txt`; `.pdf` when `pdftotext` is on PATH | yes (offline) |
| Reddit (public) | none (default); `SCRAPECREATORS_API_KEY` + `LAST30DAYS_REDDIT_BACKEND=scrapecreators` to pin SC primary with public fallback | always on; SC pin requires `SCRAPECREATORS_API_KEY` | yes |
| Hacker News | none | always on | yes |
| Polymarket | none | always on | yes |
| StockTwits | none | auto-on for ticker/crypto topics only (gated by symbol detection); never registered for non-financial topics | yes (public API, ~200 req/hr per IP) |
| DripStack | none | opt-in only: per run with `--search dripstack`, or persistently with `INCLUDE_SOURCES=dripstack` in `.env`. Searches premium financial newsletters and analyst writeups via a free, public search API — no key needed. Never active without the opt-in. | yes when opted in (public API, no auth) |
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
| YouTube | `yt-dlp` CLI installed; `SCRAPECREATORS_API_KEY` adds a server-side transcript fallback used only when yt-dlp fails (429 / bot-gate) | always on if `yt-dlp` present; SC transcript fallback default-on when key set (no credit spent unless yt-dlp fails) | yes |
| YouTube comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `youtube_comments` (**on by default** — written by the Step 5 Recommended tier) | top comments (by likes) on the top ~3 videos by engagement | ~3 calls/run; 10K free calls |
| TikTok comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `digg_count`) on the top ~3 TikTok posts | ~3 calls/run; 10K free calls |
| Instagram comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `comment_like_count`) on the top ~3 Instagram posts, via `/v2/instagram/post/comments` | ~3 calls/run; 10K free calls |
| Digg | `digg-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only`; binary defaults to `$HOME/.local/bin` — Hermes/OpenClaw agent subprocesses must inherit that dir on PATH for Digg to activate; prior pp-digg installs use the same path) | always on if `digg-pp-cli` on PATH | yes (free, keyless, read-only) |
| arXiv | `arxiv-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install arxiv --cli-only`) | always on if `arxiv-pp-cli` on PATH; fires on research/technical topics and stays quiet otherwise (relevance + 365-day recency gating) | yes (free, keyless) |
| Techmeme | `techmeme-pp-cli` on PATH (auto-installed via `... install techmeme --cli-only`) | always on if `techmeme-pp-cli` on PATH; searches Techmeme's live archive and keeps only headlines dated within the research window (undated headlines flow through as low-confidence) | yes (free, keyless) |
| Trustpilot | `trustpilot-pp-cli` on PATH (NOT auto-installed; install on demand via `npx -y @mvanhorn/printing-press-library@0.1.16 install trustpilot --cli-only`) + `INCLUDE_SOURCES` contains `trustpilot` | **opt-in, off by default**; when enabled, activates only on company/brand topics — or on any topic when `--trustpilot-domain=<domain>` pins the review page explicitly (bypasses the brand-shape gate; also the per-entity `trustpilot_domain` key in `--competitors-plan`). Bare company names auto-resolve to the review-page domain via the CLI's search. The session warms once before the search fan-out; a stale session does a ~10s headless-Chrome WAF-cookie harvest (set `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` to disable in cron/CI) | yes (no API key; cookie-replay after the one-time harvest) |
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `XQUIK_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; Xquik / xAI / ScrapeCreators = key-based |
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
| LinkedIn | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `linkedin` | LinkedIn posts + articles (articles rank as high signal on person topics) | 10K free calls; power-user opt-in, not offered during first-run onboarding |
| Xiaohongshu (RED) | logged-in x-mcp browser plugin or `xiaohongshu-mcp` service; optional `XIAOHONGSHU_API_BASE` for custom URLs | requested-only via `--search xhs` or `--search xiaohongshu`; auto-probes `http://localhost:18060` then `http://host.docker.internal:18060` | no last30days API key; depends on your local browser-session service |
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
| Perplexity Sonar / Search API / Deep Research | `PERPLEXITY_API_KEY` (preferred) or `OPENROUTER_API_KEY` (Sonar fallback) | `INCLUDE_SOURCES=perplexity`; `--deep-research` flag (~$0.90/query) | no |
| Caption-free transcription | `GROQ_API_KEY` (free tier, preferred) or `OPENAI_API_KEY` (paid backstop); requires `ffmpeg` | Whisper transcription for audio/video without captions (groundwork: module shipped, not yet auto-invoked by the engine) | Groq free tier is generous; needs ffmpeg installed |
| Jobs / careers pages | none for public ATS pages; web backend improves fallback discovery | `--hiring-signals` and strong Hiring Signals in standard company reports | yes |
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
**X on cookie-less hosts.** Bird (the free X source) scrapes X using your logged-in browser cookies (`AUTH_TOKEN`/`CT0`), which agent hosts like OpenClaw, CI, or headless runs often can't supply — and scraping carries some account risk. On those, set `XQUIK_API_KEY` (or `XAI_API_KEY`) for full, ranked X coverage from a single API key: the same engagement-based ranking, first-party authorship, and handle (from/mentions) lanes the native X source gets. `--diagnose` reports whether the key is working (and flags an unpaid key).
**Example `.env` skeleton** (placeholders only - replace with your own values):
```bash
# Reasoning + planning (one provider; see priority below)
GOOGLE_API_KEY=<your-gemini-key>
# Web search backend (one is enough; Brave is the cheapest)
BRAVE_API_KEY=<your-brave-key>
# Optional sources
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
INCLUDE_SOURCES=tiktok,instagram
# Xiaohongshu is requested-only: run with --search xhs after starting a local
# browser-session service. Defaults probe localhost, then host.docker.internal.
# XIAOHONGSHU_API_BASE=http://localhost:18060
# Add perplexity to INCLUDE_SOURCES when you want the paid Perplexity source.
# PERPLEXITY_API_KEY=<your-perplexity-key>
# INCLUDE_SOURCES=tiktok,instagram,perplexity
# LAST30DAYS_PERPLEXITY_MODE=sonar # sonar | search | both
# LAST30DAYS_PERPLEXITY_MODEL=sonar-pro # sonar | sonar-pro | sonar-reasoning-pro
# X authentication (one option only)
AUTH_TOKEN=<your-auth-token>
CT0=<your-ct0-token>
# OR xAI API key (paid)
# XAI_API_KEY=<your-xai-key>
# OR Xquik key-based X search
# XQUIK_API_KEY=<your-xquik-key>
# OR cookie-jar (free; logs in via your browser session).
# Unset = no browser-cookie reads. FROM_BROWSER=auto tries Firefox/Safari and
# the Chromium family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it
# only prompts for macOS Keychain access on the browser that actually holds your
# X cookies. Or name a single browser, e.g. brave/edge. On Windows only Firefox
# is supported.
# FROM_BROWSER=firefox
# Bluesky
BSKY_HANDLE=<your-handle>.bsky.social
BSKY_APP_PASSWORD=<your-app-password>
```
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --preflight` for a human permission summary or `python3 scripts/last30days.py --diagnose` for full JSON diagnostics. Both are safe: they report source availability, config source, browser-cookie plan, external command availability, write destinations, and ignored untrusted project config without reading browser cookies or running live provider probes.
### Perplexity source modes
Perplexity is a paid opt-in source. A direct `PERPLEXITY_API_KEY` unlocks first-party Perplexity features. `OPENROUTER_API_KEY` remains a Sonar compatibility fallback only; Perplexity Search API and async Deep Research call Perplexity directly.
`LAST30DAYS_PERPLEXITY_MODE` controls normal `perplexity` source runs:
| Value | Behavior | Calls |
|---|---|---|
| `sonar` (default) | Sonar synthesis plus citations. | one Sonar call |
| `search` | Raw ranked Search API rows; best when you want source aggregation over prose. | one Search API call |
| `both` | Sonar synthesis plus raw ranked Search API rows, deduped by URL. | one Search API call and one Sonar call |
`--deep-research` ignores `LAST30DAYS_PERPLEXITY_MODE` and uses `sonar-deep-research`. With `PERPLEXITY_API_KEY`, it submits to Perplexity's async Sonar endpoint and polls with a hard wall-clock timeout. The async request uses a deterministic idempotency key derived from the request body. If the request is still running at timeout, fails remotely, or polling hits a transport/rate-limit error after the async id exists, the raw artifact records the async request id, idempotency key, last status, lifecycle timestamps returned by Perplexity, poll count, and timeout/error fields so you can inspect or resume by id outside the run. With only `OPENROUTER_API_KEY`, it keeps the OpenRouter synchronous fallback.
Perplexity-specific env vars:
| Env var | Default | Applies to | Notes |
|---|---|---|---|
| `LAST30DAYS_PERPLEXITY_MODE` | `sonar` | normal Perplexity source runs | `sonar`, `search`, or `both`; `search` and `both` require `PERPLEXITY_API_KEY`. |
| `LAST30DAYS_PERPLEXITY_MODEL` | `sonar-pro` | direct Sonar only | Supported: `sonar`, `sonar-pro`, `sonar-reasoning-pro`. `--deep-research` forces `sonar-deep-research`. |
| `LAST30DAYS_PERPLEXITY_MAX_RESULTS` | `10` | Search API | Clamped to Perplexity's 1..20 range. |
| `LAST30DAYS_PERPLEXITY_SEARCH_CONTEXT_SIZE` | provider default | Search API | `low`, `medium`, or `high`; omitted unless set. |
| `LAST30DAYS_PERPLEXITY_SEARCH_MODE` | provider default | direct Sonar | `web`, `academic`, or `sec`. |
| `LAST30DAYS_PERPLEXITY_DOMAIN_FILTER` | unset | Search API and direct Sonar | Comma-separated domains, max 20. |
| `LAST30DAYS_PERPLEXITY_LANGUAGE_FILTER` | unset | Search API and direct Sonar | Comma-separated ISO 639-1 language codes, max 20. |
| `LAST30DAYS_PERPLEXITY_COUNTRY` | unset | Search API | Two-letter country code such as `US`. |
| `LAST30DAYS_PERPLEXITY_RECENCY_FILTER` | unset | Search API and direct Sonar | `hour`, `day`, `week`, `month`, or `year`. |
| `LAST30DAYS_PERPLEXITY_REASONING_EFFORT` | unset | direct Sonar | `minimal`, `low`, `medium`, or `high`. |
| `LAST30DAYS_PERPLEXITY_DEEP_TIMEOUT_SECONDS` | `600` | direct async Deep Research | Wall-clock polling deadline. |
### Encrypted credential sources (Keychain / pass)
If you'd rather not keep keys in a plaintext `.env`, the loader has two
encrypted sources that decrypt secrets transiently at call time (never written
to disk, never logged). Both are **lowest-priority and additive** — an explicit
`.env` or process-env value always overrides them, so you can mix and match. The
`pass` source is only consulted for keys still missing after the higher-priority
sources, so a box that merely has `pass` installed pays no decrypt cost when
everything is already in `.env`.
Effective credential priority is: process env > trusted project config
(`.claude/last30days.env`) > global config (`~/.config/last30days/.env`) >
macOS Keychain > `pass`(1). The SessionStart status hook also checks for
Keychain item **presence** under `last30days-<KEY>` without reading secret
values, so a Keychain-only setup is treated as configured instead of showing the
first-run welcome again.
| Platform | Source | Store keys with | Lookup convention |
|---|---|---|---|
| macOS | Keychain | `scripts/setup-keychain.sh` | service name `last30days-<KEY>` |
| Linux / Unix (anywhere `pass` exists, incl. macOS) | [`pass`(1)](https://www.passwordstore.org/) | `scripts/setup-pass.sh` | pass path `last30days/<KEY>` |
```bash
# macOS Keychain
./scripts/setup-keychain.sh # interactive; --list / --delete KEY
# pass(1) — Linux/Unix analog
./scripts/setup-pass.sh # interactive; --list / --delete KEY
./scripts/setup-pass.sh SCRAPECREATORS_API_KEY # just one key
```
The `pass` source honors `PASSWORD_STORE_DIR`. If your store organizes secrets
under a different prefix, point the loader at it with `LAST30DAYS_PASS_PREFIX`
(works from your `.env` too, and must match where `setup-pass.sh` wrote them).
The prefix is used verbatim, so keep the trailing separator:
```bash
export LAST30DAYS_PASS_PREFIX="secrets/last30days/" # default: last30days/
```
Both sources cover the same key set as the `.env` skeleton above.
#### Reusing existing macOS Keychain items
If you already have keys stored under another Keychain naming convention, you
can reference them without copying the secret by setting non-secret alias
metadata in `LAST30DAYS_KEYCHAIN_ALIASES`. The loader still checks
`last30days-<KEY>` first; aliases are fallback lookups only.
```bash
# ~/.config/last30days/.env
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":{"account":"keychain-user","service":"existing-xai-api-key"},"BRAVE_API_KEY":"existing-brave-api-key"}
```
Each JSON key must be one of the supported env-var names (`XAI_API_KEY`,
`SCRAPECREATORS_API_KEY`, `BRAVE_API_KEY`, etc). A string value means "use this
service name with the current user account"; an object can specify both
`account` and `service`. Lists are allowed for fallback order:
```bash
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":[{"account":"keychain-user","service":"existing-xai-api-key"},{"service":"last-resort-xai"}]}
```
The alias value contains no secret material; it is safe to keep in `.env` as
configuration. The secret itself remains in its original Keychain item and is
read directly by the engine process.
Write `LAST30DAYS_KEYCHAIN_ALIASES` as a single-line JSON value in `.env`.
Multiline JSON formatting is not supported because `.env` files are parsed
line-by-line.
### Bluesky app-password format and search host
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
```bash
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
```
### Default source set (`LAST30DAYS_DEFAULT_SEARCH`)
By default the engine decides the source set per query (everything available, minus `EXCLUDE_SOURCES`). To pin a **fixed** source set for every run without passing `--search` each time — and without patching `SKILL.md`, which a release would overwrite — set:
```bash
LAST30DAYS_DEFAULT_SEARCH=reddit,x,youtube,hn
```
Accepts the same comma-separated names and aliases as `--search` (`web` → grounding, `hn` → hackernews, `bsky` → bluesky, `xhs` → xiaohongshu). Precedence: an explicit `--search` on the command line always wins; `LAST30DAYS_DEFAULT_SEARCH` applies only when the flag is omitted; when neither is set, per-query behavior is unchanged. `INCLUDE_SOURCES` / `EXCLUDE_SOURCES` keep their existing additive/subtractive roles on whichever set is selected.
### Audience register (`LAST30DAYS_REGISTER`)
The default standard brief stays balanced and byte-compatible with prior releases. To keep a named audience preset across runs, set one of the supported values:
```bash
LAST30DAYS_REGISTER=exec # default | exec | dev | creator | eli5
```
An explicit `--register` wins over `LAST30DAYS_REGISTER`; the environment/config value defaults to `default`. Presets are intentionally named and bounded - arbitrary prompt or template files are not accepted. Existing `ELI5_MODE=true` configurations continue to resolve to the `eli5` register when no explicit register is selected, but new configuration should use `LAST30DAYS_REGISTER=eli5`.
---
## Reasoning provider priority
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
2. **OpenAI** - `OPENAI_API_KEY` only. Codex ChatGPT auth at `~/.codex/auth.json` is intentionally not used as an OpenAI provider credential.
3. **xAI** - `XAI_API_KEY`
4. **OpenRouter** - `OPENROUTER_API_KEY` (Sonar fallback for the Perplexity source / `--deep-research`; also usable as a reasoning provider)
5. **Local / deterministic** - always available, lowest quality
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
---
## Web search backend priority
The search-source preference ladder, strict best-to-floor:
1. **Host web search** - whatever web-search capability the agent session already has: built-in search, a deferred web-search tool that must be loaded first, or an installed connector such as Brave, Firecrawl, Exa, Serper, or another provider. Best results; used automatically on hosts that have it. A failed lookup for one specific tool name is not fatal when another web-search capability is available. Signalled to the engine via `LAST30DAYS_NATIVE_SEARCH=1` (the skill sets this for you when your agent session has web search) so the engine does not run a worse search underneath it.
2. **Paid engine backend** - one of `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY`, auto-detected in that order. Override per-run with `--web-backend=<name>`.
3. **Keyless engine floor** - zero-key web search (DuckDuckGo, plus an optional SearXNG instance) and zero-key page fetch (Jina Reader). Runs only when the agent session has **no** host web search **and** no paid key is set, so headless/cron and hosts without a search tool still get general-web coverage. Force it explicitly with `--web-backend=keyless`.
Relevant env vars:
| Var | Effect |
| --- | --- |
| `LAST30DAYS_NATIVE_SEARCH=1` | Tells the engine your agent session has host-side web search; suppresses the keyless floor. Set automatically by the skill when web search is available. Leave unset when the agent has no web-search tool so the floor runs. |
| `LAST30DAYS_SEARXNG_URL=<base-url>` | Optional. A SearXNG instance used as the keyless-search fallback rung when DuckDuckGo returns nothing. |
| `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` | Optional. Truthy value disables the Trustpilot source's one-time headless-Chrome WAF-cookie harvest, so an automated/headless run (cron, CI, the eval harness) never spawns a browser. Trustpilot still degrades to empty gracefully. |
Privacy note: the keyless floor sends the query (to DuckDuckGo / your SearXNG instance) and any fetched URL (to Jina Reader) to those third parties. It is intended for public-research use; results may be cached snapshots. It never runs when native search or a paid backend is in play.
Visible quality difference between hosts with vs without native search or a configured backend. If your client setup produces thinner results than yours, this is usually why.
---
### `--hiring-signals` flag
Use `--hiring-signals` for a focused company hiring-signal report:
```bash
python3 skills/last30days/scripts/last30days.py "Listen Labs" --hiring-signals
```
The engine treats public jobs/careers postings as evidence of focus or priority shifts, not exact roadmap predictions. Standard company runs may include Hiring Signals automatically when multiple current roles support the same interpretation; weak or unavailable hiring evidence is omitted.
---
## Health check (`doctor`)
One command answers "what's broken, what's serving, and what do I run to fix it" — per source: a rollup tier (ok / warn / off / error), the specific probe state, the backend the next run will use (for chained sources), and an exact fix on any non-ok tier:
```bash
python3 skills/last30days/scripts/last30days.py doctor # grouped text report
python3 skills/last30days/scripts/last30days.py doctor --json # machine contract
python3 skills/last30days/scripts/last30days.py doctor --cached # serve the cached report while fresh
```
Slash-command form: `/last30days doctor`. Reporting problems is a successful run — the exit code is always 0, no browser cookies are read, no network calls are made, and no secret values appear anywhere (key presence is booleans only). Backends within a chained source are probed sequentially with a 5-second budget per binary probe, so a chained source's worst-case check time is additive across its backends (only reached when several binaries hang at once).
Every live run writes its JSON result to `~/.config/last30days/doctor-cache.json` (beside `last-run.json`; honors `LAST30DAYS_CONFIG_DIR`). `doctor --cached` returns that stored report when it is younger than the TTL, and falls through to a live run — rewriting the cache — when it is stale, absent, or corrupt. The cache also self-invalidates on configuration change: the payload carries a schema stamp plus a fingerprint of non-secret config signals (which credentials are present as booleans, the `LAST30DAYS_X_BACKEND` / `LAST30DAYS_REDDIT_BACKEND` pin values, and `INCLUDE_SOURCES`), so adding or removing a key, changing a pin, or toggling an opt-in source makes the next `--cached` call run live — no raw secret ever enters the fingerprint or the file. Every report also carries `from_cache` (true/false) and `generated_at` (when the report was built), in the `--json` top level and as a final `generated: … (cached|live)` text line, so you can always tell how old a cached answer is. A failed cache write is never fatal — doctor prints a one-line stderr warning and continues. An explicit `doctor` without `--cached` always runs live and refreshes the cache.
| Var | Effect |
| --- | --- |
| `LAST30DAYS_DOCTOR_TTL` | Freshness window for `doctor --cached`, in **seconds**. Defaults to `900` (15 minutes). `0` makes every `--cached` call run live. |
| `LAST30DAYS_X_BACKEND` | Pins the X backend (`xai` / `bird` / `xurl` / `xquik`); doctor renders the pin and predicts "will use" accordingly. |
| `LAST30DAYS_REDDIT_BACKEND` | `scrapecreators` makes ScrapeCreators the primary Reddit backend; doctor renders Reddit's conditional routing with the pin applied. |
Web search has **no** env pin — pin it per-run with `--web-backend=<name>` only (see [Web search backend priority](#web-search-backend-priority)).
### Strict exit for degraded runs
By default a research run exits `0` even when a source failed mid-run (rate-limited, auth-failed, unreachable, timeout, schema-drift) — the report still renders, with the failure annotated in the per-source footer and a partial-coverage warning. Wrappers that need to distinguish degraded coverage from success (cron briefs, CI, downstream agents) can opt in:
| Var | Effect |
| --- | --- |
| `LAST30DAYS_STRICT_EXIT` | Truthy (`1`/`true`/`yes`/`on`): the engine exits `3` when any source outcome is neither `ok`, `no-results`, nor `skipped-unconfigured`. A one-line `strict-exit: degraded sources: ...` note goes to stderr. Default (unset): exit `0`, unchanged behavior. |
Exit codes with the flag on: `0` clean run, `3` completed-but-degraded (report was produced), non-zero others unchanged (hard failures). Same hybrid pattern as `LAST30DAYS_DEBUG` — works shell-exported or in `.env`.
---
## Trend monitoring (`--store` + watchlist + briefings)
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
### `--store` flag
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
### `watchlist.py` - recurring topics
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
```bash
# 1. Add the topic to the watchlist
# Default schedule daily 8am; --weekly switches to Mondays 8am
python3 scripts/watchlist.py add "british airways middle east" --weekly
# 2. Configure delivery and budget (optional)
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
python3 scripts/watchlist.py config budget 5.00
# 3. Trigger via cron / Task Scheduler / GitHub Actions
python3 scripts/watchlist.py run-one "british airways middle east"
# or run every enabled topic, gated by daily_budget
python3 scripts/watchlist.py run-all
```
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
### `briefing.py` - daily / weekly digests
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
### Recommended cadence pattern
| Step | Cadence | Command |
|---|---|---|
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
---
## Per-client patterns
The skill is built to flex around different client environments. Four patterns that compose well:
**Codex note:** the repository includes `.codex-plugin/plugin.json` so Codex can treat the existing
`skills/last30days/SKILL.md` tree as plugin metadata without maintaining a separate Codex copy.
The Codex marketplace catalog points at the repository root URL: Codex clones the repo, reads the
root `.codex-plugin/plugin.json`, and loads skills from `./skills/`. The Agent Skills install
command documented in the README remains the broadest cross-host path.
**Grok note:** the repository includes `.grok-plugin/plugin.json` and `.grok-plugin/marketplace.json`
so xAI's Grok Build CLI (`grok`) can install last30days as a native plugin. Grok also reads the
Claude Code manifests for compatibility; the native pair is the first-class lane. The Grok
marketplace catalog uses a bare Git URL source (no commit pin) so `grok plugin marketplace add
mvanhorn/last30days-skill` tracks HEAD — the same pattern as the Codex catalog. `npx skills add`
remains a valid cross-host fallback.
### 1. Trusted per-client `.claude/last30days.env`
When each client has its own working directory, drop a `.claude/last30days.env` into the client folder and opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or global `~/.config/last30days/.env`. The skill loads the project file only after that trust signal. Typical contents:
```bash
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
INCLUDE_SOURCES=tiktok,instagram
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
```
`cd` into the client folder, run `/last30days <topic>` as normal, no wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
### 2. Per-client save dir + suffix wrapper
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
PowerShell example:
```powershell
function Run-L30D-Client {
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
/last30days @Args --save-suffix=$ClientSlug
}
# Usage: Run-L30D-Client acme "british airways middle east"
```
Bash example:
```bash
l30d-client() {
local client=$1; shift
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
/last30days "$@" --save-suffix="$client"
}
# Usage: l30d-client acme "british airways middle east"
```
### 3. Custom category-peer subreddits
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
### 4. Pre-built `--competitors-plan` JSON
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
```json
{
"Competitor B": {
"x_handle": "competitor_b_handle",
"subreddits": ["sub1", "sub2"],
"github_user": "competitor-b-org",
"context": "Founded 2019, focused on ..."
},
"Competitor C": { ... }
}
```
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
---
## Beta channel
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
---
## Cross-references
- The CLI flag surface: `python3 scripts/last30days.py --help`
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
- Shared package vocabulary and engine/harness terminology: [`CONCEPTS.md`](CONCEPTS.md)
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
CONTRIBUTORS.md
+60
View File
@@ -0,0 +1,60 @@
# Contributors
last30days is built by [@mvanhorn](https://github.com/mvanhorn) with help from the community.
## v3 Inspiration
These contributors submitted PRs and issues that directly inspired v3 features. The v3 engine was a ground-up rewrite, so their original code wasn't merged, but their ideas shaped what shipped.
Want to claim your entry? Submit a PR replacing the placeholder line below your name with your bio, website, or anything you'd like.
---
### @uppinote20
[PR #143](https://github.com/mvanhorn/last30days-skill/pull/143) - Rich Reddit comments, top 3 per post
v3 ships top comments with upvote counts on every thread.
> _Add your bio, website, or anything you'd like here._
### @zerone0x
[Issue #134](https://github.com/mvanhorn/last30days-skill/issues/134) + [PR #136](https://github.com/mvanhorn/last30days-skill/pull/136) - GitHub as a first-class data source
v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos with live star counts.
> _Add your bio, website, or anything you'd like here._
### @thinkun
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
### @thomasmktong
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
v3 Reddit is 100% pure Python with zero external dependencies.
> _Add your bio, website, or anything you'd like here._
### @fanispoulinakisai-boop
[Issue #100](https://github.com/mvanhorn/last30days-skill/issues/100) - Reddit timeout report
Drove the timeout resilience work that made v3 Reddit bulletproof.
> _Add your bio, website, or anything you'd like here._
### @pejmanjohn
[Issue #78](https://github.com/mvanhorn/last30days-skill/issues/78) - ScrapeCreators silent failures
v3 surfaces all API errors with clear diagnostics instead of silently returning empty results.
> Repping the mighty MI; home of the most cracked agentic engineers. https://github.com/pejmanjohn
### @zl190
[PR #115](https://github.com/mvanhorn/last30days-skill/pull/115) - HN trending merge
v3 merges trending and keyword HN results with deduplication for better coverage.
> Healthcare AI engineer. [Blog](https://zl190.github.io/blog)
### @hnshah
[PR #84](https://github.com/mvanhorn/last30days-skill/pull/84), [#85](https://github.com/mvanhorn/last30days-skill/pull/85), [#86](https://github.com/mvanhorn/last30days-skill/pull/86) - Watchlist delivery, 90-day scanning window, HN/Polymarket storage
v3 has durable watchlist with multi-source storage and extended time windows.
> Hiten Shah. Founder. Builds in public. https://github.com/hnshah
---
## Past Contributors
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); .env permission auto-fix ([#573](https://github.com/mvanhorn/last30days-skill/pull/599)); MCP Go tests in CI ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
- [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
- [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
- [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback
HERMES_SETUP.md
+115
View File
@@ -0,0 +1,115 @@
# Hermes Setup Guide for last30days
This guide covers installing last30days on Hermes AI Agent.
## Prerequisites
1. **Hermes installed** - See https://github.com/NousResearch/hermes-agent
2. **Python 3.12+** - `brew install python@3.12` or similar
3. **yt-dlp** (optional, for YouTube) - `brew install yt-dlp`
## Installation
```bash
hermes skills install mvanhorn/last30days-skill/skills/last30days --force
```
The explicit `skills/last30days` path fetches the skill straight from this repo's current default branch and deploys it under `~/.hermes/skills/`. `--force` is required because Hermes's install-time security scanner returns a `caution` verdict for this skill — it flags benign patterns such as reading your own API keys from the environment and calling `subprocess` to run `yt-dlp`/`bird`. `--force` accepts the caution verdict and installs (it also reinstalls over any existing copy).
**Why the explicit path?** The shorter `hermes skills install mvanhorn/last30days-skill` currently resolves through the skills.sh index, which is serving an older cached snapshot of this repo (from before the skill moved under `skills/last30days/`). Use the explicit `.../skills/last30days` path above until the index re-crawls — tracked in [vercel-labs/skills#1602](https://github.com/vercel-labs/skills/issues/1602).
### Developer / live-edit alternative
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
```bash
git clone https://github.com/mvanhorn/last30days-skill.git
mkdir -p ~/.hermes/skills/research
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
```
## Usage
In Hermes, invoke with:
```
last30days "your research topic"
```
Or with options:
```
last30days "best mechanical keyboards 2025" --search=reddit,youtube
last30days "AI news" --days=7 --deep
```
## First Run Setup
On first run, the skill will guide you through setup:
1. **Auto setup** (~30 seconds)
- Scans browser cookies for X/Twitter
- Checks/installs yt-dlp for YouTube
- Best-effort install of `digg-pp-cli` for Digg AI-news clusters (via `@mvanhorn/printing-press-library`; binary lands in `$HOME/.local/bin` — ensure your Hermes gateway PATH includes it, or Digg stays off even after install)
- Configures free sources (Reddit, HN, Polymarket)
2. **Optional: ScrapeCreators**
- Adds TikTok, Instagram, Reddit backup
- 100 free credits (no expiration)
- Sign up at scrapecreators.com
3. **Optional: API Keys**
- XAI_API_KEY for X/Twitter (alternative to browser cookies)
- BRAVE_API_KEY for web search
## Available Sources
### Free (No API Key)
- **Reddit** - Public discussions and comments
- **Hacker News** - Tech discussions via Algolia
- **Polymarket** - Prediction markets
- **YouTube** - Search and transcripts (requires yt-dlp)
- **Digg** - AI-news story clusters (requires `digg-pp-cli` on the agent PATH; auto-installed to `$HOME/.local/bin` during setup when `npx` is available)
### Requires API Key
- **X/Twitter** - xAI API key or browser cookies
- **TikTok** - ScrapeCreators API
- **Instagram** - ScrapeCreators API
- **Web Search** - Brave Search API
## Troubleshooting
### Python not found
```bash
# Find Python 3.12+
which python3.12 python3.13 python3.14
# If not installed
brew install python@3.12
```
### yt-dlp not found
```bash
brew install yt-dlp
# or
pip install yt-dlp
```
### Check what's configured
```bash
cd ~/.hermes/skills/research/last30days
python3.12 scripts/last30days.py --diagnose
```
## Updating
```bash
hermes skills install mvanhorn/last30days-skill --force
```
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
## Support
- Original repo: https://github.com/mvanhorn/last30days-skill
- Hermes: https://github.com/mercurial-tf/hermes
- Issues: Please report in the original repo
LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Matt Van Horn
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+380
View File
@@ -0,0 +1,380 @@
# /last30days
<p align="center">
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
</p>
<p align="center">
<a href="https://github.com/mvanhorn/last30days-skill">
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
</a>
<br/>
<a href="https://trendshift.io/repositories/21997" target="_blank">
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
</a>
</p>
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
**Claude Code (recommended — auto-updates via marketplace):**
```
/plugin marketplace add mvanhorn/last30days-skill
/plugin install last30days
```
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
```
npx skills add mvanhorn/last30days-skill -g
```
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, arXiv, Techmeme, and more in 30 seconds.
---
Reddit upvotes. X likes. YouTube transcripts. TikTok engagement. Polymarket odds backed by real money and insider information. That's millions of people voting with their attention and their wallets every day. /last30days searches all of it in parallel, scores it by what real people actually engage with, and an AI agent judge synthesizes it into one brief.
Google aggregates editors. /last30days searches people.
You can't get this search anywhere else because no single AI has access to all of it. Google search doesn't touch Reddit comments or X posts. ChatGPT has a deal with Reddit but can't search X or TikTok. Gemini has YouTube but not Reddit. Claude has none of them natively. Each platform is a walled garden with its own API, its own tokens, its own auth. But you can bring your own keys and browser sessions, and suddenly an AI agent can search all of them at once, score them against each other, and tell you what actually matters.
That's the unlock. Not one better search engine. A dozen disconnected platforms, bridged by an agent.
```
/last30days Peter Steinberger
```
You have a meeting tomorrow. You Google them. You get their LinkedIn from 2023. /last30days gives you what they're actually doing this month: joined OpenAI to work on Codex, fighting Anthropic's ban on third-party agents, shipping 23 PRs at 85% merge rate, building "LobsterOS" for cross-device agent control, and r/ClaudeCode hit 569 upvotes debating whether he's a hero or "insufferable." Scattered across X posts, Reddit threads, YouTube transcripts, and GitHub commits. None of it was on Google.
## Why this exists
I built it to keep up in AI. Everything changes every day and the Reddit and X nerds are always on top of it first. I needed better prompts, and the training data was always months behind what the community had already figured out.
But it turned into something bigger. Now I run it before a sales call to know the last 30 days truth about a business. Before a meeting to read someone's recent tweets and podcast transcripts. Before a Disney World trip to know which rides are closed and what the community says about Genie+. Before I build anything to know what problems people are actually hitting.
If you're meeting with a CEO, have you read all their tweets and YouTube transcripts from the last 30 days? I have.
## Sources, scored by the people
| Source | What the people tell you |
|--------|--------------------------|
| **Reddit** | The unfiltered take. Top comments with real upvote counts, free, no API key. The real opinions that Google buries. |
| **X / Twitter** | The hot take, the expert thread, the breaking reaction. First to know, first to argue. |
| **YouTube** | The 45-minute deep dive. Full transcripts searched for the 5 quotable sentences that matter. |
| **TikTok** | The creator reaching 3.6M people with a take you'll never find on Google. |
| **Instagram Reels** | The influencer perspective with spoken-word transcripts. The visual culture signal. |
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
| **arXiv** | The papers behind the hype. New research in the window, free, no API key. Auto-enabled when `arxiv-pp-cli` is on PATH (first-run setup installs it). |
| **Techmeme** | The tech-news editorial layer, date-windowed to your 30 days. Free, no API key. Auto-enabled when `techmeme-pp-cli` is on PATH (first-run setup installs it). |
| **LinkedIn** | The professional signal. Posts and articles, with articles weighted as high signal. |
| **StockTwits** | Trader sentiment. Auto-activates when your topic is a ticker or crypto. |
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
| **Xiaohongshu (RED)** | Chinese lifestyle, product, and creator signals. Requested explicitly with `--search xhs` when a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service is running locally. |
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
| **Perplexity** | Grounded Sonar synthesis, raw Search API rows, and Deep Research. |
| **Web** | The editorial coverage, the blog comparisons. One signal of many, not the only one. |
Community contributors keep adding more. Truth Social and other niche sources are in the engine with more on the way.
A Reddit thread with 1,500 upvotes is a stronger signal than a blog post nobody read. A TikTok with 3.6M views tells you more about what's culturally relevant than a press release. Polymarket odds backed by $66K in volume are harder to argue with than a pundit's guess.
The synthesis ranks by what real people actually engaged with. Social relevancy, not SEO relevancy.
## What people actually use it for
**Before a meeting.** `/last30days Peter Steinberger` - joined OpenAI's Codex team, fighting Anthropic's ban on third-party agents, 23 PRs merged at 85% merge rate on GitHub, building LobsterOS for cross-device agent control. r/ClaudeCode: "Ever since OpenClaw released, it was widely known that if you run it through anything other than the API, you were gonna get banned eventually" (227 upvotes). That's not on LinkedIn.
**To read hiring signals.** `/last30days Listen Labs --hiring-signals` - current jobs and careers pages become cited evidence for focus shifts: hiring into enterprise security, customer success, infrastructure, or product expansion. The report says what the hiring appears to signal, not what the roadmap will ship.
**To find the topic before it peaks.** Ask `/last30days what's exploding in AI agents?` and the skill switches to discovery mode: it sweeps Reddit category listings, Hacker News front/best stories, Digg's AI 1000 feed, and X when authenticated, then returns 5-10 engagement-velocity-ranked topics. Every result includes cross-source numbers, a momentum label, and a ready-to-run `/last30days "<topic>"` follow-up.
**When something drops.** `/last30days Kanye West` - UK blocked his visa, Wireless Festival canceled, sponsors fled. But BULLY debuted #2 on Billboard. Fantano came back from his "Yay sabbatical" to review it (653K views). SoFi Homecoming brought out Lauryn Hill and Travis Scott for 44 songs. Polymarket: "Will Kanye tweet again?" 86% Yes. 23 Reddit threads, 17 YouTube videos, 86K upvotes.
**To compare tools.** `/last30days OpenClaw vs Hermes vs Paperclip` - "These aren't competitors, they're layers." OpenClaw is the executor (351K GitHub stars, live), Hermes is the self-improving brain (31K stars), Paperclip is the org chart (49K stars). Star counts pulled live from the GitHub API, not stale blog posts. Side-by-side table with architecture, memory, security, best-for. Per @IMJustinBrooke: "OpenClaw = Charmander, Hermes = Charizard."
**To understand the world.** `/last30days Iran vs USA` - Day 38 of the war. Trump's Tuesday deadline for Iran to reopen the Strait of Hormuz. Two US warplanes downed. Oil at $126/barrel. The IEA called it "the largest supply disruption in the history of the global oil market." Polymarket: ceasefire by Dec 31 at 74%. 27 X posts, 10 YouTube videos, 20 prediction markets.
**Before a trip.** `/last30days Universal Epic Universe` - Expansion already under construction. "Project 680" permit filed. Fireworks show confirmed by infrastructure but unannounced. Wait times: Mine-Cart Madness averaging 148 minutes. No annual pass yet, and locals are frustrated. Stardust Racers down for refurbishment through April 5.
**To learn something fast.** `/last30days Nano Banana Pro prompting` - JSON-structured prompts are replacing tag soup. @pictsbyai's nested format prevents "concept bleeding." Edit-first workflow beats regeneration. Then it writes you a production prompt using exactly what the community said works.
## What's new
Since the v3.3 announcement in May, as of v3.11.1 (July 2026): 175 merged PRs - 122 of them from 52 community contributors - across 15 releases. This is what landed.
### First-class on OpenAI Codex
/last30days is now a native Codex plugin with guided setup - not a port, a first-class citizen. Renderer-aware citations mean Codex output reads like a brief instead of URL soup (#694), and the same engine runs on Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw, and 50+ Agent Skills hosts. Codex plugin manifest by [@rfoust](https://github.com/rfoust) (#686), Codex auth fix by [@tmchow](https://github.com/tmchow) (#698).
### arXiv, Techmeme, and Digg - free, no API keys
arXiv brings the papers behind the hype and Techmeme brings the editorial tech-news layer - free, zero keys, and first-run setup installs their CLIs so they activate automatically (#709). Digg's AI 1000 story clusters arrive without X auth the same way - setup installs the free Digg CLI for you (#590). Trustpilot ships opt-in for consumer-brand research.
### Free Reddit grew real scores and top comments
Reddit's public .json API died; the free path came back stronger. Keyless RSS + shreddit scraping (#457), dedicated-subreddit discovery with real upvote counts via arctic-shift (#696), and a relevance floor so a viral off-topic post can't hijack your brief (#488, thanks [@rzachsmith](https://github.com/rzachsmith)). No API key. Real scores. Top comments included.
### The best comments in every brief
Comments are now a default-on layer across sources: Instagram comments with rank-based diversity so five hot takes don't all come from one post (#751), YouTube comments plus a ScrapeCreators transcript backup for when yt-dlp strikes out (#637), and crowd-voted comments weighted into Best Takes so the community's funniest lines survive scoring (#592, #608).
### One doctor command
Ask for a health check and the doctor runs every source, then prescribes exact fixes - which key is missing, which CLI is off PATH, which cookie expired (#753). No more guessing why X came back thin.
### X search, rebuilt
The X pipeline got a ground-up overhaul: FROM and ABOUT lanes so a person's own posts and the conversation about them both rank (#610), person-aware subquery disambiguation (#611), first-party authorship grounding with interaction-signal ranking (#613), and a single X source with automatic backend failover (#622). Plus an honest `--diagnose` that actually probes auth (#609).
### More sources joined
LinkedIn via ScrapeCreators, with articles as high signal ([@ravstr](https://github.com/ravstr), #702). StockTwits auto-activates for ticker and crypto topics ([@wtiwana](https://github.com/wtiwana), #658). Perplexity grew direct API modes and async Deep Research ([@sk-holmes](https://github.com/sk-holmes), #629).
### Hardened by the community
The security wave was almost entirely community work: stored-XSS fixes in the HTML renderer ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), locked-down cookie temp files, supply-chain-hardened CI with OpenSSF Scorecard and build provenance attestation ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), Semgrep and OSV-Scanner scans plus a PR dependency-review gate ([@23241a6749](https://github.com/23241a6749)), a test-coverage floor introduced at 60% and since raised to 84% ([@gourab5139014](https://github.com/gourab5139014)), and a Hermes security scan cleared of every CRITICAL finding (#768).
### Reaches further
Hebrew and non-Latin languages ([@dudyme](https://github.com/dudyme)). CJK-aware tokenization for Chinese sources ([@An-idd](https://github.com/An-idd)). A Windows compatibility wave. Cookie extraction across the full Chromium family - Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) - plus macOS Keychain and Linux pass(1) credential sources. `--as-of` historical lookback ([@chiyi-creator](https://github.com/chiyi-creator)). Auto-provisioned Python 3.12 via uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` for reading a company's job pages. Watchlist deltas between runs.
### Still in the box from v3
The v3 foundations are all still here: the pre-research brain that resolves the right handles, subreddits, and hashtags before a single API call fires (built by [@j-sperling](https://github.com/j-sperling)); Best Takes scoring for humor and virality alongside relevance; cross-source cluster merging; single-pass comparisons ("CLI vs MCP" in 3 minutes, not 12); auto-discovered `--competitors` comparisons; GitHub person-mode (`--github-user=steipete`); ELI5 mode ("eli5 on" after any run); and shareable, self-contained HTML briefs (`--emit=html`). Configuration knobs live in [CONFIGURATION.md](CONFIGURATION.md).
## Install
| Surface | Install | Updates |
|---------|---------|---------|
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` then `grok plugin install last30days` | `grok plugin update last30days` |
| **Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via claude.ai > Customize > Skills > + > Create skill > Upload a skill | Re-download and re-upload |
| **Claude Desktop** | [Download the `.mcpb` for your platform](https://github.com/mvanhorn/last30days-skill/releases/latest) and drag into Settings > Extensions | Re-download and drag the new bundle in |
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
### Claude Code (recommended)
```
/plugin marketplace add mvanhorn/last30days-skill
```
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
```
npx skills add mvanhorn/last30days-skill -g -a claude-code
```
The native plugin and the `npx skills` install can coexist. Note that Claude Code does not dedupe across install methods: if you have both the marketplace plugin and the `npx skills` copy active, `/last30days` will show two entries. Use one install method per machine.
### Grok (xAI Build CLI)
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) installs last30days as a native plugin. Direct install tracks the repository:
```bash
grok plugin install mvanhorn/last30days-skill
```
Or add this repo as a marketplace source, then install by plugin name:
```bash
grok plugin marketplace add mvanhorn/last30days-skill
grok plugin install last30days
```
Add `--trust` to skip the install confirmation. Update with `grok plugin update last30days`. Grok also reads the Claude Code manifests for compatibility; the native `.grok-plugin/` pair is the first-class lane (and what an official [xAI marketplace](https://github.com/xai-org/plugin-marketplace) listing points at). `npx skills add` remains a valid cross-host fallback.
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
```bash
npx skills add mvanhorn/last30days-skill -g
```
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
Codex desktop and other folder-mode hosts can work in ordinary folders as well as Git repos. Before first research, ask the host agent to run the bundled `scripts/last30days.py --preflight` from the loaded skill directory; in a source checkout, the equivalent command is `python3 skills/last30days/scripts/last30days.py --preflight`. It shows the config source, browser-cookie plan, planned writes, optional commands, and ignored project config without reading cookies, writing files, or running research.
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
```bash
npx skills add mvanhorn/last30days-skill -g -a codex
npx skills add mvanhorn/last30days-skill -g -a cursor
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
```
Update later with:
```bash
npx skills update last30days -g
```
Or update everything you've installed globally via `npx skills`:
```bash
npx skills update -g
```
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
### claude.ai (web)
1. [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) from the latest release
2. Go to [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
3. Click the `+` button in the Skills panel > click on `Create skill` > `Upload a skill` and browse/drop the file in
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
### Claude Desktop
Claude Desktop installs `/last30days` as an MCP server via a `.mcpb` bundle (a one-click Model Context Protocol package).
1. Go to the [latest release](https://github.com/mvanhorn/last30days-skill/releases/latest) and download the `.mcpb` for your platform:
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
2. Open Claude Desktop, go to Settings > Extensions, and drag the file in.
3. When prompted, paste API keys for the sources you want to enable. Every field is optional — the engine degrades to web-only mode if you skip them all. Keys are stored in your OS keychain.
4. Restart Claude Desktop. Ask Claude to "research Peter Steinberger" or any topic and it will call the `research` tool.
**Host requirement:** Python 3.12+ on PATH. The bundle ships the engine source but uses your local Python interpreter. Install from [python.org](https://www.python.org/downloads/) on Windows; macOS and most Linux distros ship a compatible version.
**Keys don't sync with the Code skill.** Claude Desktop and Claude Code maintain separate credential stores by design. If you already configured `~/.config/last30days/.env` for the Code skill, you'll re-enter the same keys here once.
Windows support is deferred until per-platform manifest entry points are sorted out; track in a follow-up issue.
### OpenClaw
```bash
clawhub install last30days-official
```
For X/Twitter action workflows outside `/last30days` research, such as posting
tweets or replies, follower export, media handling, monitors, and giveaway
draws, use [TweetClaw](https://github.com/Xquik-dev/tweetclaw) as the companion
OpenClaw plugin. TweetClaw is maintained by Xquik-dev and is listed only as an
optional companion path, not a last30days dependency or endorsement.
### Manual (developer)
```bash
git clone https://github.com/mvanhorn/last30days-skill.git
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
```
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds, including the free arXiv and Techmeme CLIs.
## Bring your own keys
These platforms don't have relationships with each other. X doesn't know what Reddit thinks. YouTube doesn't see TikTok. But you can bring your own API keys and browser tokens, and suddenly you have access to all of them at once.
| Sources | What you need | Cost |
|---------|---------------|------|
| Reddit (with comments) + HN + Polymarket + GitHub + StockTwits | Nothing | Free |
| arXiv + Techmeme | Free CLIs, auto-installed by first-run setup | Free |
| X / Twitter | Log into x.com in any browser, or set `XQUIK_API_KEY` / `XAI_API_KEY` | Browser cookies are free; keys are provider-specific |
| YouTube | `brew install yt-dlp` | Free |
| Bluesky | App password from bsky.app | Free |
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube comments | ScrapeCreators key | 10,000 free calls, then PAYG |
| Xiaohongshu (RED) | Run a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service and opt in with `--search xhs` per run or `INCLUDE_SOURCES=xiaohongshu` in `.env`; last30days auto-probes `http://localhost:18060` then `http://host.docker.internal:18060`, or use `XIAOHONGSHU_API_BASE` for a custom URL | No last30days API key; depends on your local browser-session service |
| DripStack (premium financial newsletters) | Opt-in: `--search dripstack` per run, or `INCLUDE_SOURCES=dripstack` in `.env` | No key; free public search API |
| Perplexity Sonar / Search API / Deep Research | Perplexity key, or OpenRouter key as Sonar fallback | Pay as you go |
| Web search | Brave Search key | 2,000 free queries/month |
### macOS Keychain (optional)
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
```bash
# Interactive setup — prompts for each known key, skip with empty input
skills/last30days/scripts/setup-keychain.sh
# Or store a single key by hand
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
# Inspect / clean up
skills/last30days/scripts/setup-keychain.sh --list
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
```
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
Already have keys under different Keychain service names? Set the non-secret `LAST30DAYS_KEYCHAIN_ALIASES` mapping described in [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) instead of copying secrets.
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
## Configuration
Two things you'll likely want to know on day one:
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--output <file>` when you need the rendered result at an exact path, using the format selected by `--emit`. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each `--save-dir` run produces `<slug>-raw[-suffix].md`. Run `python3 skills/last30days/scripts/last30days.py --preflight` to review planned writes before a research run.
**Structured output for agents and workflows.** Ask `/last30days` for machine-readable JSON to receive the stable, versioned agent profile. For direct engine use in scripts or development, run `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`; add `--json-profile=raw` only when you need the unversioned internal `Report` dump. See the [JSON export field reference and versioning policy](docs/reference/json-export.md).
**Topic-less discovery.** Ask `/last30days what's trending in AI agents?` to get a ranked discovery brief instead of researching a topic you already know. For direct engine use in scripts or development, run `python3 skills/last30days/scripts/last30days.py --discover "AI agents"`; add `--emit=json` for the versioned discovery contract. Discovery is mutually exclusive with a positional topic and `--drill`.
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
**A subscribable research library.** Ask `/last30days` to build your library feed, or use `python3 skills/last30days/scripts/last30days.py library feed` directly for scripting and development. It turns saved briefs into `index.html`, a local Atom `feed.xml`, and readable brief pages. Add `--publish` only when you want the HTML index and brief pages hosted; publishing is explicit opt-in and public by default. To make the Atom feed subscribable, host the generated output directory on a static host such as GitHub Pages.
**Search everything you've researched.** Ask `/last30days search my library for MCP servers` or `/last30days have I researched MCP servers before?`. For direct engine use, run `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. Search is offline and deterministic: it incrementally indexes the same saved briefs used by the library feed, merges matching per-run store sightings, and groups results by topic and date. Fresh runs also surface a compact **From your library** section when prior research overlaps the current topic; set `LAST30DAYS_LIBRARY_CONTEXT=off` to disable that passive context.
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
## Showcase: community research feeds
Published a recurring AI update, market watch, or wonderfully narrow obsession with last30days? Share the public library URL—or the Atom URL after hosting `feed.xml` on a static host—in [the community showcase thread](https://github.com/mvanhorn/last30days-skill/issues/532). Community feeds will be linked here as their owners submit them; the thread is the collection point in the meantime.
## How it works
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
2. **The agent resolves who matters.** Finds X handles (including founders), GitHub repos, subreddits, TikTok hashtags, YouTube channels. For "Kanye West" it knows r/hiphopheads, @kanyewest, and "bully review" on YouTube. For "OpenClaw" it resolves openclaw/openclaw on GitHub and fetches live star counts.
3. **All sources searched in parallel.** Multi-query expansion. Results scored by engagement, relevance, freshness.
4. **The depth nobody else has.** Full YouTube transcripts from reaction videos. Top Reddit comments with upvote counts. TikTok captions. Polymarket odds. Not just titles and links.
5. **Same story, merged.** Wireless Festival announced on Reddit, discussed on X, ticket prices on TikTok = one cluster, not three separate items.
6. **Synthesized into one brief.** Grounded in specific data. Cited by source. Ranked by what people actually engage with. Not "here's what I found." It's "here's what matters."
7. **Then it becomes your expert.** After one run, your Claude session knows everything the community knows. Ask follow-up questions. Have it write prompts, draft emails, plan trips, architect systems - all grounded in what's real right now.
## What people are saying
> "I found a Claude Code skill that researches any topic across Reddit, X, YouTube, and HN from the last 30 days. Then writes the prompts for you. I've been manually searching Reddit and X for research before every piece of content I write. Tab by tab. Thread by thread. That's the part that takes 90 minutes. This eliminates it." -@itsjasonai
> "This one skill replaced my entire research workflow. You give it a topic, it scrapes Reddit, X, and the web for what people are actually talking about. Not old blog posts. Real conversations from the last 30 days." -@itswilsoncharles
> "5 of the 10 trending repos on GitHub today are Claude tools. #1: mvanhorn/last30days-skill" -@yieldhunter95
## Open source
MIT license. No tracking. No analytics. Your research stays on your machine. 2,700+ tests.
Built with Python 3.12+, yt-dlp, Node.js (vendored Bird client for X search), and ScrapeCreators API. v3 engine architecture by [@j-sperling](https://github.com/j-sperling).
See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list of community contributors and [CHANGELOG.md](CHANGELOG.md) for version history.
## Star History
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
</picture>
</a>
---
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
README.wehub.md
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目：`mvanhorn/last30days-skill`
- 原始仓库：https://github.com/mvanhorn/last30days-skill
- 导入方式：上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
docs/how-search-works.md
+196
View File
@@ -0,0 +1,196 @@
# How Reddit & X Search Work in last30days
## Architecture Overview
```
User: /last30days "kanye west"
↓
┌─────┴─────┐
↓ ↓ (concurrent via ThreadPoolExecutor)
[REDDIT] [X/TWITTER]
↓ ↓
OpenAI Bundled Bird or
API xAI API
↓ ↓
Parse Parse
↓ ↓
Enrich ───┘
(fetch ↓
actual [MERGE]
upvotes) ↓
↓ [NORMALIZE → FILTER → SCORE → DEDUPE]
└───────────↓
[OUTPUT to SKILL.md agent]
```
Both searches run **in parallel** using Python's `ThreadPoolExecutor(max_workers=2)`.
---
## Reddit Search
### How it works
Reddit search uses the **OpenAI Responses API** with the `web_search` tool, domain-filtered to `reddit.com` only.
**API Call:**
```
POST https://api.openai.com/v1/responses
Authorization: Bearer {OPENAI_API_KEY}
```
**Payload:**
```json
{
"model": "gpt-5.2",
"tools": [{
"type": "web_search",
"filters": { "allowed_domains": ["reddit.com"] }
}],
"input": "Search Reddit for threads about {topic}..."
}
```
The prompt asks the model to:
1. Extract core subject (strip noise words like "best", "tips", "top")
2. Search 3 patterns: `"{topic} site:reddit.com"`, `"reddit {topic}"`, `"{topic} reddit"`
3. Return JSON with `title`, `url`, `subreddit`, `date`, `relevance`
4. URLs must contain `/r/` AND `/comments/` (real threads only)
**Model fallback chain:** `gpt-5.2 → gpt-5.1 → gpt-5 → gpt-4.1 → gpt-4o → gpt-4o-mini`
Triggers on HTTP 400/403 with access error keywords.
### Enrichment (the secret sauce)
After search, each thread gets **enriched** by hitting Reddit's free JSON API:
```
GET https://reddit.com/r/{sub}/comments/{id}/{slug}/.json
```
No API key needed. This returns the actual thread data:
| Data Point | Source |
|---|---|
| Upvotes (score) | Reddit JSON API |
| Comment count | Reddit JSON API |
| Upvote ratio | Reddit JSON API |
| Top 10 comments (text + score) | Reddit JSON API |
| 7 key comment insights | Extracted via heuristics |
| Actual post date | `created_utc` timestamp |
**This is why Reddit results have real engagement metrics** — the enrichment step fetches actual upvote/comment data, not AI estimates.
### Depth settings
| Depth | Threads requested | Timeout |
|---|---|---|
| `--quick` | 15-25 | 90s |
| default | 30-50 | 120s |
| `--deep` | 70-100 | 180s |
---
## X/Twitter Search
X search has **two backends** — the skill auto-detects which to use.
### Priority: Bundled Bird (env auth) → xAI API (paid)
```python
if node_available and AUTH_TOKEN and CT0:
use bundled Bird # Free, popup-free, env-authenticated
elif XAI_API_KEY:
use xAI API # Paid, uses grok-4-1-fast
else:
skip X entirely # No X results
```
### Backend 1: xAI API
**API Call:**
```
POST https://api.x.ai/v1/responses
Authorization: Bearer {XAI_API_KEY}
```
**Payload:**
```json
{
"model": "grok-4-1-fast",
"tools": [{ "type": "x_search" }],
"input": "Search X for posts about {topic} from {from_date} to {to_date}..."
}
```
The prompt asks grok to return JSON with:
- `text`, `url`, `author_handle`, `date`
- `engagement`: `{ likes, reposts, replies, quotes }`
- `why_relevant`, `relevance` score
**Engagement data comes from grok's x_search tool** - it has direct access to X's data.
### Backend 2: Bundled Bird client (free alternative)
The repo vendors a search-only subset of Bird's Twitter GraphQL client and shells out to it with Node.js. No global `bird` install is required. The Python wrapper passes `AUTH_TOKEN` and `CT0` via env, which keeps normal local runs headless and avoids browser-cookie prompts.
**Bundled Bird returns raw X API data** - likes, reposts, replies are real engagement metrics from X's API, not estimates.
| Metric | Bundled Bird | xAI API |
|---|---|---|
| Post text | Real | Real |
| Likes/reposts | Real (X API) | Real (x_search tool) |
| Replies/quotes | Real | Real |
| Author handle | Real | Real |
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
### Depth settings
| Depth | xAI posts | Bundled Bird results | xAI timeout | Bird timeout |
|---|---|---|---|---|
| `--quick` | 8-12 | 12 | 90s | 30s |
| default | 20-30 | 30 | 120s | 45s |
| `--deep` | 40-60 | 60 | 180s | 60s |
---
## Post-Processing (both sources)
After both searches complete:
1. **Normalize** — consistent formatting, timezone handling
2. **Date filter** — hard filter to requested date range
3. **Score** — relevance scoring (engagement-weighted)
4. **Sort** — highest scores first
5. **Deduplicate** — remove duplicate URLs
6. **Fallback** — if all items filtered out, keep top 3 by relevance
---
## Error Handling
| Layer | Strategy |
|---|---|
| HTTP requests | 3 retries with exponential backoff (1s → 2s → 3s) |
| Model access errors | Automatic fallback to next model in chain |
| Reddit enrichment | Per-item try/catch; keeps unenriched item on failure |
| X source detection | Silent fallback from Bird → xAI → skip |
| Overall pipeline | Errors stored as `reddit_error`/`x_error`, shown to user |
---
## Key Files
| File | Purpose |
|---|---|
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
docs/pr-credits.md
+33
View File
@@ -0,0 +1,33 @@
# PR Credits — Thank After V2 Goes Live
When V2 is pushed to the public repo, comment on each PR to thank the contributor and let them know their work was integrated.
## Integrated (cherry-picked into V2)
| PR | Author | What | Status |
|---|---|---|---|
| [#17](https://github.com/mvanhorn/last30days-skill/pull/17) | **@JosephOIbrahim** | Windows Unicode fix (cp1252 emoji crash) | Merge or close with thanks |
| [#16](https://github.com/mvanhorn/last30days-skill/pull/16) | **@levineam** | Handle 403 model access errors + gpt-4.1 fallback | Merge or close with thanks |
| [#18](https://github.com/mvanhorn/last30days-skill/pull/18) | **@jonthebeef** | `--days=N` configurable lookback flag | Merge or close with thanks |
| [#1](https://github.com/mvanhorn/last30days-skill/pull/1) | **@galligan** (Matt Galligan) | Marketplace plugin conversion — we took a lighter approach inspired by his PR | Close with thanks, explain lighter approach |
## Already Fixed in V2 (close with thanks)
| PR | Author | What |
|---|---|---|
| [#15](https://github.com/mvanhorn/last30days-skill/pull/15) | **@rszrszrsz** | YAML argument-hint fix — already fixed in V2 |
| [#11](https://github.com/mvanhorn/last30days-skill/pull/11) | **@nerveband** | Same YAML fix (earlier) — already fixed in V2 |
## Not Integrated (close with explanation)
| PR | Author | What | Why |
|---|---|---|---|
| [#5](https://github.com/mvanhorn/last30days-skill/pull/5) | **@jblwilliams** | Codex auth with OpenAI Responses API | Good idea, too complex for now (358 lines SSE/JWT). May revisit. |
| [#14](https://github.com/mvanhorn/last30days-skill/pull/14) | **@thangman1** | WebSearch-first, API keys optional | Philosophical shift — V2 already does WebSearch in parallel |
| [#10](https://github.com/mvanhorn/last30days-skill/pull/10) | **@thetechreviewer** | OpenRouter API integration | Too large (1029 lines), adds MCP server |
## Suggested Comment Template
> Thanks for this PR! We integrated your [fix/feature] into V2 (commit XXXXX). Really appreciate the contribution. 🙏
>
> Closing this PR since the changes are now in main via a different commit, but full credit to you for the idea and implementation.
docs/reference/eval.md
+87
View File
@@ -0,0 +1,87 @@
# Research-quality eval harness
The eval suite measures the quality properties that ordinary unit tests do not: whether ranked evidence is grounded in retrieved inputs, stays inside the requested window, forms coherent clusters, accounts for every usable fixture source, and remains deterministic.
It runs the production pipeline offline. Recorded HTTP exchanges replay at `lib/http.py`; CLI-backed adapters such as yt-dlp, Digg, arXiv, Techmeme, and Trustpilot replay their parsed result at the source-module seam. Planning is supplied by each fixture manifest, and normalization, date filtering, scoring, fusion, clustering, source outcomes, and the versioned agent JSON export all run normally. The harness never calls an LLM or the network.
## Run it
From the repository root:
```bash
uv run pytest tests/eval -x -s
```
The `-s` keeps the score table visible. To print only the scored run and return a nonzero exit when a floor is missed:
```bash
uv run python tests/eval/harness.py
```
CI runs the pytest command in the `eval` job of `.github/workflows/validate.yml`, so every pull request gets a score table and a hard baseline check.
## Metrics
| Metric | Deterministic definition |
|---|---|
| Citation grounding | Fraction of exported result URLs that occur in the recorded fixture inputs. |
| Recency compliance | Fraction of ranked source items whose known publication date is inside the report's inclusive date window. Undated evidence is not falsely classified as stale. |
| Cluster coherence | Fraction of within-cluster candidate pairs meeting the production entity-overlap threshold (`0.45`). Singleton clusters are coherent by definition. |
| Coverage | Fraction of fixture sources represented by usable report items or an explicit `Report.source_status` outcome. |
| Determinism | `schema.to_dict()` equality for two runs with fixed time and identical recorded inputs. |
Aggregate floors live in `tests/eval/baseline.json`. The fixture matrix covers a tech product, a person, a comparison, breaking events, a niche technical topic, and a non-English CJK topic.
## Add or refresh a fixture
Fixture directories contain:
- `manifest.json`: topic archetype, fixed `as_of_date`, sources, safe dummy config, and a deterministic external query plan.
- `http.json`: scrubbed HTTP exchanges and any CLI-backed source exchanges.
Use the direct engine invocation below only for development/fixture capture; `/last30days <topic>` remains the product interface:
```bash
python3 skills/last30days/scripts/last30days.py \
"<topic>" \
--quick \
--as-of 2026-07-10 \
--search grounding,hackernews \
--plan /tmp/eval-plan.json \
--record-fixtures tests/eval/fixtures/<fixture-name>
```
`--record-fixtures` is intentionally hidden from `--help`. It records the live run's shared HTTP traffic and the bounded CLI-adapter seams, scrubs credential-shaped query/body/response fields, and writes `http.json`. It does not create the manifest because archetype, fixed date, source contract, and query plan are review decisions.
Before committing a recording:
1. Inspect `http.json` for cookies, keys, tokens, personal identifiers, and unnecessary long bodies.
2. Truncate content to the smallest structure that exercises the adapter and pipeline.
3. Replace irrelevant real usernames with obvious fixture identities.
4. Add the manifest and run both commands above with networking unavailable.
The replay is fail-closed: an unrecorded request or an unused recorded exchange fails the run.
## Fixture flags
- `expects_clusters` (bool): fixtures whose topic historically forms multi-member clusters set this true; if cluster formation regresses to singletons on such a fixture, coherence scores 0.0 instead of a vacuous 1.0. Sparse topics (niche, non-english-cjk, tech-product) set it false because singletons are their legitimate shape.
- Post-ranking enrichment (YouTube transcripts, Digg posts) is recorded and replayed by merging recorded `metadata` onto freshly computed items by item_id, so normalization/scoring/dedupe regressions stay visible to the eval rather than being overwritten by fixture state.
- Post-rerank GitHub star enrichment records its repo->stars map and replays via `github.apply_star_map`, keeping runs offline even when `GITHUB_TOKEN` is set in CI. GitHub project-mode (`--github-repo`) and person-mode (`--github-user`) runs are not yet fixture-recordable; the network guard fails loudly if a fixture attempts them.
## Known seams
- Module-backed sources (yt-dlp, digg-pp-cli and other CLI adapters) record post-parse items at the module boundary, so replay does not re-exercise their parsing/normalization code the way HTTP-backed sources do (those replay raw responses through the real pipeline). A normalization regression in a module adapter is covered by that adapter's unit tests, not the eval. Recording raw CLI stdout is a possible future upgrade.
- Cluster coherence shares `entity_extract` with production clustering. The pinned-predicate test (`test_entity_overlap_predicate_pinned`) guards against the shared predicate drifting permissive, and per-fixture floors in baseline.json catch a single archetype collapsing even when the cross-fixture average stays green.
## Move a baseline
Baseline edits are explicit quality-policy changes, not snapshot refreshes. Move a floor only when an intentional product change makes the old threshold invalid or when a new fixture legitimately changes the measured distribution.
Include in the review:
1. The old and new score tables.
2. The reason the metric changed.
3. A focused test proving the intended behavior.
4. An explanation for any lower floor; never lower a floor solely to make CI green.
`test_intentional_out_of_window_regression_fails_recency_floor` is the standing negative control: it injects stale ranked evidence and proves the baseline check detects the regression.
docs/reference/json-export.md
+132
View File
@@ -0,0 +1,132 @@
# Agent JSON export
The agent JSON profile is the stable machine-readable research contract for downstream agents, scripts, dashboards, and workflow tools. Ask the slash command for machine-readable JSON:
```text
/last30days AI coding agents — return the versioned agent JSON export
```
For direct engine use in scripts, cron jobs, or development, use:
```bash
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --output results.json
```
`--emit=json` defaults to `--json-profile=agent`. The full internal report remains available for debugging and power users:
```bash
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --json-profile=raw
```
The raw profile is intentionally unversioned and may change when pipeline internals change. It preserves the JSON serialization used before the agent profile was introduced.
### Local corpus privacy
Evidence from `--corpus` / `LAST30DAYS_CORPUS_DIRS` is excluded from the versioned agent profile by default. The exclusion removes corpus results, corpus-only clusters, corpus source outcomes, freshness verdicts, and titles derived from a corpus representative. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only for a run whose JSON is intentionally allowed to contain local file contents. This opt-in does not change the schema shape or version; it permits `source: "corpus"` entries in the existing result fields. The unversioned `raw` profile is a complete local debug dump and may contain corpus paths and text.
## Discovery export
Discovery mode has a separate versioned contract so its topic results do not change the normal research export:
```bash
python3 skills/last30days/scripts/last30days.py --discover "AI agents" --emit=json
```
Its top level contains `schema_version` (`1.0`), `kind` (`"discovery"`), `domain`, `generated_at`, `window_days`, `source_status`, `feeds`, `results`, and `warnings`. Each ranked result contains `rank`, `topic`, `why_spiking`, `momentum` (`new-this-week` or `building`), `velocity_score`, `sources`, per-source native `engagement`, a ready-to-run `command`, and `evidence_urls`. The discovery contract follows the same versioning policy below but evolves independently of the normal agent export. `--json-profile=raw` returns the unversioned internal `DiscoveryReport` dataclass instead.
When `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` route a run through a configured remote API, the server does not return the local `Report` needed to build this profile. In that mode, `--json-profile=agent` exits with status 2 instead of emitting a misleading shape; use `--json-profile=raw` to retain the remote backend's existing server-response JSON contract.
## Top-level fields
| Field | Type | Meaning |
| --- | --- | --- |
| `schema_version` | string | Agent export contract version. The current version is `1.2`. |
| `query` | string | The research topic supplied to the engine. |
| `generated_at` | string | UTC generation timestamp in RFC 3339 format. |
| `window_days` | integer | Number of days between the report's start and end dates. |
| `source_status` | object | Map of source name to the outcome observed during this run. |
| `freshness_verdicts` | array | Per-claim act-time verdicts produced by `--verify-freshness`; empty when verification was not requested or no conservative claims were extractable. |
| `clusters` | array | Ranked groups of related results. |
| `results` | array | Ranked, flat evidence results for downstream processing. |
All top-level fields are always present. Empty runs contain empty `clusters` and `results` arrays. Sources appear in `source_status` when the run recorded an outcome for them.
## `freshness_verdicts`
Each entry identifies the grounded claim and candidate, its primary source item, the typed `verdict` (`current`, `stale`, `contradicted`, or `unsupported`), the original and re-derived values when applicable, and source/evidence URLs and timestamps. `stale` means a successful point re-fetch returned a moved value; `contradicted` means a newer item in the report window explicitly disagrees; `unsupported` means the datum could not be re-checked, including degraded `source_status` outcomes. Consumers can gate actions on `verdict == "current"` without treating an unreachable source as evidence that a claim moved.
## `source_status`
Each value distinguishes a clean empty result from incomplete coverage:
| State | Meaning |
| --- | --- |
| `ok` | The source completed and returned one or more items. |
| `no-results` | The source completed successfully but found no matching items. |
| `partial` | The source returned some items before a later failure. |
| `rate-limited` | Retrieval was stopped by a provider rate limit. |
| `auth-failed` | Credentials were missing, rejected, or expired during retrieval. |
| `unreachable` | The source or network endpoint could not be reached. |
| `timeout` | Retrieval exceeded its time limit. |
| `schema-drift` | The provider response no longer matched the expected shape. |
| `skipped-unconfigured` | The source was intentionally skipped because required configuration was absent. |
| `error` | Retrieval failed for another reason. |
Consumers must not interpret failure states as evidence that a source had no discussion. Only `no-results` means the source completed cleanly with zero matches.
## Cluster fields
| Field | Type | Meaning |
| --- | --- | --- |
| `title` | string | Cluster headline. |
| `summary` | string | Summary from the cluster's representative ranked result. |
| `sources` | array of strings | Sources represented by the cluster. |
| `engagement_total` | number | Sum of one headline native engagement counter per result. Known sources use their primary count (for example, Digg uses `postCount`); otherwise the largest counter-like field is used. Ranking, ratio, rating, and computed-score metadata are excluded. |
Cluster array order is ranking order. A result's `cluster` value is the zero-based index into this array.
## Result fields
| Field | Type | Meaning |
| --- | --- | --- |
| `candidate_id` | string | Stable identifier joining this result to `freshness_verdicts[].candidate_id`. Added in `1.2`. |
| `title` | string | Result title. |
| `source` | string | Primary source name, such as `reddit`, `x`, `youtube`, or `grounding`. |
| `url` | string | Canonical result URL. It may be empty when the provider supplies no link. |
| `published_at` | string | Primary source item's publication date or timestamp. Omitted when unknown. |
| `summary` | string | Normalized snippet, with the relevance explanation or body used as fallback. |
| `engagement` | object | Native engagement counters from the primary source item, such as Reddit `score` and `num_comments` or X `likes` and `reposts`. |
| `relevance_score` | number | Engine final score normalized to the inclusive `0.0`–`1.0` range. |
| `cluster` | integer | Zero-based index into `clusters`. Omitted when the result is not assigned to a cluster. |
Fields whose value is unknown are omitted rather than emitted as JSON `null`. Strings and collection fields otherwise remain present, including empty strings, objects, or arrays.
## Comparison runs
Comparison queries use an envelope so each entity keeps its own contract:
```json
{
"schema_version": "1.2",
"comparison": true,
"entities": ["OpenAI", "Anthropic"],
"reports": [
{"entity": "OpenAI", "report": {"schema_version": "1.2", "query": "OpenAI"}},
{"entity": "Anthropic", "report": {"schema_version": "1.2", "query": "Anthropic"}}
]
}
```
The abbreviated reports above only illustrate the envelope; real reports contain every documented top-level field.
## Versioning policy
- `schema_version` uses `major.minor` numbering.
- Any breaking field removal, rename, type change, semantic change, or envelope change requires a major-version bump.
- Backward-compatible field additions may use a minor-version bump. Consumers should ignore fields they do not recognize.
- The checked-in golden snapshot test locks the complete current shape. Contract changes must update the version and snapshot deliberately.
- `1.2` added `candidate_id` to each `results` entry so verdicts can be joined to the result they annotate.
- `--json-profile=raw` is outside this compatibility policy because it mirrors internal pipeline dataclasses.
`--preflight --emit=json` is a different machine contract for permission and configuration inspection. `--json-profile` does not alter preflight output.
docs/reference/old-nux-wizard-v3.0.0.md
+205
View File
@@ -0,0 +1,205 @@
# Original v3.0.0 First-Run NUX Wizard (reference capture)
Captured verbatim from `SKILL.md` at git commit `0a9ff16` (v3.0.0, 2026-04-08),
the first-run setup wizard Matt built. Preserved here for provenance and as the
source for the restored modal NUX (see docs/plans/2026-06-22-001-feat-restore-nux-wizard-plan.md).
This is a historical snapshot - the live wizard in SKILL.md Step 0 uses the CURRENT
source inventory (Digg, youtube_comments, SC backups) and omits Threads/Pinterest.
```markdown
## Step 0: First-Run Setup Wizard
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section **silently** and proceed to Step 1. **Do NOT say "Setup is complete" or any other status message — just move on.** The user doesn't need to be told setup is done every time they run the skill.
**When first run is detected, detect your platform first:**
**If you do NOT have WebSearch capability (OpenClaw, Codex, raw CLI):** Run the OpenClaw setup flow below.
**If you DO have WebSearch (Claude Code):** Run the standard setup flow below.
---
### OpenClaw / Non-WebSearch Setup Flow
Run environment detection first:
```bash
python3 "${SKILL_ROOT}/scripts/last30days.py" setup --openclaw
```
Read the JSON output. It tells you what's already configured. Display a status summary:
```
👋 Welcome to /last30days!
Detected:
{✅ or ❌} yt-dlp (YouTube search)
{✅ or ❌} X/Twitter ({method} configured)
{✅ or ❌} ScrapeCreators (TikTok, Instagram, Reddit backup)
{✅ or ❌} Web search ({backend} configured)
```
Then for each missing item, offer setup in priority order:
1. **ScrapeCreators** (if not configured): "ScrapeCreators adds TikTok and Instagram search (plus a Reddit backup if public Reddit gets rate-limited). 10,000 free calls, no credit card. (No referrals, no kickbacks - we don't get a cut.)"
- Option A: "ScrapeCreators via GitHub (recommended)" -- Check if `gh` CLI was detected in the environment detection output above. If gh IS detected: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". Before running the command, display: "Registering via GitHub CLI..." If gh is NOT detected: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". Before running the command, display: "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 "${SKILL_ROOT}/scripts/last30days.py" setup --github`, parse JSON output. Tries PAT first (if `gh` is installed), falls back to device flow which copies a one-time code to your clipboard and opens your browser. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to .env.
- Option B: "I have a key" -- accept paste, write to .env
- Option C: "Skip for now"
2. **X/Twitter** (if not configured): "X search finds tweets and conversations. To unlock X: add FROM_BROWSER=auto (reads browser cookies, free), XAI_API_KEY (no browser access, api.x.ai), or AUTH_TOKEN+CT0 (manual cookies)."
- Option A: "I have an xAI API key" (recommended for servers -- persistent, no expiry). Write XAI_API_KEY to .env.
- Option B: "I have AUTH_TOKEN + CT0 from my browser" -- accept both, write to .env
- Option C: "Skip for now"
3. **YouTube** (if yt-dlp not found): "YouTube search needs yt-dlp. Run: `pip install yt-dlp`"
4. **Web search** (if no Brave/Exa/Serper key): "A web search key enables smarter results. Brave Search is free for 2,000 queries/month at brave.com/search/api"
After setup, write `SETUP_COMPLETE=true` to .env and proceed to research.
**Skip to "END OF FIRST-RUN WIZARD" below after completing the OpenClaw flow.**
---
### Claude Code Setup Flow (Standard)
**You MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text -> (2) setup modal -> (3) run setup if chosen -> (4) optional ScrapeCreators modal -> (5) topic picker. You MUST start at step 1.**
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion - do NOT repeat any of the welcome text inside the AskUserQuestion call.**
Welcome to /last30days!
I research any topic across Reddit, X, YouTube, and other sources - synthesizing what people are actually saying right now.
Auto setup gives you 5 core sources for free in 30 seconds:
- X/Twitter - reads your x.com browser cookies to authenticate (not saved to disk). Chrome on macOS will prompt for Keychain access.
- Reddit with comments - public JSON, no API key needed
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars)
- Hacker News + Polymarket + GitHub (if `gh` CLI installed) - always on, zero config
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
**Then call AskUserQuestion with ONLY this question and these options - no additional text:**
Question: "How would you like to set up?"
Options:
- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp for YouTube"
- "Manual setup - show me what to configure"
- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if gh installed), Web"
**If the user picks 1 (Auto setup):**
**Before running the setup command, get cookie consent:**
Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`. If it does, skip the consent prompt and run setup directly.
If `BROWSER_CONSENT=true` is NOT present, **call AskUserQuestion:**
Question: "Auto setup will scan your browser for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. Chrome on macOS will prompt for Keychain access. OK to proceed?"
Options:
- "Yes, scan my cookies for X" - Run setup as normal. Append `BROWSER_CONSENT=true` to .env after setup completes.
- "Skip X, just set up YouTube" - Run setup with YouTube only (install yt-dlp). Do not scan cookies.
- "I have an xAI API key instead" - Ask them to paste it, write XAI_API_KEY to .env. Then install yt-dlp.
Run the setup subcommand:
```bash
cd {SKILL_DIR} && python3 scripts/last30days.py setup
```
Show the user the results (what cookies were found, whether yt-dlp was installed).
**Then show the optional ScrapeCreators offer (plain text, then modal):**
Want TikTok and Instagram too? ScrapeCreators adds those platforms - 10,000 free calls, no credit card. It also serves as a Reddit backup if public Reddit ever gets rate-limited.
**Before showing the ScrapeCreators modal, check for `gh` CLI:** Run `which gh` via Bash silently. Store the result as gh_available (true if found, false if not).
**Call AskUserQuestion:**
Question: "Want to add TikTok, Instagram, and Reddit backup via ScrapeCreators? (We don't get a cut.)"
Options:
- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". If NOT gh_available: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". After the user selects this option: If gh_available, display "Registering via GitHub CLI..." before running the command. If NOT gh_available, display "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V on Mac, Ctrl+V on Windows/Linux)." Then run `cd {SKILL_DIR} && python3 scripts/last30days.py setup --github` via Bash with a 5-minute timeout. This tries PAT auth first (if `gh` CLI is installed, zero browser needed), then falls back to GitHub device flow which copies a one-time code to your clipboard and opens GitHub in your browser. Parse the JSON stdout. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to `~/.config/last30days/.env`. If `method` is `pat`, show: "You're in! Registered via GitHub CLI - zero browser needed. 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is true, show: "You're in! (The authorization code was copied to your clipboard automatically.) 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is false, show: "You're in! 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `status` is `timeout` or `error`, show: "GitHub auth didn't complete. No worries - you can sign up at scrapecreators.com instead or try again later." Then offer the web signup option.
- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
- "I have a key" - accept the key, write to .env
- "Skip for now" - proceed without ScrapeCreators
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
Your ScrapeCreators key powers TikTok, Instagram, Threads, Pinterest, and YouTube comments. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
**Call AskUserQuestion:**
Question: "Which ScrapeCreators sources do you want on?"
Options:
- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "TikTok and Instagram are on, plus Reddit backup if public Reddit has issues. You can add threads, pinterest, youtube_comments to INCLUDE_SOURCES anytime."
- "Everything - TikTok, Instagram, Threads, Pinterest, YouTube comments" - append `INCLUDE_SOURCES=tiktok,instagram,threads,pinterest,youtube_comments` to ~/.config/last30days/.env. Confirm: "All ScrapeCreators sources are on."
- "Just the basics - let's run our first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will serve as Reddit backup. You can add sources to INCLUDE_SOURCES in your .env anytime."
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
**Call AskUserQuestion:**
Question: "What do you want to research first?"
Options:
- "Claude Code vs Codex" - tech comparison
- "Sam Altman" - person in the news
- "Warriors Basketball" - sports
- "AI Legal Prompting Techniques" - niche/professional
- "Type my own topic"
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
**If the user picks 2 (Manual setup):**
Show them this guide (present as plain text, not blockquoted):
The magic of /last30days is Reddit comments + X posts together - and both are free. Here's how to unlock each source.
Add these to `~/.config/last30days/.env`:
X/Twitter (pick one - this is the most important):
- `FROM_BROWSER=auto` - free. Reads your x.com login cookies at search time to authenticate. Cookies are read live each run, not saved to disk. Chrome on macOS will prompt for Keychain access the first time. Firefox and Safari don't.
- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers or if you don't want cookie scanning.
- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com -> F12 -> Application -> Cookies)
Reddit (free, works out of the box):
- Public JSON gives you threads + top comments with upvote counts. No setup required.
- `SCRAPECREATORS_API_KEY=xxx` - optional backup source if public Reddit gets rate-limited.
- `OPENAI_API_KEY=xxx` - optional fallback if public Reddit search has trouble finding threads.
YouTube (free, open source):
- Run `brew install yt-dlp` - free, open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
Bonus: TikTok, Instagram, Threads, Pinterest, YouTube comments (ScrapeCreators):
- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com.
- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the most popular ones. Add threads, pinterest, youtube_comments for more.
GitHub Issues/PRs (free, no key needed):
- If you have the `gh` CLI installed (`brew install gh`), GitHub search is automatic. No API key required.
Perplexity Sonar Pro (AI-synthesized research via OpenRouter):
- `OPENROUTER_API_KEY=xxx` - adds AI-synthesized research with citations as an additive source alongside Reddit/X/YouTube. Returns structured narratives with specific dates, names, and numbers that social sources miss. ~$0.02/run.
- After adding your key, set `INCLUDE_SOURCES=perplexity` (or append to existing, e.g. `INCLUDE_SOURCES=tiktok,instagram,perplexity`).
- Use `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query) on topics that need serious investigation.
- Bonus: also powers the planning and reranking engine if you don't have a Gemini/OpenAI/xAI key.
Other bonus sources (add anytime):
- `EXA_API_KEY=xxx` - semantic web search, 1K free/month (exa.ai)
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password)
- `BRAVE_API_KEY=xxx` - Brave web search
Always add this last line: `SETUP_COMPLETE=true`
**CRITICAL: NEVER overwrite an existing .env file.** Before writing ANY key to `~/.config/last30days/.env`:
1. Check if the file exists: `test -f ~/.config/last30days/.env`
2. If it exists, READ it first, then APPEND only missing keys using `>>` (double redirect)
3. NEVER use `>` (single redirect) which destroys existing content
4. If it doesn't exist, create it: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env`
**Then call AskUserQuestion:**
Question: "How do you want to add your keys?"
Options:
- "Open .env in my editor" - Creates the file with a commented template and opens it. You edit, save, and come back.
- "Paste keys here" - Paste your API keys and I'll write the file for you.
- "I'll do it myself" - I'll tell you the file path and you handle it.
**If the user picks "Open .env in editor":**
Create `~/.config/last30days/.env` if it doesn't exist (check first!), pre-populated with this template:
```
```
docs/releases/v3.0.9.md
+112
View File
@@ -0,0 +1,112 @@
# v3.0.9 - The Self-Debug Release
## Highlights
**v3.0.9 is live.** New user-facing capabilities, broader cross-platform support, and a skill that now runs reliably on Claude Code, Codex, Hermes, Gemini, claude.ai, and OpenClaw. The headline fix: the engine refuses "birthday gift for 40 year old" style queries with a clarifying question instead of 5 minutes of junk output. The headline feature: TikTok and YouTube top comments now render alongside Reddit's, so the most-engaged voice from every source makes it into the synthesis.
**The label - "The Self-Debug Release":** I handed 5 separate Opus 4.7 instances their own failed outputs and asked them to debug themselves. Three converged on "SKILL.md is too big and the LAWs are too deep." Two converged on "the engine should refuse demographic-shopping queries outright" and "the WebSearch Sources reminder is overriding LAW 1." I copy-pasted their diagnoses into code. Validation: 5/5 canonical compliance on the topics that had failed.
## New capabilities
- **TikTok and YouTube top comments render alongside Reddit's.** PR [#260](https://github.com/mvanhorn/last30days-skill/pull/260) made the top-engagement comment from each TikTok video and YouTube video first-class in the output - same prominent `💬 Top comment` treatment Reddit's top comment already got. This is the biggest user-facing output change since 3.0.0 and it was never announced. The community inspiration trace: @uppinote20's original push for richer Reddit comments ([PR #143](https://github.com/mvanhorn/last30days-skill/pull/143)) seeded the pattern; this PR generalized it across TikTok and YouTube. PR [#265](https://github.com/mvanhorn/last30days-skill/pull/265) followed up by fixing the ScrapeCreators `url=` param + new response shape for YouTube comments/transcripts so the enrichment actually works.
- **last30days runs on Hermes AI Agent now.** @stephenmcconnachie's PR ([#228](https://github.com/mvanhorn/last30days-skill/pull/228)) added Hermes as a first-class deploy target. `scripts/sync.sh` detects `~/.hermes/skills/research` and deploys the full skill (SKILL.md, scripts, lib modules, fixtures) to Hermes's skills directory alongside Claude Code and Codex. This is one of the biggest surface-area expansions in v3 - last30days is now usable inside the Hermes agent's research workflows without any manual wiring.
- **Multi-key SCRAPECREATORS_API_KEY rotation.** @zaydiscold's PR ([#268](https://github.com/mvanhorn/last30days-skill/pull/268)) added automatic key rotation. Set `SCRAPECREATORS_API_KEY_1`, `SCRAPECREATORS_API_KEY_2`, etc. and the engine rotates when a key hits rate limits instead of failing the whole run. For power users running daily queries, this is the difference between rate-limit 429s and zero-touch reliability.
- **The skill works on Windows now.** @Chelebii's PR ([#227](https://github.com/mvanhorn/last30days-skill/pull/227)) stabilized the vendored Bird X search client on Windows. Previously the bundled X backend had subtle runtime issues on Windows terminals; now it runs clean. Pair this with @Gujiassh's UTF-8 encoding fix ([#225](https://github.com/mvanhorn/last30days-skill/pull/225)) for saved output and Windows users get the full v3 experience without workarounds.
- **Linux permission checks stopped false-warning.** @george231224's PR ([#216](https://github.com/mvanhorn/last30days-skill/pull/216)) fixed `check_perms` on Linux by preferring GNU stat's syntax over the BSD stat that the skill was calling. Linux users were getting spurious permission warnings on `.env` files that were already correctly 600-chmod'd. Now the check matches reality.
- **Gemini CLI got a first-class install path.** @hnshah's docs PR ([#224](https://github.com/mvanhorn/last30days-skill/pull/224)) added the Gemini CLI install note and workaround for a rough edge in the Gemini skill loader. Gemini users now have a one-paragraph install flow in the README instead of having to reverse-engineer the plugin layout.
- **Offline quality evaluation.** @j-sperling's PR ([#233](https://github.com/mvanhorn/last30days-skill/pull/233)) added `eval_topics.json` as a fixture. Contributors and I can now run quality-regression checks on synthesis output without burning live API credits. This is the scaffolding that made the plan 015 validation gate affordable - without eval fixtures, testing 5/5 canonical compliance on every release would cost real money every time. Ships as contributor infrastructure but shows up as stability for end users.
- **Reddit client got a cleaner HTTP layer.** @iliaal shipped three architecture PRs back-to-back ([#207](https://github.com/mvanhorn/last30days-skill/pull/207), [#208](https://github.com/mvanhorn/last30days-skill/pull/208), [#209](https://github.com/mvanhorn/last30days-skill/pull/209)) that consolidated Reddit's HTTP handling into `http.get(params=...)`, rejected garbage input in `_parse_date`, and unified `_sc_headers` into `http.scrapecreators_headers`. End-user benefit: fewer flaky timeouts, fewer "weird parse error" crashes, a codebase that's easier for future contributors to touch without breaking Reddit. These aren't sexy PRs; they're the kind of refactor that prevents six future bug reports.
- **The `--days=N` flag keeps working.** @BryanTegomoh's PR ([#230](https://github.com/mvanhorn/last30days-skill/pull/230)) restored backcompat for the legacy `--days` alias so anyone who'd scripted against it in 2.x doesn't break on v3. Small PR, meaningful reliability gain for existing users.
- **INCLUDE_SOURCES has a sane default.** @hnshah's PR ([#223](https://github.com/mvanhorn/last30days-skill/pull/223)) defaulted the env var to empty string instead of unset. Missing env no longer breaks source inclusion on fresh installs.
- **Version metadata stays in sync.** @Gujiassh's PR ([#217](https://github.com/mvanhorn/last30days-skill/pull/217)) aligned the SKILL.md version header with the sync target version, and @shalomma's PR ([#229](https://github.com/mvanhorn/last30days-skill/pull/229)) closed the remaining drift between the SKILL.md header and plugin.json. "Which version am I actually on" is no longer an adventure.
- **Bird X engagement handling got hardened.** @j-sperling's PR ([#234](https://github.com/mvanhorn/last30days-skill/pull/234)) made `bird_x` skip all-None engagement dicts instead of crashing on them. Rare condition, but the kind of thing that silently kills a run on a specific topic.
- **Dev workflow hygiene.** @j-sperling's gitignore PR ([#232](https://github.com/mvanhorn/last30days-skill/pull/232)) dropped `.venv`, `.coverage`, `htmlcov`, and `.memsearch` from the tracked tree. Contributor quality-of-life; keeps PR diffs clean.
- **The skill installs to claude.ai.** PRs [#242](https://github.com/mvanhorn/last30days-skill/pull/242) and [#244](https://github.com/mvanhorn/last30days-skill/pull/244) shipped `scripts/build-skill.sh` plus the `.gitattributes` + `export-ignore` plumbing that packages last30days into a claude.ai-upload-ready `.skill` file under the 200-file cap. The skill is no longer Claude-Code-only - it installs directly on claude.ai, too. README has the upload workflow.
- **OpenAI Codex CLI discovers the skill natively.** PR [#219](https://github.com/mvanhorn/last30days-skill/pull/219) added `.agents/skills/last30days/SKILL.md` as a real file (not symlinked - Codex's loader skips symlinks) plus `.codex-plugin/plugin.json` as the namespace marker. The skill now shows up as `last30days:last30days` when Codex runs in a checkout. Inspired by @Jah-yee ([#153](https://github.com/mvanhorn/last30days-skill/pull/153)) and @dannyshmueli on X.
- **`/last30days` as a slash command.** PR [#267](https://github.com/mvanhorn/last30days-skill/pull/267) added `commands/last30days.md` so plugin users can type `/last30days <topic>` and Claude Code autocomplete prefix-matches it to the canonical `/last30days:last30days` form. No more typing the double-namespace.
## The self-debug technique, for anyone rebuilding this elsewhere
The breakthrough wasn't the individual fixes. It was the realization that instead of guessing why the model was ignoring the rules, I should ask the model. Five separate Opus 4.7 sessions debugged their own outputs:
- "Did you read SKILL.md?" → "I tried Read, hit the 25K token cap, and bailed instead of chunked-reading."
- "Why the trailing Sources block?" → "The WebSearch tool's own reminder said MANDATORY. Precedence was unclear."
- "Why the section headers?" → "I had strong priors on Peter Steinberger and wrote my thesis instead of passing through."
- "Why the wrong file?" → "I read `.agents/skills/last30days/SKILL.md` first because it appeared in the path glob."
Three of the five said "move the LAWs to the top." Two said "make the engine enforce it so the model can't not comply." I shipped both. That's the whole technique: when the LLM-under-orchestration keeps breaking the contract, don't argue with it - ask it to debug itself, and build structural enforcement around whatever it names as the root cause.
## Thank you
**Community PR authors since v3.0.0:**
- @j-sperling - v3 engine architecture, eval fixtures, gitignore hygiene, Bird X hardening ([#232](https://github.com/mvanhorn/last30days-skill/pull/232), [#233](https://github.com/mvanhorn/last30days-skill/pull/233), [#234](https://github.com/mvanhorn/last30days-skill/pull/234))
- @stephenmcconnachie - Hermes AI Agent support ([#228](https://github.com/mvanhorn/last30days-skill/pull/228))
- @zaydiscold - Multi-key SCRAPECREATORS rotation ([#268](https://github.com/mvanhorn/last30days-skill/pull/268))
- @iliaal - Reddit HTTP helper + GitHub date parsing + ScrapeCreators header consolidation ([#207](https://github.com/mvanhorn/last30days-skill/pull/207), [#208](https://github.com/mvanhorn/last30days-skill/pull/208), [#209](https://github.com/mvanhorn/last30days-skill/pull/209))
- @Chelebii - Windows Bird X stability ([#227](https://github.com/mvanhorn/last30days-skill/pull/227))
- @george231224 - Linux check_perms stat ([#216](https://github.com/mvanhorn/last30days-skill/pull/216))
- @Gujiassh - UTF-8 saved output + version metadata alignment ([#217](https://github.com/mvanhorn/last30days-skill/pull/217), [#225](https://github.com/mvanhorn/last30days-skill/pull/225))
- @hnshah - INCLUDE_SOURCES default + Gemini install docs ([#223](https://github.com/mvanhorn/last30days-skill/pull/223), [#224](https://github.com/mvanhorn/last30days-skill/pull/224))
- @shalomma - SKILL.md v3.0.0 version header ([#229](https://github.com/mvanhorn/last30days-skill/pull/229))
- @BryanTegomoh - --days alias backcompat ([#230](https://github.com/mvanhorn/last30days-skill/pull/230))
**v3 roadmap contributors (issues and PRs that shaped the v3 feature set):**
- @uppinote20 - rich Reddit comments ([#143](https://github.com/mvanhorn/last30days-skill/pull/143))
- @zerone0x - GitHub as a first-class source ([#134](https://github.com/mvanhorn/last30days-skill/issues/134), [#136](https://github.com/mvanhorn/last30days-skill/pull/136))
- @thinkun - Reddit enrichment timeout handling ([#116](https://github.com/mvanhorn/last30days-skill/pull/116))
- @thomasmktong - pure-Python Reddit fallback ([#124](https://github.com/mvanhorn/last30days-skill/pull/124))
- @fanispoulinakisai-boop - Reddit timeout report ([#100](https://github.com/mvanhorn/last30days-skill/issues/100))
- @pejmanjohn - plugin directory naming ([#99](https://github.com/mvanhorn/last30days-skill/issues/99), [#78](https://github.com/mvanhorn/last30days-skill/issues/78))
- @zl190 - HN trending merge ([#115](https://github.com/mvanhorn/last30days-skill/pull/115))
- @hnshah - Watchlist features ([#84](https://github.com/mvanhorn/last30days-skill/pull/84), [#85](https://github.com/mvanhorn/last30days-skill/pull/85), [#86](https://github.com/mvanhorn/last30days-skill/pull/86))
- @Jah-yee, @dannyshmueli - Codex CLI discovery
- @Cody-Coyote - marketplace validation bug report ([#204](https://github.com/mvanhorn/last30days-skill/issues/204))
**The five Opus 4.7 instances that debugged their own failures on v3.0.7 and v3.0.8 and converged on the fixes.** The convergence was the breakthrough; this release is their diagnosis in code.
## Install / Update
```
/plugin marketplace add mvanhorn/last30days-skill
/plugin install last30days@last30days-skill
```
Or if already installed:
```
/plugin update last30days
/reload-plugins
```
## Verify
```
cat ~/.claude/plugins/cache/last30days-skill/last30days/*/.claude-plugin/plugin.json | grep version
```
Should print `"version": "3.0.9"`.
## Smoke test
```
/last30days birthday gift for 40 year old
```
Should ask a clarifying question before running. If it runs the engine anyway, the cache is stale - repeat the plugin update.
**Full Changelog:** https://github.com/mvanhorn/last30days-skill/compare/v3.0.5...v3.0.9
docs/search-quality-eval.md
+48
View File
@@ -0,0 +1,48 @@
# Search Quality Eval
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
What it does:
- runs a baseline revision (default `origin/main`) against a candidate checkout
- evaluates the fixed 5 reviewer topics by default
- computes deterministic stability metrics:
- `Jaccard` overlap vs baseline
- retention vs baseline
- per-source counts and overlap
- optionally calls Gemini as a judge for graded relevance labels and then computes:
- `Precision@5`
- `nDCG@5`
- source-coverage recall across the judged union pool
Recommended usage:
```bash
uv run python skills/last30days/scripts/evaluate_search_quality.py
```
Useful flags:
```bash
uv run python skills/last30days/scripts/evaluate_search_quality.py \
--baseline-rev origin/main \
--candidate-rev HEAD \
--no-default-topics \
--topic "cursor IDE pricing" \
--per-source-limit 5
```
Gemini configuration:
- preferred on this workspace: set `GOOGLE_API_KEY`
- also accepted: `GEMINI_API_KEY` or `GOOGLE_GENAI_API_KEY`
- optional: set `GEMINI_MODEL`
- default model is `gemini-3-pro-preview` for the direct Gemini API
Notes:
- The script forces a clean env-based auth path when it shells out to `last30days.py`.
- It passes `XAI_API_KEY`, `OPENAI_API_KEY`, and `SCRAPECREATORS_API_KEY`, but intentionally does not pass browser-cookie X auth. That keeps evaluation runs on the popup-free path.
- It also strips `node` from the eval `PATH` and wraps `yt-dlp` with `--ignore-config`, so older revisions do not inherit local browser-cookie config either.
- `Jaccard` and retention are regression guards, not truth metrics.
- `Precision@5` and `nDCG@5` are only as good as the judged pool. They help compare revisions, but they are not a substitute for a larger labeled benchmark.
docs/solutions/architecture/search-quality-eval-manual-by-default-2026-05-10.md
+82
View File
@@ -0,0 +1,82 @@
---
title: Search-quality eval is manual by default, not a CI gate on every PR
date: 2026-05-10
category: docs/solutions/architecture
module: skills/last30days/scripts/evaluate_search_quality.py
problem_type: design_decision
component: ci_policy
severity: low
applies_when:
- a contributor proposes wiring search-quality eval into PR CI
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
related_components:
- search_quality_evaluation
- ci_workflow
- llm_judging
tags:
- ci-policy
- eval
- design-decision
- cost-vs-signal
- non-determinism
- manual-gates
---
# Search-quality eval is manual by default, not a CI gate on every PR
## Context
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
## Guidance
### 1. Keep search-quality eval available, just not automatic
The script stays runnable by maintainers and contributors. The pattern is:
```bash
LAST30DAYS_PYTHON=python3.13 \
python3 skills/last30days/scripts/evaluate_search_quality.py \
--baseline main --candidate HEAD
```
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
### 2. Standard PR CI gates remain deterministic and contract-shaped
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
### 4. Revisit if the eval can ever be made offline-deterministic
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
## What this means in practice
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
## Links
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
- `docs/search-quality-eval.md` — user-facing usage documentation
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
---
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md
+80
View File
@@ -0,0 +1,80 @@
---
title: Digg NUX must match printing-press-library install paths and agent subprocess PATH
date: 2026-06-17
category: docs/solutions/integration-issues
module: lib/setup_wizard
problem_type: integration_issue
component: development_workflow
severity: medium
symptoms:
- Digg source silently off after first-run setup reports success on Hermes or OpenClaw
- Users who already installed pp-digg via printing-press-library still see Digg missing from --diagnose available_sources
- Setup wizard probed ~/go/bin while the catalog installer writes to ~/.local/bin (printing-press-library 0.1.16+)
- OpenClaw setup --openclaw path skipped Digg install entirely
root_cause: config_error
resolution_type: code_fix
related_components:
- lib/pipeline
- lib/digg
- CONFIGURATION.md
tags:
- digg
- setup-wizard
- printing-press-library
- agent-path
- hermes
- openclaw
- nux
- optional-cli-sources
---
# Digg NUX must match printing-press-library install paths and agent subprocess PATH
## Problem
First-run setup auto-install for `digg-pp-cli` could report success while the engine still omitted Digg, especially on Hermes and OpenClaw where the agent subprocess PATH often excludes `$HOME/.local/bin`. The initial PR also used the deprecated `@mvanhorn/printing-press` package and probed legacy `~/go/bin` fallbacks instead of the current Printing Press default install dir.
## Symptoms
- `--diagnose` `available_sources` lacks `digg` even though pp-digg or setup "installed" the CLI.
- Hermes/OpenClaw users with a prior `npx @mvanhorn/printing-press-library install digg --cli-only` run hit false failures or false "now active" messages depending on probe logic.
- OpenClaw `setup --openclaw` never attempted Digg install (desktop NUX only).
## What Didn't Work
- **Treating "binary exists somewhere" as installed** — `pipeline.available_sources()` and `digg._is_available()` gate on `shutil.which("digg-pp-cli")` only. Probing `~/go/bin` without PATH visibility produced false positives.
- **Assuming Hermes vs OpenClaw use different binary locations** — both harnesses use the same printing-press-library default (`$HOME/.local/bin`); only the focused pp-digg *skill* wiring differs.
- **Using `@mvanhorn/printing-press`** — superseded by `@mvanhorn/printing-press-library`; install defaults moved from `$GOPATH/bin` to `$HOME/.local/bin` in npm 0.1.16.
## Solution
Align setup wizard with the catalog installer and the engine PATH gate:
1. **Pin installer:** `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only` (`--cli-only` only — last30days embeds Digg as an engine source, not pp-digg skill).
2. **Split outcomes:** `already_installed` / `installed` only when `shutil.which` resolves; `installed_off_path` when the binary exists under known dirs (`~/.local/bin`, legacy `~/go/bin`, Windows PrintingPress bin) but is not PATH-visible; surface `digg_path` and PATH-restart guidance in status text.
3. **OpenClaw parity:** `run_openclaw_setup()` runs the same `_install_digg_cli()` and returns `digg_cli`, `digg_action`, optional `digg_path`.
4. **Docs:** CONFIGURATION.md, SKILL.md Step 0, HERMES_SETUP.md, AGENTS.md rule for CLI-gated sources.
Key helper shape in `setup_wizard.py`:
```python
def _digg_on_path() -> Optional[str]:
return shutil.which(DIGG_CLI_BIN) # engine gate
def _digg_off_path_binary() -> Optional[str]:
for candidate in _digg_bin_candidate_paths(): # ~/.local/bin first
if candidate.is_file() and os.access(candidate, os.X_OK):
return str(candidate)
return None
```
## Why This Works
The engine never reads "is pp-digg skill installed?" — every research run shells out to `digg-pp-cli` by name on PATH. Printing Press already installs to a managed user bin dir and warns when that dir is off PATH; last30days setup must mirror that contract instead of inventing a separate success definition. Detecting off-PATH binaries lets setup reuse prior pp-digg installs without lying about activation.
## Prevention
- When adding NUX auto-install for a CLI-gated source, match the upstream installer's default bin dir and pin the npm semver.
- Success messaging must use the same probe as `available_sources()` (`shutil.which`), with a separate off-PATH outcome when the binary exists on disk.
- Cover Hermes/OpenClaw in tests with redirected `HOME` and mocked PATH; add OpenClaw JSON fields when server setup should mirror desktop NUX.
- Search `docs/solutions/` for `digg`, `setup-wizard`, and `agent-path` before changing optional-source onboarding.
docs/solutions/logic-errors/entity-grounding-full-phrase-false-demotion.md
+117
View File
@@ -0,0 +1,117 @@
---
title: Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
date: 2026-06-09
category: docs/solutions/logic-errors
module: lib/rerank
problem_type: logic_error
component: search_ranking
severity: high
symptoms:
- on-entity, high-engagement items that name the brand but omit the trailing descriptor of a multi-word query are demoted in keyless/fallback rerank results
- observed case is a 323-point HN thread about Stripe scoring 0 on a "Stripe payments" query
- the entity-miss demotion lands twice (ENTITY_MISS_PENALTY on rerank_score plus a secondary final_score penalty), so a false miss guarantees burial regardless of engagement
- reddit keyless comment-enrichment slot selection skips the same on-entity threads via an independently duplicated full-phrase check in _slot_priority
root_cause: logic_error
resolution_type: code_fix
related_components:
- reddit_keyless
- comment_enrichment
tags:
- entity-grounding
- rerank
- keyless-fallback
- multi-word-entity
- substring-match
- false-demotion
- reddit-keyless
- duplicated-logic
---
# Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
## Problem
The keyless/fallback rerank path's entity-grounding demotion required the FULL multi-word primary-entity phrase as a contiguous substring of the candidate's text (`primary_entity.lower() not in haystack`), so on-entity items that omitted a trailing search descriptor were falsely flagged as entity misses and buried by a deliberately decisive double penalty.
## Symptoms
- On a "Stripe payments" query, a 323-point HN thread titled "Stripe is friendly to 'friendly fraud'" was demoted to score 0 — purely because its text never contained the literal phrase "stripe payments" (the trailing word "payments" was missing).
- The burial is guaranteed by design, not incidental: a flagged entity miss takes −25 `ENTITY_MISS_PENALTY` on `rerank_score` in `_fallback_tuple`, PLUS `ENTITY_MISS_FINAL_PENALTY` applied directly in `_final_score` (added 2026-04-19 after engagement + freshness drowned the diluted penalty). A false positive on the check means confirmed-good signal cannot recover.
- The same over-strict check had been independently re-implemented in `reddit_keyless._slot_priority` (keyless Reddit comment-enrichment slot selection), so scarce comment slots were also steered away from head-token-only posts.
## What Didn't Work
- **Naively relaxing the check** — the full-phrase check existed for a real reason: on 2026-04-19 an off-topic video with zero brand mentions ranked #2 on a Hermes query (documented in the `ENTITY_MISS_FINAL_PENALTY` comment in `skills/last30days/scripts/lib/rerank.py`). Any fix had to keep that demotion firing.
- **Word-boundary matching** — rejected; it re-introduces over-demotion on plurals/possessives/compounds ("stripes", "Stripe's").
- **Graded penalty** (full-phrase = 0, head-only = half, none = full) — rejected; it half-punishes items that are 100% about the entity. Lexical coverage is not topical degree.
- **Any-token grounding** — rejected; "payments" alone would ground completely generic posts.
- **Distinctiveness gate for generic heads** — rejected as complexity to patch a failure mode that is already a safe no-op (see Why This Works).
- **Trusting the docstring** — `reddit_keyless._slot_priority`'s docstring claimed to "mirror rerank's demotion signal," but its inline reimplementation (`entity in _post_text(post).lower()`) had silently drifted from being a mirror into being a second copy of the bug. It was found only by a code-reuse review, not by tests.
## Solution
Ground on the **head token** of the primary entity instead of the full phrase, via one shared helper used by both paths.
**Site 1 — new helper in `skills/last30days/scripts/lib/rerank.py`:**
```python
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
tokens = primary_entity.lower().split()
if not tokens:
return True
return tokens[0] in haystack
```
`_fallback_tuple` switches from the inline phrase check to the helper:
```python
# before
if haystack.strip() and primary_entity.lower() not in haystack:
# after
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
```
**Site 2 — secondary penalty in `_final_score`: no code change needed.** It keys off the explanation string set by site 1, so it inherits the fix automatically:
```python
if candidate.explanation and "entity-miss" in candidate.explanation:
base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY)
```
**Site 3 — `skills/last30days/scripts/lib/reddit_keyless.py` `_slot_priority`:** replace the drifted reimplementation with a call to the shared helper:
```python
# before
return entity in _post_text(post).lower()
# after
return rerank._entity_grounded(_post_text(post).lower(), entity)
```
Tests: `tests/test_rerank_v3.py` gained `test_fallback_grounds_on_head_token_not_full_phrase` (the Stripe regression) and `test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` (guards the 2026-04-19 behavior). `tests/test_reddit_keyless.py`'s two old-contract tests were rewritten as `test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match`.
## Why This Works
- **Root cause:** trailing tokens of a multi-word query ("payments" in "Stripe payments") are usually category descriptors the user/planner appended for search, not part of the entity name. Requiring the whole phrase conflates "doesn't repeat my search phrasing" with "isn't about my entity." The brand head token alone is sufficient grounding; items that never name the brand at all still miss the head token and stay demoted — so the original 2026-04-19 fix keeps firing.
- **Asymmetry argument:** the demotion is engineered to be decisive (double penalty across `rerank_score` and `final_score`), so a false entity-miss is fatal-by-design, while a false grounding merely defers the item to normal relevance/freshness/quality ranking. When the punishment is capital, the conviction standard should be conservative.
- **Substring (not word-boundary) is deliberate:** it catches plurals/possessives/compounds ("stripes", "Stripe's"). Degenerate short heads ("X", "Go", "C") make the check vacuously true, which merely **disables** the penalty — reverting to the pre-grounding baseline — rather than burying good items. Every failure mode of this rule degrades toward "no penalty," never toward "bury good signal."
- **Accepted, bounded limitation:** head-collision with a different famous entity ("Hermes Agent" → a "Hermes Birkin" thread now escapes demotion). This is lexically unfixable — any token rule strong enough to kill the collision re-kills the Stripe case; the discriminator is semantic. The LLM rerank path (which receives the full phrase as prompt guidance and judges semantically) covers this when API keys exist; the keyless path accepts the bounded risk.
## Prevention
- **Shared helper as single source of truth:** when one module's behavior must "mirror" another's signal, it must *call* the same function, not re-implement the check. The `reddit_keyless._slot_priority` drift happened precisely because the mirror was a copy. The fix wires it to `rerank._entity_grounded`, and the docstring now states this explicitly: "keying on the same head token keeps the two paths from diverging."
- **Docstrings record deliberate trade-offs:** `_entity_grounded`'s docstring documents WHY head-token (not phrase), why substring (not word-boundary), and the safe-failure direction. Future readers see the rejected alternatives were considered, not overlooked — and won't "tighten" the check into a regression.
- **Both directions pinned by named tests:**
- `tests/test_rerank_v3.py::test_fallback_grounds_on_head_token_not_full_phrase` — false-demotion regression (the Stripe HN thread must not be flagged).
- `tests/test_rerank_v3.py::test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` — the fix must not neuter the demotion (guards the 2026-04-19 off-topic-video incident).
- `tests/test_reddit_keyless.py::test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match` — the mirrored path asserts the same contract.
- **Audit tests when changing a contract:** tests that encode the old behavior as correct must be rewritten to the new contract, not worked around — the two old `test_reddit_keyless.py` tests would have silently re-blessed the bug.
- **For decisive penalties, route through one flag:** the `_final_score` backstop keys off `"entity-miss" in candidate.explanation` rather than re-running the check — so there was exactly one site to fix and the second penalty inherited it for free. Prefer this signal-propagation pattern over duplicating predicate logic at each penalty site.
## Related Issues
- [PR #484](https://github.com/mvanhorn/last30days-skill/pull/484) — "fix(reddit): relevance-aware comment-enrichment slot selection in keyless path" — introduced the `_slot_priority` mirror this fix reroutes through the shared helper.
- [PR #457](https://github.com/mvanhorn/last30days-skill/pull/457) — "fix(reddit): restore free path via keyless RSS + shreddit scrape" — established the keyless Reddit path.
- [PR #488](https://github.com/mvanhorn/last30days-skill/pull/488) (open) — "fix(reddit): relevance floor + relevance-first ranking" — external PR touching the same ranking surface; coordinate before merging both.
- [Issue #468](https://github.com/mvanhorn/last30days-skill/issues/468) (open) — relevance scoring over-pruning on-topic YouTube items; same symptom family in a different source.
- [../architecture/search-quality-eval-manual-by-default-2026-05-10.md](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) — how to validate ranking/grounding changes like this one (manual eval, not CI-gated).
- [../workflow-issues/release-consistency-test-cascade-2026-05-16.md](../workflow-issues/release-consistency-test-cascade-2026-05-16.md) — sibling prevention pattern: lockstep artifacts drift unless mechanically unified.
docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md
+219
View File
@@ -0,0 +1,219 @@
---
title: Release-time consistency tests cause cascade CI failures across all open PRs
date: 2026-05-16
category: docs/solutions/workflow-issues
module: ci-release-engineering
problem_type: workflow_issue
component: testing_framework
severity: high
applies_when:
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
- multiple long-lived PRs are open simultaneously against the same base branch
symptoms:
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
- PR authors must rebase and manually fix an artifact they did not touch
root_cause: missing_workflow_step
resolution_type: code_fix
related_components:
- development_workflow
- documentation
tags:
- ci
- release-engineering
- consistency-test
- version-pin
- cascade-failure
- test-design
- workflow
---
# Release-time consistency tests cause cascade CI failures across all open PRs
## Context
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
2. The release PR merges to `main`.
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
## Guidance
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
This pattern looks safe but is not:
```python
def test_sync_cache_path_uses_skill_version(self) -> None:
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
version = _skill_version() # reads SKILL.md
self.assertIn(
f'last30days-skill/last30days/{version}"',
sync_text, # asserts sync.sh contains that string
)
```
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
Remove the hardcoded pin from `sync.sh` and compute it:
```bash
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
```
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
```python
def test_sync_cache_path_uses_skill_version(self) -> None:
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
if not sync_sh.exists():
self.skipTest("sync.sh not present; skipping pin consistency check")
sync_text = sync_sh.read_text(encoding="utf-8")
version = _skill_version()
self.assertIn(
f'last30days-skill/last30days/{version}"',
sync_text,
)
```
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
### 4. Run consistency tests against the merge-base diff, not main
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
```yaml
- name: Check sync.sh version pin consistency
run: |
BASE=$(git merge-base HEAD origin/main)
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
echo "SKILL.md version bumped but sync.sh pin was not updated"
exit 1
fi
```
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
### 5. Ask whether you actually need this test
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
## Why This Matters
The damage from a stale-pin consistency test is asymmetric. It:
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
- Blocks work that has already been reviewed and approved.
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
## When to Apply
Apply this guidance whenever you find yourself:
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
## Examples
### Before — the pattern that caused the cascade
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
```python
import re
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = ROOT / "skills" / "last30days"
def _skill_version() -> str:
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
if not match:
raise AssertionError("SKILL.md version frontmatter not found")
return match.group(1)
class TestVersionConsistency(unittest.TestCase):
def test_sync_cache_path_uses_skill_version(self) -> None:
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
version = _skill_version() # source 1: SKILL.md frontmatter
self.assertIn( # assertion: sync.sh must contain
f'last30days-skill/last30days/{version}"',
sync_text, # source 2: hardcoded string in sync.sh
)
```
`sync.sh` contained a line like:
```bash
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
```
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
### After — what we did: delete both
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
### After — what we could have done instead: derive at runtime
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
```bash
#!/usr/bin/env bash
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
if [ -z "$SKILL_VERSION" ]; then
echo "error: could not parse version from SKILL.md" >&2
exit 1
fi
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
# ... rest of sync logic
```
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
## Related
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
docs/v2.1-launch-copy.md
+120
View File
@@ -0,0 +1,120 @@
# v2.1 Launch Copy (DRAFT — do not publish)
## Context
Bird CLI (@steipete/bird) has been deprecated on npm and the GitHub repo deleted. steipete was asked to take it down (likely by X). Nobody has forked and republished it. Our v2.1 vendors a search-only subset of Bird v0.8.0 (MIT licensed) so users don't need any external tools.
YouTube transcripts are the second headline feature. Inspired by Peter Steinberger's yt-dlp + summarize toolchain. We use yt-dlp directly (no summarize dependency) — search YouTube, grab transcripts, feed them into the synthesis. Zero API keys, zero cost.
---
## README: "New in V2.1" blurb
**New in V2.1 — two headline features:**
- **YouTube transcripts as a 4th source.** When yt-dlp is installed, /last30days automatically searches YouTube, grabs view counts, and extracts auto-generated transcripts from the top videos. A 20-minute review contains 10x the signal of a tweet — now the skill reads it. Inspired by @steipete's yt-dlp + summarize toolchain.
- **X search is fully bundled.** No external `bird` CLI install needed. Add `AUTH_TOKEN` and `CT0` once, and the vendored Bird client runs locally without browser-cookie prompts. `XAI_API_KEY` remains an optional fallback.
---
## README: X Search Authentication section
### X Search Authentication
X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts.
**Recommended setup:** While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`.
Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env` or `.claude/last30days.env`:
```bash
AUTH_TOKEN=your_auth_token
CT0=your_ct0_token
```
**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and use xAI's `x_search` backend instead.
**Verify it's working:**
```bash
node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs --whoami
```
---
## README: Install block env line
```bash
AUTH_TOKEN=... # recommended for X search
CT0=... # recommended for X search
XAI_API_KEY=xai-... # optional X fallback
```
---
## SKILL.md: Stats line
```
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts
```
---
## GitHub issue #19 response (post AFTER publishing)
> Thanks for reporting this. Bird CLI was deprecated and the GitHub repo was deleted. steipete was asked to take it down.
>
> The good news: you don't need Bird anymore. v2.1 (just shipped) bundles X search directly. No external CLI, no `npm install`, no brew. Just Node.js 22+ plus `AUTH_TOKEN` and `CT0`, or `XAI_API_KEY` as fallback.
>
> It also adds **YouTube as a 4th source**. When yt-dlp is installed, the skill automatically searches YouTube and extracts transcripts from the top videos. A 20-minute tutorial has 10x the signal of a tweet, and now the synthesis engine reads it.
>
> The recommended setup is to copy `auth_token` and `ct0` from x.com once and store them as `AUTH_TOKEN` and `CT0` in your env. That avoids browser-cookie and Keychain prompts during normal runs.
>
> The xAI API (`XAI_API_KEY`) also still works as a fallback.
---
## X/Social launch post
### Short (280 chars)
/last30days v2.1 is out 🚀
Two new features:
→ YouTube transcripts as a 4th source (yt-dlp)
→ X search fully bundled (no bird CLI install needed)
Research any topic across Reddit, X, YouTube & web in one command.
h/t @steipete for the inspiration on both.
github.com/mvanhorn/last30days-skill
### Thread version (post 1)
/last30days v2.1 just shipped — two headline features:
1️⃣ YouTube transcripts as a 4th source
When yt-dlp is installed, the skill searches YouTube, grabs view counts, and extracts auto-generated transcripts from top videos. A 20-min review has 10x the signal of a tweet — now the synthesis reads it.
### Thread version (post 2)
2️⃣ X search is fully bundled
Bird CLI was deprecated. Instead of requiring an external tool, v2.1 vendors a search-only subset. Add `AUTH_TOKEN` and `CT0` once, then it runs locally with no npm install. `XAI_API_KEY` still works as fallback.
Both features inspired by @steipete's tooling.
### Thread version (post 3)
YouTube goes through the same scoring pipeline as Reddit and X — relevance, recency, engagement. Transcripts get truncated to ~500 words per video and fed into the synthesis engine alongside social posts.
Zero API keys for YouTube. Zero cost. Just `brew install yt-dlp`.
### Thread version (post 4)
Try it:
```
/last30days [any topic]
```
Reddit + X + YouTube + Web. Four sources, one command, copy-paste prompts.
github.com/mvanhorn/last30days-skill
docs/v2.5-launch-tweets.md
+360
View File
@@ -0,0 +1,360 @@
# last30days v2.5 Launch Thread
## FINAL THREAD (6 tweets)
### 1/6 - Announcement
I can't believe it's been 30 days since I launched @slashlast30days. 3.2k stars later, time for v2.5.
Three big additions:
1. @Polymarket prediction markets as a 6th source - helps you predict the future
2. Cross-source linking + massively better results - detects when the same story trends across multiple platforms. Ran a 15-way blinded comparison, v2.5 scored 4.38 vs 3.73 for the original. Won all 5 topics.
3. Hacker News as a 5th source - a window into the tech and developer insider world
github.com/mvanhorn/last30days-skill
### 2/6 - Demo: Anthropic vs Pentagon
"/last30days Anthropic Pete Hegseth"
14 Reddit threads. 29 X posts (11,559 likes). 20 YouTube videos (739K views). 5 HN stories. 9 Polymarket markets.
This story broke TODAY. Hegseth designated Anthropic a "supply chain risk." Trump ordered every agency to stop using their tech.
Polymarket: Anthropic still 99% for best AI model. $500B+ valuation: 68%. IPO >$600B: 97%. Hegseth out by March: only 6%.
Markets say Anthropic wins regardless. That's the kind of signal you can't get from opinion threads.
### 3/6 - Demo: Seedance Prompting
"/last30days Seedance prompting"
13 Reddit threads. 33 X posts. 20 YouTube videos (1.2M views, 4 transcripts). 15 web pages.
Top finding: Seedance 2.0 prompts follow a director's shot-list format, not freeform text. 30-100 words. Subject + Action + Camera + Scene + Style. Beyond 100 words, results degrade.
Then I said: "a cinematic drone shot over a city at golden hour"
It wrote me a copy-paste prompt using the exact patterns from the research. Research first, then create from what you learned.
### 4/6 - Demo: Arizona Basketball
"/last30days arizona basketball"
6 Polymarket markets. 37 X posts (4,200 likes). 15 YouTube videos (517K views). 2 Reddit threads.
Arizona is 25-2, set a program record with a 22-0 start, and holds a 2-game Big 12 lead with 3 games left. The Field of 68 called them "the TOUGHEST team in America" after escaping Baylor shorthanded. Kansas rematch Saturday - the highlight video from their first meeting has 248K views on ESPN's YouTube.
Polymarket: Championship 13%. #1 seed: 88%. Duke and Michigan each at 18% to win it all.
That's not a sports blog. That's Reddit reactions + X engagement + YouTube analysis + prediction market odds from one command.
### 5/6 - Demo: Iran War
"/last30days iran war"
2 Reddit threads. 34 X posts (10,048 likes). 20 YouTube videos (1.6M views, 5 transcripts). 4 HN stories (850 points). 14 Polymarket markets ($473M volume).
Geneva talks just ended without a deal. 150+ US aircraft deployed. Two carrier strike groups in position. F-22s sent to Israel. Members of Congress who saw the secret war plan came out "terrified." @cenkuygur: "they are about to drag us into a war that 70-85% of Americans oppose" (7,700 likes).
Polymarket ($473M in volume - one of their biggest markets ever): strikes by 2026: 80%. By March 31: 68%. War Powers invoked: 51%. Formal war declaration: only 12%.
Markets say: strikes are very likely, declared war is not. That's the sharpest signal in the entire research.
### 6/6 - Thank You
Thank you to ARJ999 and wkbaran on GitHub who filed three separate issues asking for Hacker News support. v2.5 delivers.
It's been a crazy 30 days. 3.2k stars. Six sources. Massively better results. Super excited to get this out.
Try it: /last30days [any topic]
github.com/mvanhorn/last30days-skill
---
---
## REFERENCE MATERIAL BELOW
## Context
- 3.2k stars on GitHub
- V2.5 headline features: Polymarket (6th source), Hacker News (5th source), cross-source linking
- Ran 15-way blinded comparison: 4.38/5.0 vs 3.73/5.0
- Won all 5 topics, zero regressions
- Cross-source linking: 3 -> 13 linked items
- Demo topics: Anthropic odds (11 markets), Arizona basketball (6 markets), Iran war ($425M volume)
---
## Post 1: Lead (Announcement)
V2.5 of @slashlast30days is out. Now with @Polymarket prediction markets, cross-source linking, and massively better results.
1. Polymarket as a 6th source - real money on outcomes, no API key needed
2. Hacker News as a 5th source
3. Cross-source linking - detects when the same story trends across multiple platforms
Ran a 15-way blinded comparison across 5 topics. v2.5 scored 4.38 vs 3.73 for the original. Won all 5. Zero regressions.
github.com/mvanhorn/last30days-skill
---
## Post 2: POLYMARKET AS A 6TH SOURCE.
Reddit tells you what people think. X tells you what people share. YouTube tells you what people watch. HN tells you what developers discuss.
Polymarket helps you predict the future.
"/last30days anthropic odds"
11 markets found. Best AI model February: Anthropic 98%. IPO before OpenAI: 64%. $500B+ valuation: 87%. Pentagon ban odds: only 22%.
Free API. No key. Real money on outcomes.
---
## Post 3: CROSS-SOURCE LINKING.
When a Seedance 2.0 tutorial has 44K YouTube views AND trends on HN AND gets discussed on Reddit, v2.5 flags it: [also on: HN, YouTube]
Old version linked 3 items across 5 test topics. New version links 13. The difference is hybrid similarity - combining character-trigram and token-level matching at a tuned threshold.
Cross-platform convergence is the strongest signal that something actually matters. Not engagement on one platform. Convergence across all of them.
---
## Post 4: 15-WAY BLINDED EVALUATION.
I don't trust vibes for measuring quality. So I ran a scientific comparison.
5 topics x 3 versions. Stripped version labels. Randomized as A/B/C. Scored on groundedness, specificity, coverage, actionability, and format.
v2.5: 4.38/5.0
v2.2 (HN only): 4.10/5.0
v2.0 (original): 3.73/5.0
Won all 5 topics. Zero regressions. Biggest gains: specificity (+0.8) and format (+1.0) from cross-source linking giving the synthesis better material to work with.
---
## Post 5: Demo - Anthropic Odds
Asked it about Anthropic odds.
11 Polymarket markets. 25 X posts. 13 YouTube videos (719K views). 6 HN stories (471 points).
Best AI model February: 98%. IPO before OpenAI: 64%. $500B+ valuation: 87%. FrontierMath 50% score: 48% (up 28% today). Pentagon ban: only 22%.
Markets say Anthropic is winning the model race AND the valuation race. The Pentagon thing is noise.
---
## Post 6: Demo - Arizona Basketball
"/last30days arizona basketball"
6 Polymarket markets. 37 X posts (4,200 likes). 15 YouTube videos (517K views). 2 Reddit threads.
Championship odds: 13%. #1 seed: 88%. Big 12 title: Arizona leads by 2.
That's not a sports blog. That's Reddit reactions + X engagement + YouTube analysis + prediction market odds from one command.
The Polymarket integration uses two-pass query expansion. First pass finds "Arizona Big 12." Second pass discovers the championship and #1 seed markets via tag-based domain bridging.
---
## Post 7: Demo - Iran War
The best Polymarket demo is news.
"/last30days iran war"
14 Polymarket markets. $425M+ in volume. 7 Reddit threads. 30 X posts. 20 YouTube videos (2M views). 18 HN stories (1,187 points).
US strikes Iran by 2026: 70%. War Powers by March: 60%. Israel strikes by June: 64%. Formal war declaration: only 8%.
Markets say: limited strikes with War Powers, NOT a declared war. Breaking Points (435K views) covered leaked Pentagon opposition. r/Conservative "imploding" per r/SubredditDrama.
One command. Six sources. Real money.
---
## Post 8: Credits + CTA
Also in v2.5: YouTube synonym expansion ("hip hop" now matches "rap" - relevance jumped 0.33 to 0.71), X handle resolution, and HN OR queries for framework topics.
The difference between "good research" and "research you'd actually trust" is in details like this.
Try it: /last30days [any topic]
github.com/mvanhorn/last30days-skill
---
## Post 9: Demo - Claude Code (ALL 6 sources)
"/last30days Claude Code"
3 Reddit threads (199 upvotes). 35 X posts (5,239 likes). 15 YouTube videos (1.4M views, 5 transcripts). 30 HN stories (~8,500 points). 8 Polymarket markets. 20 web pages.
All six sources hit. Top finding: the planning-first workflow has won. The #1 HN post this month (969 pts, 590 comments) is about separating planning from execution. Boris Cherny (Head of Claude Code) on Lenny's Podcast: "100% of my code is written by Claude Code - I have not edited a single line by hand since November."
Polymarket: Anthropic 99% for best AI model in February. 58% for March. Claude on FrontierMath at 55%. The US government rejected Claude - Polymarket has the Hegseth ban at 32%.
Then I asked it to dig deeper into the planning-first workflow. No new searches - it answered from what it already learned.
---
## Post 10: Demo - March Madness Odds (Polymarket + Sports)
"/last30days March Madness Odds"
2 Reddit threads. 31 X posts. 6 YouTube videos (46K views, 4 transcripts). 2 Polymarket markets.
Tournament winner: Duke 18%, Michigan 18%, Arizona 13%. #1 seeds: Michigan 98%, Duke 91%, Arizona 88%.
Duke is the hottest mover - went from +700 to +450 in one week. The skill surfaced that from sportsbook data, X commentary, and Polymarket odds simultaneously.
Then I asked it to break down Michigan vs Duke vs Arizona. Full analysis from the research it already had.
---
## Post 11: Demo - Seedance Prompting (Expert + Prompt Mode)
"/last30days Seedance prompting"
13 Reddit threads. 33 X posts. 20 YouTube videos (1.2M views, 4 transcripts). 1 HN story. 15 web pages.
Top finding: Seedance 2.0 prompts follow a director's shot-list format, not freeform text. 30-100 words. Subject + Action + Camera + Scene + Style + Constraints. Beyond 100 words, results degrade.
Then I said: "a cinematic drone shot over a city at golden hour"
It wrote me a copy-paste prompt using the exact patterns from the research. That's the skill's real power - research first, then create from what you learned.
---
## Post 12: Thank You + CTA (Final)
Thank you to ARJ999 and wkbaran on GitHub who kept asking for Hacker News support. Three separate issues. v2.5 delivers.
30 days. 3.2k stars. 6 sources. Massively better results.
Try it: /last30days [any topic]
github.com/mvanhorn/last30days-skill
---
## Post 13: Demo - Anthropic vs Pentagon (Breaking News + Polymarket)
"/last30days Anthropic Pete Hegseth"
14 Reddit threads. 29 X posts (11,559 likes). 20 YouTube videos (739K views, 5 transcripts). 5 HN stories. 9 Polymarket markets. 10 web pages.
This story broke TODAY. Defense Secretary Hegseth designated Anthropic a "supply chain risk" - believed to be the first time an American company has ever received this designation. Trump ordered every federal agency to stop using Anthropic tech.
Polymarket: Anthropic still 99% for best AI model. $500B+ valuation: 68%. IPO >$600B: 97%. Hegseth out by March 31: only 6%.
Markets say: Anthropic wins the model race regardless. Bettors don't think Hegseth survives this. That's the kind of signal you can't get from opinion threads.
---
## Post 14: Demo - OpenAI Insider Trading (News + Polymarket)
"/last30days OpenAI Insider Trading"
2 Reddit threads. 29 X posts. 4 YouTube videos (360K views, 4 transcripts). 2 HN stories. 15 Polymarket markets. 15 web pages.
An OpenAI employee was just fired for using confidential info to bet on Polymarket. 13 brand-new wallets appeared 40 hours before the browser launch. $309K bet on the right outcome. Unusual Whales flagged 77 suspected insider positions across 60 wallets.
Meanwhile Polymarket has OpenAI's IPO at $1.25-1.5T: 54%. Anthropic IPOs first: 62%. Best AI model: Anthropic 99%.
The prediction markets are both the story AND the source. One command pulled all of it together.
---
## Standalone Tweet: Polymarket Stats Line
"/last30days Anthropic Pete Hegseth"
The Pentagon just designated Anthropic a supply chain risk. First time ever for an American company. Trump ordered every agency to stop using their tech.
Here's what Polymarket says:
📊 9 markets │ Best AI model: 99% │ $500B+ valuation: 68% │ IPO >$600B: 97% │ Hegseth out by March: 6%
Bettors with real money on the line think Anthropic wins the model race, goes public at a massive valuation, and Hegseth doesn't survive this.
That's the gap between headlines and reality. One command, six sources.
github.com/mvanhorn/last30days-skill
---
## Recommended Thread Order (pick 8-10)
The full thread above is 12 posts. Here's what I'd cut to keep it tight:
**Must include (core story):**
1. Post 1 - Lead announcement
2. Post 2 - Polymarket ("Reddit tells you what people think...")
3. Post 3 - Cross-source linking
4. Post 4 - Blinded evaluation
**Best demos (pick 3-4):**
- Post 13 (Anthropic vs Pentagon) - STRONGEST. Breaking news today. Polymarket cuts through the noise. "Markets say Anthropic wins regardless."
- Post 9 (Claude Code) - All 6 sources. Massive numbers. Shows follow-up flow.
- Post 10 (March Madness) - Sports/Polymarket crossover. Timely with tournament approaching.
- Post 11 (Seedance) - Shows prompting flow. 1.2M YouTube views.
- Post 5 (Anthropic Odds) - Overlaps with Post 13 now. Skip.
**Skip or save for standalone tweets:**
- Post 5 (Anthropic Odds) - Redundant with Post 13
- Post 6 (Arizona Basketball) - Covered by March Madness now
- Post 7 (Iran War) - Great standalone tweet, not for launch thread
- Post 8 (Credits/minor features) - Fold into CTA
**My recommended 8-post thread:**
1. Lead (Post 1)
2. Polymarket (Post 2)
3. Cross-source linking (Post 3)
4. Blinded evaluation (Post 4)
5. Demo: Anthropic vs Pentagon (Post 13) - breaking news, best Polymarket showcase
6. Demo: Claude Code (Post 9)
7. Demo: March Madness (Post 10)
8. Thank you + CTA (Post 12)
---
## Video Script (~60 seconds)
**[Talking to camera]**
Oh my god, I can't believe it's been 30 days since I launched last30days. 3,200 stars on GitHub. This has been the craziest month.
Today I'm shipping v2.5 and I'm really excited about this one. Three big things.
**[Screen recording: typing /last30days Anthropic Pete Hegseth]**
First - Polymarket prediction markets as a 6th source. So this Anthropic-Pentagon story broke today. Hegseth designated Anthropic a supply chain risk, Trump ordered agencies to stop using their tech. Scary headline, right?
But Polymarket says: Anthropic still 99% for best AI model. IPO above 600 billion: 97%. Hegseth out by March: 6%. Real money on outcomes helps you predict the future. That's a different story than the headlines.
**[Screen recording: typing /last30days arizona basketball]**
Second - it now searches Hacker News and does cross-source linking. When the same story shows up on Reddit AND YouTube AND HN, it flags it. I ran a 15-way blinded comparison and v2.5 scored 4.38 versus 3.73 for the original. Won all 5 test topics.
**[Back to camera]**
Thank you to everyone who starred it, filed issues, and kept pushing me to make this better. Shoutout to the people on GitHub who literally filed three separate issues asking for Hacker News. v2.5 delivers.
Link in bio. Try it on anything.
---
## Scoring Note
The 4.38 vs 3.73 score is from a custom 5-dimension rubric (30% groundedness, 25% specificity, 20% coverage, 15% actionability, 10% format compliance) evaluated by Claude on blinded outputs. The relative ranking is meaningful; the absolute numbers are not. It's an LLM grading LLM output - useful for A/B comparison, not for claiming "4.38 out of 5 quality."
If using the score in a tweet, frame it as "scored X vs Y on a blinded comparison" not "rated 4.38/5.0 quality" - the former is honest, the latter implies an objective standard that doesn't exist.
fixtures/eval_topics.json
+42
View File
@@ -0,0 +1,42 @@
[
{
"topic": "OpenClaw vs NanoClaw vs ZeroClaw",
"query_type": "comparison",
"rationale": "Multi-entity extraction, 3-way split across AI agent frameworks."
},
{
"topic": "how to set up a GLP-1 supplement routine",
"query_type": "how_to",
"rationale": "Trending health topic. Tests non-tech how_to."
},
{
"topic": "2026 March Madness",
"query_type": "breaking_news",
"rationale": "Live sporting event. Tests broad breaking news recall."
},
{
"topic": "best budget noise cancelling headphones 2026",
"query_type": "product",
"rationale": "Evergreen consumer query. Tests product review aggregation."
},
{
"topic": "thoughts on OpenAI Codex pricing",
"query_type": "opinion",
"rationale": "Active developer debate. Tests opinion mining."
},
{
"topic": "odds of US recession 2026",
"query_type": "prediction",
"rationale": "Major macro topic. Tests prediction market + news synthesis."
},
{
"topic": "what is retrieval augmented generation",
"query_type": "concept",
"rationale": "Widely discussed AI concept. Tests explanation quality."
},
{
"topic": "Google Wiz acquisition price and timeline",
"query_type": "factual",
"rationale": "Completed event ($32B). Tests factual precision."
}
]
fixtures/models_openai_sample.json
+41
View File
@@ -0,0 +1,41 @@
{
"object": "list",
"data": [
{
"id": "gpt-5.2",
"object": "model",
"created": 1704067200,
"owned_by": "openai"
},
{
"id": "gpt-5.1",
"object": "model",
"created": 1701388800,
"owned_by": "openai"
},
{
"id": "gpt-5",
"object": "model",
"created": 1698710400,
"owned_by": "openai"
},
{
"id": "gpt-5-mini",
"object": "model",
"created": 1704067200,
"owned_by": "openai"
},
{
"id": "gpt-4o",
"object": "model",
"created": 1683158400,
"owned_by": "openai"
},
{
"id": "gpt-4-turbo",
"object": "model",
"created": 1680566400,
"owned_by": "openai"
}
]
}
fixtures/models_xai_sample.json
+23
View File
@@ -0,0 +1,23 @@
{
"object": "list",
"data": [
{
"id": "grok-4-latest",
"object": "model",
"created": 1704067200,
"owned_by": "xai"
},
{
"id": "grok-4",
"object": "model",
"created": 1701388800,
"owned_by": "xai"
},
{
"id": "grok-3",
"object": "model",
"created": 1698710400,
"owned_by": "xai"
}
]
}
fixtures/openai_sample.json
+22
View File
@@ -0,0 +1,22 @@
{
"id": "resp_mock123",
"object": "response",
"created": 1706140800,
"model": "gpt-5.2",
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "{\n \"items\": [\n {\n \"title\": \"Best practices for Claude Code skills - comprehensive guide\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-15\",\n \"why_relevant\": \"Detailed discussion of skill creation patterns and best practices\",\n \"relevance\": 0.95\n },\n {\n \"title\": \"How I built a research skill for Claude Code\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/def456/how_i_built_a_research_skill\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-10\",\n \"why_relevant\": \"Real-world example of building a Claude Code skill with API integrations\",\n \"relevance\": 0.90\n },\n {\n \"title\": \"Claude Code vs Cursor vs Windsurf - January 2026 comparison\",\n \"url\": \"https://reddit.com/r/LocalLLaMA/comments/ghi789/claude_code_vs_cursor_vs_windsurf\",\n \"subreddit\": \"LocalLLaMA\",\n \"date\": \"2026-01-08\",\n \"why_relevant\": \"Compares Claude Code features including skills system\",\n \"relevance\": 0.85\n },\n {\n \"title\": \"Tips for effective prompt engineering in Claude Code\",\n \"url\": \"https://reddit.com/r/PromptEngineering/comments/jkl012/tips_for_claude_code_prompts\",\n \"subreddit\": \"PromptEngineering\",\n \"date\": \"2026-01-05\",\n \"why_relevant\": \"Discusses prompt patterns that work well with Claude Code skills\",\n \"relevance\": 0.80\n },\n {\n \"title\": \"New Claude Code update: improved skill loading\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/mno345/new_claude_code_update_improved_skill_loading\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-03\",\n \"why_relevant\": \"Announcement of new skill features in Claude Code\",\n \"relevance\": 0.75\n }\n ]\n}"
}
]
}
],
"usage": {
"prompt_tokens": 150,
"completion_tokens": 500,
"total_tokens": 650
}
}
fixtures/reddit_listing_cards_sample.html
+8
View File
@@ -0,0 +1,8 @@
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/community-more-posts/top/?name=technology&t=week on 2026-05-29; trimmed to 5 post cards (start-tag attrs only). -->
<div id="feed">
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tq0zk7/the_netherlands_just_blocked_a_us_company_from/" content-href="https://www.techspot.com/news/112552-netherlands-blocked-us-company-buying-app-dutch-citizens.html" view-context="SubredditFeed" comment-count="1743" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="0" award-count="23" award-id="award_obsessed_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/Obsessed_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-28T11:37:01.506000+0000" domain="techspot.com" id="t3_1tq0zk7" post-title="The Netherlands just blocked a US company from buying the app Dutch citizens use for everything" post-language="en" post-type="link" score="52692" upvote-ratio="0.9606269354736776" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&amp;height=64&amp;frame=1&amp;auto=webp&amp;crop=64%3A64%2Csmart&amp;s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1toe7m2/erin_brockovich_launches_map_of_over_4200_data/" content-href="https://www.newsweek.com/erin-brockovich-asks-americans-for-help-as-she-launches-data-center-map-11989813" view-context="SubredditFeed" comment-count="673" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="2" award-count="6" award-id="award_this_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/this_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T17:39:43.272000+0000" domain="newsweek.com" id="t3_1toe7m2" post-title="Erin Brockovich launches map of over 4,200 data centres in the US, appeals for local communities to report environmental impact and other costs" post-language="en" post-type="link" score="33567" upvote-ratio="0.973297166968053" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_fj9vsvfd" author="marketrent" icon="https://www.redditstatic.com/avatars/defaults/v2/avatar_default_1.png" data-expected-lcp subreddit-name="technology"></shreddit-post>
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tollgz/majority_of_americans_support_ban_on_surveillance/" content-href="https://gizmodo.com/majority-of-americans-support-ban-on-surveillance-pricing-and-electronic-shelf-labels-2000762717" view-context="SubredditFeed" comment-count="1043" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="3" award-count="7" award-id="award_free_bravo" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/bravo_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T21:55:07.322000+0000" domain="gizmodo.com" id="t3_1tollgz" post-title="Majority of Americans Support Ban on Surveillance Pricing and Electronic Shelf Labels" post-language="en" post-type="link" score="29791" upvote-ratio="0.9815063671850003" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_98wao505" author="Plastic_Ninja_9014" icon="https://preview.redd.it/snoovatar/avatars/69af2b53-b0a1-4ab6-b119-d90f21c423fe-headshot.png?width=64&amp;height=64&amp;crop=smart&amp;auto=webp&amp;s=f3661eb511798004968f8b115a689dcee30f1428" data-expected-lcp subreddit-name="technology"></shreddit-post>
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tp5qz2/tech_ceos_are_apparently_suffering_from_ai/" content-href="https://techcrunch.com/2026/05/27/tech-ceos-are-apparently-suffering-from-ai-psychosis/" view-context="SubredditFeed" comment-count="1653" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="4" award-count="6" award-id="award_free_regret_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/regret_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-27T13:33:49.280000+0000" domain="techcrunch.com" id="t3_1tp5qz2" post-title="Tech CEOs are apparently suffering from AI psychosis" post-language="en" post-type="link" score="26419" upvote-ratio="0.9605741880002646" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&amp;height=64&amp;frame=1&amp;auto=webp&amp;crop=64%3A64%2Csmart&amp;s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tn5g7s/pope_leo_issues_ai_encyclical_warning_that_opaque/" content-href="https://variety.com/2026/biz/global/pope-leo-ai-encyclical-algorithms-threaten-dehumanisation-1236758186/" view-context="SubredditFeed" comment-count="608" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="6" award-count="7" award-id="award_hooray_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/FTUE_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-25T10:45:04.093000+0000" domain="variety.com" id="t3_1tn5g7s" post-title="Pope Leo Issues AI Encyclical Warning That ‘Opaque Algorithms’ Controlled by a ‘Few’ Companies Can Bring ‘New Forms of Dehumanisation’" post-language="en" post-type="link" score="25835" upvote-ratio="0.9760626539506095" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_1i1zizibn9" author="yourfavchoom" icon="https://styles.redditmedia.com/t5_dgdrt8/styles/profileIcon_k9x929ihm8rg1.png?width=64&amp;height=64&amp;frame=1&amp;auto=webp&amp;crop=64%3A64%2Csmart&amp;s=2e8a5042cccc4555167f98d28bc0de4e13fd3ca5" data-expected-lcp subreddit-name="technology"></shreddit-post>
</div>
fixtures/reddit_search_rss_sample.xml
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- FIXTURE: captured live from reddit.com/r/Rakuten/top.rss on 2026-05-29; trimmed to 5 entries. Atom shape identical to search.rss. --><feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/"><category term="Rakuten" label="r/Rakuten"/><updated>2026-05-29T14:14:32+00:00</updated><icon>https://www.redditstatic.com/icon.png/</icon><id>/r/Rakuten/top.rss?t=month</id><link rel="self" href="https://www.reddit.com/r/Rakuten/top.rss?t=month" type="application/atom+xml" /><link rel="alternate" href="https://www.reddit.com/r/Rakuten/top?t=month" type="text/html" /><subtitle>This is an unofficial subreddit for Rakuten Rewards, the cash back website. We are not affiliated with, endorsed by, or sponsored by Rakuten or any of its subsidiaries.</subtitle><title>top scoring links : Rakuten</title><entry><author><name>/u/InternetUser52</name><uri>https://www.reddit.com/user/InternetUser52</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&lt;!-- SC_OFF --&gt;&lt;div class=&quot;md&quot;&gt;&lt;p&gt;I&amp;#39;m rich!!&lt;/p&gt; &lt;/div&gt;&lt;!-- SC_ON --&gt; &amp;#32; submitted by &amp;#32; &lt;a href=&quot;https://www.reddit.com/user/InternetUser52&quot;&gt; /u/InternetUser52 &lt;/a&gt; &lt;br/&gt; &lt;span&gt;&lt;a href=&quot;https://i.redd.it/q8fgmxs29c2h1.jpeg&quot;&gt;[link]&lt;/a&gt;&lt;/span&gt; &amp;#32; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/&quot;&gt;[comments]&lt;/a&gt;&lt;/span&gt;</content><id>t3_1tiv013</id><link href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/" /><updated>2026-05-20T18:48:31+00:00</updated><published>2026-05-20T18:48:31+00:00</published><title>LETS GOO! $0.02!!!</title></entry>
<entry><author><name>/u/Immediate-Duck-6351</name><uri>https://www.reddit.com/user/Immediate-Duck-6351</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&lt;!-- SC_OFF --&gt;&lt;div class=&quot;md&quot;&gt;&lt;p&gt;I don’t travel and I’m buying a house in a few weeks so cash back is amazing 🙌 hoping to keep the pace in the next quarter so I can buy new kitchen appliances lol. &lt;/p&gt; &lt;/div&gt;&lt;!-- SC_ON --&gt; &amp;#32; submitted by &amp;#32; &lt;a href=&quot;https://www.reddit.com/user/Immediate-Duck-6351&quot;&gt; /u/Immediate-Duck-6351 &lt;/a&gt; &lt;br/&gt; &lt;span&gt;&lt;a href=&quot;https://i.redd.it/d2a4s0ipvb1h1.jpeg&quot;&gt;[link]&lt;/a&gt;&lt;/span&gt; &amp;#32; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/&quot;&gt;[comments]&lt;/a&gt;&lt;/span&gt;</content><id>t3_1te1fp8</id><link href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/" /><updated>2026-05-15T16:29:28+00:00</updated><published>2026-05-15T16:29:28+00:00</published><title>So excited 🥳</title></entry>
<entry><author><name>/u/gnibgnib</name><uri>https://www.reddit.com/user/gnibgnib</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&lt;!-- SC_OFF --&gt;&lt;div class=&quot;md&quot;&gt;&lt;p&gt;128k for the May transfer&lt;/p&gt; &lt;p&gt;41k pending for August &lt;/p&gt; &lt;p&gt;Got another 9k at Asics not showing but overall pretty happy with Rakuten&lt;/p&gt; &lt;p&gt;P2 was able to secure 85k for May transfer&lt;/p&gt; &lt;/div&gt;&lt;!-- SC_ON --&gt; &amp;#32; submitted by &amp;#32; &lt;a href=&quot;https://www.reddit.com/user/gnibgnib&quot;&gt; /u/gnibgnib &lt;/a&gt; &lt;br/&gt; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/gallery/1tb8674&quot;&gt;[link]&lt;/a&gt;&lt;/span&gt; &amp;#32; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/&quot;&gt;[comments]&lt;/a&gt;&lt;/span&gt;</content><id>t3_1tb8674</id><link href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/" /><updated>2026-05-12T17:17:19+00:00</updated><published>2026-05-12T17:17:19+00:00</published><title>Had a great run so far this year thanks to this sub!</title></entry>
<entry><author><name>/u/TravelVet93</name><uri>https://www.reddit.com/user/TravelVet93</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&amp;#32; submitted by &amp;#32; &lt;a href=&quot;https://www.reddit.com/user/TravelVet93&quot;&gt; /u/TravelVet93 &lt;/a&gt; &lt;br/&gt; &lt;span&gt;&lt;a href=&quot;https://i.redd.it/x6b9whvupb1h1.jpeg&quot;&gt;[link]&lt;/a&gt;&lt;/span&gt; &amp;#32; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/&quot;&gt;[comments]&lt;/a&gt;&lt;/span&gt;</content><id>t3_1te0hom</id><link href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/" /><updated>2026-05-15T15:56:40+00:00</updated><published>2026-05-15T15:56:40+00:00</published><title>My best payout so far</title></entry>
<entry><author><name>/u/Beautiful-Piece-4252</name><uri>https://www.reddit.com/user/Beautiful-Piece-4252</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&lt;!-- SC_OFF --&gt;&lt;div class=&quot;md&quot;&gt;&lt;p&gt;The amount of $$ available in sign up bonuses is amazing. It&amp;#39;s kind of a part time job ensuring Rakuten captures everything, but my August and November payout should be sizeable. I&amp;#39;m new to this and it always seemed like a lot of work for little reward. I know it&amp;#39;s not sustainable, but wow!&lt;/p&gt; &lt;/div&gt;&lt;!-- SC_ON --&gt; &amp;#32; submitted by &amp;#32; &lt;a href=&quot;https://www.reddit.com/user/Beautiful-Piece-4252&quot;&gt; /u/Beautiful-Piece-4252 &lt;/a&gt; &lt;br/&gt; &lt;span&gt;&lt;a href=&quot;https://i.redd.it/1vqvajsci42h1.jpeg&quot;&gt;[link]&lt;/a&gt;&lt;/span&gt; &amp;#32; &lt;span&gt;&lt;a href=&quot;https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/&quot;&gt;[comments]&lt;/a&gt;&lt;/span&gt;</content><id>t3_1thsnm1</id><link href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/" /><updated>2026-05-19T16:46:17+00:00</updated><published>2026-05-19T16:46:17+00:00</published><title>How can this be real?</title></entry>
</feed>
fixtures/reddit_shreddit_comments_sample.html
+29
View File
@@ -0,0 +1,29 @@
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0 on 2026-05-29;
trimmed to 6 real comment elements (real attrs + real bodies) + 2 synthetic edge cases. -->
<shreddit-comment-tree-stats total-comments="14"></shreddit-comment-tree-stats>
<shreddit-comment-tree id="comment-tree" post-id="t3_1taeiw0">
<shreddit-comment created="2026-05-11T20:16:57.590000+0000" author="Obvious_Painting_881" thingId="t1_ol8tp8n" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8tp8n/" score="2" postId="t3_1taeiw0" content-type="text">
<div id="t1_ol8tp8n-comment-rtjson-content" slot="comment"><div id="t1_ol8tp8n-post-rtjson-content" dir="auto"><p dir="auto">Where do you find $750? The highest available package for Total was $284.99 when I did the lifelock promotion. I did get the full 284.99 from Rakuten.</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-12T12:26:14.973000+0000" author="Stormtrooper149" thingId="t1_olcy1iv" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/olcy1iv/" score="2" postId="t3_1taeiw0" content-type="text">
<div id="t1_olcy1iv-comment-rtjson-content" slot="comment"><div id="t1_olcy1iv-post-rtjson-content" dir="auto"><p dir="auto">It went to pending ($712.49)</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-19T01:43:48.026000+0000" author="heythereyou01" thingId="t1_omlbiqg" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/omlbiqg/" score="1" postId="t3_1taeiw0" content-type="text">
<div id="t1_omlbiqg-comment-rtjson-content" slot="comment"><div id="t1_omlbiqg-post-rtjson-content" dir="auto"><p dir="auto">Hey I PM’d. can I get the screenshot ?</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-11T20:21:16.398000+0000" author="Stormtrooper149" thingId="t1_ol8undb" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8undb/" score="1" postId="t3_1taeiw0" content-type="text">
<div id="t1_ol8undb-comment-rtjson-content" slot="comment"><div id="t1_ol8undb-post-rtjson-content" dir="auto"><p dir="auto">Family plan</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-11T20:28:33.803000+0000" author="Obvious_Painting_881" thingId="t1_ol8w8w6" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8w8w6/" score="1" postId="t3_1taeiw0" content-type="text">
<div id="t1_ol8w8w6-comment-rtjson-content" slot="comment"><div id="t1_ol8w8w6-post-rtjson-content" dir="auto"><p dir="auto">Price seems to change every time I go to the page but I see only 249.99-369.99 for Total/Advanced. No where near your $750. Just saying the Total plan for 299.99 worked for me and I got 284.99 which is 95%.</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-12T02:33:48.200000+0000" author="jwegener" thingId="t1_olaqzjk" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/olaqzjk/" score="2" postId="t3_1taeiw0" content-type="text">
<div id="t1_olaqzjk-comment-rtjson-content" slot="comment"><div id="t1_olaqzjk-post-rtjson-content" dir="auto"><p dir="auto">I did that one. Let’s pray</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-13T10:00:00.000000+0000" author="[deleted]" thingId="t1_synthdel" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/synthdel/" score="5" postId="t3_1taeiw0" content-type="text">
<div id="t1_synthdel-comment-rtjson-content" slot="comment"><div id="t1_synthdel-post-rtjson-content" dir="auto"><p dir="auto">[removed]</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-05-13T11:00:00.000000+0000" author="NegScoreUser" thingId="t1_synthneg" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/synthneg/" score="-7" postId="t3_1taeiw0" content-type="text">
<div id="t1_synthneg-comment-rtjson-content" slot="comment"><div id="t1_synthneg-post-rtjson-content" dir="auto"><p dir="auto">A downvoted but real reply with negative score for edge-case coverage.</p></div></div>
</shreddit-comment>
</shreddit-comment-tree>
fixtures/reddit_thread_sample.json
+108
View File
@@ -0,0 +1,108 @@
[
{
"kind": "Listing",
"data": {
"children": [
{
"kind": "t3",
"data": {
"title": "Best practices for Claude Code skills - comprehensive guide",
"score": 847,
"num_comments": 156,
"upvote_ratio": 0.94,
"created_utc": 1705363200,
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/",
"selftext": "After building 20+ skills for Claude Code, here are my key learnings..."
}
}
]
}
},
{
"kind": "Listing",
"data": {
"children": [
{
"kind": "t1",
"data": {
"score": 234,
"created_utc": 1705366800,
"author": "skill_expert",
"body": "Great guide! One thing I'd add: always use explicit tool permissions in your SKILL.md. Don't default to allowing everything.",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment1/"
}
},
{
"kind": "t1",
"data": {
"score": 189,
"created_utc": 1705370400,
"author": "claude_dev",
"body": "The context: fork tip is gold. I was wondering why my heavy research skill was slow - it was blocking the main thread!",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment2/"
}
},
{
"kind": "t1",
"data": {
"score": 145,
"created_utc": 1705374000,
"author": "ai_builder",
"body": "For anyone starting out: begin with a simple skill that just runs one bash command. Once that works, build up complexity gradually.",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment3/"
}
},
{
"kind": "t1",
"data": {
"score": 98,
"created_utc": 1705377600,
"author": "dev_tips",
"body": "The --mock flag pattern for testing without API calls is essential. I always build that in from day one now.",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment4/"
}
},
{
"kind": "t1",
"data": {
"score": 76,
"created_utc": 1705381200,
"author": "code_writer",
"body": "Thanks for sharing! Question: how do you handle API key storage securely in skills?",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment5/"
}
},
{
"kind": "t1",
"data": {
"score": 65,
"created_utc": 1705384800,
"author": "security_minded",
"body": "I use ~/.config/skillname/.env with chmod 600. Never hardcode keys, and definitely don't commit them!",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment6/"
}
},
{
"kind": "t1",
"data": {
"score": 52,
"created_utc": 1705388400,
"author": "helpful_user",
"body": "The caching pattern you described saved me so much on API costs. 24h TTL is perfect for most research skills.",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment7/"
}
},
{
"kind": "t1",
"data": {
"score": 34,
"created_utc": 1705392000,
"author": "newbie_coder",
"body": "This is exactly what I needed. Starting my first skill this weekend!",
"permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment8/"
}
}
]
}
}
]
fixtures/tiktok_search.json
+58
View File
@@ -0,0 +1,58 @@
{
"items": [
{
"video_id": "7543693751290481942",
"text": "This Claude Code trick saved me hours #claudecode #ai #coding",
"url": "https://www.tiktok.com/@codemaster/video/7543693751290481942",
"author_name": "codemaster",
"date": "2026-02-28",
"engagement": {
"views": 2100000,
"likes": 45000,
"comments": 1200,
"shares": 8400
},
"hashtags": ["claudecode", "ai", "coding"],
"duration": 45,
"relevance": 0.85,
"why_relevant": "TikTok: This Claude Code trick saved me hours #claude",
"caption_snippet": "So I found this insane trick with Claude Code where you can use slash commands to automate everything"
},
{
"video_id": "7543100200112345678",
"text": "AI coding tools comparison 2026 - Claude vs Copilot vs Cursor #ai #devtools",
"url": "https://www.tiktok.com/@techreviewer/video/7543100200112345678",
"author_name": "techreviewer",
"date": "2026-02-25",
"engagement": {
"views": 850000,
"likes": 22000,
"comments": 890,
"shares": 3200
},
"hashtags": ["ai", "devtools"],
"duration": 60,
"relevance": 0.7,
"why_relevant": "TikTok: AI coding tools comparison 2026 - Claude vs Copi",
"caption_snippet": ""
},
{
"video_id": "7543200300223456789",
"text": "You need to try Claude Code RIGHT NOW #programming #tips",
"url": "https://www.tiktok.com/@devtips/video/7543200300223456789",
"author_name": "devtips",
"date": "2026-03-01",
"engagement": {
"views": 500000,
"likes": 15000,
"comments": 450,
"shares": 2100
},
"hashtags": ["programming", "tips"],
"duration": 30,
"relevance": 0.6,
"why_relevant": "TikTok: You need to try Claude Code RIGHT NOW #programm",
"caption_snippet": "Let me show you why Claude Code is the best AI coding tool right now"
}
]
}
fixtures/xai_sample.json
+22
View File
@@ -0,0 +1,22 @@
{
"id": "resp_xai_mock456",
"object": "response",
"created": 1706140800,
"model": "grok-4-latest",
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "{\n \"items\": [\n {\n \"text\": \"Just shipped my first Claude Code skill! The SKILL.md format is incredibly intuitive. Pro tip: use context: fork for resource-intensive operations.\",\n \"url\": \"https://x.com/devuser1/status/1234567890\",\n \"author_handle\": \"devuser1\",\n \"date\": \"2026-01-18\",\n \"engagement\": {\n \"likes\": 542,\n \"reposts\": 87,\n \"replies\": 34,\n \"quotes\": 12\n },\n \"why_relevant\": \"First-hand experience building Claude Code skills with practical tips\",\n \"relevance\": 0.92\n },\n {\n \"text\": \"Thread: Everything I learned building 10 Claude Code skills in 30 days. 1/ Start simple. Your first skill should be < 50 lines of markdown.\",\n \"url\": \"https://x.com/aibuilder/status/1234567891\",\n \"author_handle\": \"aibuilder\",\n \"date\": \"2026-01-12\",\n \"engagement\": {\n \"likes\": 1203,\n \"reposts\": 245,\n \"replies\": 89,\n \"quotes\": 56\n },\n \"why_relevant\": \"Comprehensive thread on skill building best practices\",\n \"relevance\": 0.95\n },\n {\n \"text\": \"The allowed-tools field in SKILL.md is crucial for security. Don't give skills more permissions than they need.\",\n \"url\": \"https://x.com/securitydev/status/1234567892\",\n \"author_handle\": \"securitydev\",\n \"date\": \"2026-01-08\",\n \"engagement\": {\n \"likes\": 328,\n \"reposts\": 67,\n \"replies\": 23,\n \"quotes\": 8\n },\n \"why_relevant\": \"Security best practices for Claude Code skills\",\n \"relevance\": 0.85\n },\n {\n \"text\": \"Loving the new /skill command in Claude Code. Makes testing skills so much easier during development.\",\n \"url\": \"https://x.com/codeenthusiast/status/1234567893\",\n \"author_handle\": \"codeenthusiast\",\n \"date\": \"2026-01-05\",\n \"engagement\": {\n \"likes\": 156,\n \"reposts\": 23,\n \"replies\": 12,\n \"quotes\": 4\n },\n \"why_relevant\": \"Discusses skill development workflow\",\n \"relevance\": 0.78\n }\n ]\n}"
}
]
}
],
"usage": {
"prompt_tokens": 180,
"completion_tokens": 450,
"total_tokens": 630
}
}
gemini-extension.json
+67
View File
@@ -0,0 +1,67 @@
{
"name": "last30days-skill",
"version": "3.13.0",
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
"settings": [
{
"name": "Extension Directory",
"description": "Extension installation directory (auto-set by Gemini CLI)",
"envVar": "GEMINI_EXTENSION_DIR",
"sensitive": false
},
{
"name": "ScrapeCreators API Key",
"description": "ScrapeCreators API Key for Reddit, TikTok, and Instagram search (required)",
"envVar": "SCRAPECREATORS_API_KEY",
"sensitive": true
},
{
"name": "OpenAI API Key",
"description": "OpenAI API Key - optional fallback for Reddit discovery",
"envVar": "OPENAI_API_KEY",
"sensitive": true
},
{
"name": "xAI API Key",
"description": "xAI API Key for X/Twitter search (optional)",
"envVar": "XAI_API_KEY",
"sensitive": true
},
{
"name": "OpenRouter API Key",
"description": "OpenRouter API Key (optional)",
"envVar": "OPENROUTER_API_KEY",
"sensitive": true
},
{
"name": "Parallel AI API Key",
"description": "Parallel AI API Key (optional)",
"envVar": "PARALLEL_API_KEY",
"sensitive": true
},
{
"name": "Brave Search API Key",
"description": "Brave Search API Key (optional)",
"envVar": "BRAVE_API_KEY",
"sensitive": true
},
{
"name": "Apify API Token",
"description": "Apify API Token (optional legacy)",
"envVar": "APIFY_API_TOKEN",
"sensitive": true
},
{
"name": "Twitter AUTH_TOKEN",
"description": "Twitter browser AUTH_TOKEN cookie for direct X search (optional)",
"envVar": "AUTH_TOKEN",
"sensitive": true
},
{
"name": "Twitter CT0",
"description": "Twitter browser CT0 cookie (optional, pair with AUTH_TOKEN)",
"envVar": "CT0",
"sensitive": true
}
]
}
greptile.json
+4
View File
@@ -0,0 +1,4 @@
{
"triggerOnUpdates": true,
"statusCheck": true
}
hooks/hooks.json
+15
View File
@@ -0,0 +1,15 @@
{
"hooks": {
"SessionStart": [
{
"matcher": "",
"hooks": [
{
"type": "command",
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
}
]
}
]
}
}
hooks/scripts/check-config.sh
Executable
+273
View File
@@ -0,0 +1,273 @@
#!/bin/bash
set -euo pipefail
# Check last30days configuration status and show appropriate welcome message.
# Priority for this status hook:
# .claude/last30days.env > ~/.config/last30days/.env > env vars > Keychain presence
PROJECT_ENV=".claude/last30days.env"
GLOBAL_ENV="$HOME/.config/last30days/.env"
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
GLOBAL_ENV="$LAST30DAYS_CONFIG_DIR/.env"
else
GLOBAL_ENV=""
fi
fi
# Ensure LAST30DAYS_MEMORY_DIR exists for HTML-brief / raw-markdown saves.
# SKILL.md and the engine default this via the same env-var fallback. Fresh
# installs otherwise fail silently on first --emit=html run. See #395.
mkdir -p "${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}" 2>/dev/null || true
# Helper: warn if file permissions are too open
check_perms() {
local file="$1"
if [[ ! -f "$file" ]]; then return; fi
# Git-for-Windows / MSYS / Cygwin run stat in noacl mode (always 644),
# so this POSIX check is a false positive. Windows perms use ACLs.
case "$(uname -s 2>/dev/null)" in
MINGW*|MSYS*|CYGWIN*) return ;;
esac
local perms
# Try GNU stat first (Linux), fall back to BSD stat (macOS).
# On Linux, `stat -f` prints filesystem info (not permissions) and exits 0,
# so the previous BSD-first ordering left $perms as multi-line garbage on
# every Linux session start and printed a false WARNING.
perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
chmod 600 "$file" && echo "/last30days: WARNING — $file had permissions $perms — auto-fixed with chmod 600" || echo "/last30days: WARNING — $file has permissions $perms (should be 600). Fix: chmod 600 $file"
fi
}
trim_ws() {
local s="$1"
s="${s#"${s%%[![:space:]]*}"}"
s="${s%"${s##*[![:space:]]}"}"
printf '%s' "$s"
}
strip_outer_quotes() {
local s="$1"
if [[ ${#s} -ge 2 ]]; then
if [[ "${s:0:1}" == '"' && "${s: -1}" == '"' ]]; then
s="${s:1:${#s}-2}"
elif [[ "${s:0:1}" == "'" && "${s: -1}" == "'" ]]; then
s="${s:1:${#s}-2}"
fi
fi
printf '%s' "$s"
}
# Load env file into variables for inspection (without exporting)
load_env_vars() {
local file="$1"
if [[ -f "$file" ]]; then
while IFS='=' read -r key value; do
# Skip comments, empty lines
[[ "$key" =~ ^[[:space:]]*# ]] && continue
[[ -z "$key" ]] && continue
key="$(trim_ws "$key")"
value="$(strip_outer_quotes "$(trim_ws "$value")")"
# Strip inline comments (# preceded by whitespace) to prevent
# command substitution in backtick-containing comments
value="${value%%[[:space:]]#*}"
if [[ -n "$key" && -n "$value" ]]; then
# printf -v writes via assignment semantics (global from inside a
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
printf -v "ENV_${key}" '%s' "$value"
fi
done < "$file"
fi
}
# Determine which config file is active
CONFIG_FILE=""
if [[ -f "$PROJECT_ENV" ]]; then
CONFIG_FILE="$PROJECT_ENV"
check_perms "$PROJECT_ENV"
elif [[ -f "$GLOBAL_ENV" ]]; then
CONFIG_FILE="$GLOBAL_ENV"
check_perms "$GLOBAL_ENV"
fi
# Load config if found
if [[ -n "$CONFIG_FILE" ]]; then
load_env_vars "$CONFIG_FILE"
fi
# Load Keychain item presence for status checks without reading secret values.
# Runtime credential resolution still happens in lib/env.py; this hook only
# needs to avoid stale first-run/source-count messages.
load_keychain_presence() {
case "$(uname -s 2>/dev/null)" in
Darwin*) ;;
*) return 0 ;;
esac
command -v security >/dev/null 2>&1 || return 0
local user key env_var current
user="${USER:-}"
if [[ -z "$user" ]]; then
user="$(id -un 2>/dev/null || true)"
fi
[[ -n "$user" ]] || return 0
for key in SETUP_COMPLETE OPENAI_API_KEY SCRAPECREATORS_API_KEY AUTH_TOKEN CT0 XAI_API_KEY BSKY_HANDLE EXA_API_KEY; do
env_var="ENV_${key}"
current="${!env_var:-}"
if [[ -z "$current" ]]; then
current="${!key:-}"
fi
[[ -n "$current" ]] && continue
if security find-generic-password -a "$user" -s "last30days-${key}" >/dev/null 2>&1; then
printf -v "ENV_${key}" '%s' "keychain"
fi
done
return 0
}
load_keychain_presence
# Check SETUP_COMPLETE (from file, env, or Keychain presence)
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
# Compute last-run summary line (if last-run.json exists)
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
else
LAST_RUN_FILE=""
fi
else
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
fi
LAST_RUN_LINE=""
# python3 -c, NOT a heredoc: bash 5.3 feeds heredocs to the child through a
# pipe and can deadlock in heredoc_write inside command substitution, hanging
# this hook forever at session start (observed on Homebrew bash 5.3.15).
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 -c '
import datetime
import json
import os
path = os.environ["LAST_RUN_FILE"]
try:
with open(path) as fh:
d = json.load(fh)
topic = (d.get("topic") or "?")[:60]
ts = d.get("timestamp", "")
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
if delta < 60: ago = f"{int(delta)}s ago"
elif delta < 3600: ago = f"{int(delta//60)}m ago"
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
else: ago = f"{int(delta//86400)}d ago"
total = d.get("total", 0)
print(f" Last run: \"{topic}\" · {ago} · {total} results")
except Exception:
pass
' 2>/dev/null || true)
fi
# Detect capability that doesn't need a config file: yt-dlp on PATH.
# Done before the new-user early-exit so first-run users with yt-dlp
# installed see YouTube is already available. See #394.
HAS_YTDLP=""
if command -v yt-dlp &>/dev/null; then
HAS_YTDLP="yes"
fi
# If setup has never been run, show welcome message for new users
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${ENV_OPENAI_API_KEY:-${OPENAI_API_KEY:-}}" && -z "${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}" && -z "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -z "${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}" ]]; then
# printf, NOT cat-with-heredoc: see the bash 5.3 heredoc deadlock note above.
if [[ -n "$HAS_YTDLP" ]]; then
# YouTube is already working via the on-system yt-dlp binary — don't list
# it as something the wizard needs to unlock. See #394.
printf '%s\n' \
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
'' \
'Reddit, Hacker News, Polymarket, and YouTube (yt-dlp detected) work out of the box.' \
'The setup wizard can unlock X/Twitter and more.' \
' Detected: yt-dlp is installed (YouTube transcripts ready, no setup needed).'
else
printf '%s\n' \
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
'' \
'Reddit, Hacker News, and Polymarket work out of the box.' \
'The setup wizard can unlock X/Twitter, YouTube, and more.'
fi
if [[ -n "$LAST_RUN_LINE" ]]; then
echo "$LAST_RUN_LINE"
fi
exit 0
fi
# Setup done but check for ScrapeCreators
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
HAS_X=""
if [[ -n "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -n "${ENV_CT0:-${CT0:-}}" ]]; then
HAS_X="yes"
fi
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
# Count active sources
SOURCE_COUNT=2 # HN + Polymarket are always free
if [[ -n "$HAS_X" || -n "$HAS_XAI" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
# Reddit public JSON always works
SOURCE_COUNT=$((SOURCE_COUNT + 1))
if [[ -n "$HAS_YTDLP" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_EXA" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_BSKY" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
# Normalise EXCLUDED by removing whitespace; case-insensitive matches below
# mirror pipeline.py's .strip().lower() parsing without requiring sed/tr.
SC_ADD=3
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
EXCLUDED_NORM="${EXCLUDED//[[:space:]]/}"
if [[ ",$EXCLUDED_NORM," == *",[Tt][Ii][Kk][Tt][Oo][Kk],"* ]]; then
SC_ADD=$((SC_ADD - 1))
fi
if [[ ",$EXCLUDED_NORM," == *",[Ii][Nn][Ss][Tt][Aa][Gg][Rr][Aa][Mm],"* ]]; then
SC_ADD=$((SC_ADD - 1))
fi
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
# Fully configured — compact ready message
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
echo " Research any topic across social + market + web sources (last 30 days)."
if [[ -n "$LAST_RUN_LINE" ]]; then
echo "$LAST_RUN_LINE"
fi
else
# Setup done but missing ScrapeCreators — recommend it
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
echo " Research any topic across social + market + web sources (last 30 days)."
if [[ -n "$LAST_RUN_LINE" ]]; then
echo "$LAST_RUN_LINE"
fi
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
echo " 100 free credits, no credit card — scrapecreators.com"
echo " last30days has no affiliation with any API provider."
fi
# The branches above end with `[[ -n "$LAST_RUN_LINE" ]] && echo ...`. When
# LAST_RUN_LINE is empty, that test returns 1 and is the script's last command,
# leaking exit=1 to callers (e.g. SessionStart hook drivers) despite no error.
exit 0
mcp/.gitignore
+12
View File
@@ -0,0 +1,12 @@
# Mirror of skills/last30days/scripts/, populated by scripts/sync-engine.sh.
# Source of truth lives in the Python skill; never commit the mirror.
# Lives inside internal/engine/ because //go:embed cannot reach outside
# its own package directory.
internal/engine/vendored/*
!internal/engine/vendored/.gitkeep
# Local build output: cross-compiled binaries and packaged .mcpb files.
build/
# Anchor to the mcp/ root so the cmd/last30days-pp-mcp/ package directory
# is not also excluded (subdirs with the same name would otherwise match).
/last30days-pp-mcp
mcp/README.md
+36
View File
@@ -0,0 +1,36 @@
# last30days-pp-mcp
Go MCP server that wraps the last30days Python engine for Claude Desktop. Packaged as a `.mcpb` bundle (drag-drop install into Claude Desktop).
The MCP server exposes a single `research` tool that mirrors the `/last30days <topic>` slash command available in Claude Code. At runtime the binary extracts the vendored Python engine into a per-user cache and shells out to `python3` to produce the synthesis input Claude renders.
## Architecture
- `cmd/last30days-pp-mcp/` - server entry point
- `internal/engine/` - `embed.FS` of the Python engine + cache extractor + subprocess wrapper
- `internal/tools/` - MCP tool handlers (currently `research`)
- `internal/engine/vendored/` - mirror of `skills/last30days/scripts/`, generated by `scripts/sync-engine.sh` (gitignored). Lives inside the engine package because `//go:embed` cannot reach files outside its own package directory.
- `manifest.json` - MCPB v0.3 manifest consumed by Claude Desktop and `printing-press bundle`
## Local build
```bash
# Mirror the Python engine into vendored/.
bash scripts/sync-engine.sh
# Build for the current host.
go build -ldflags "-X main.Version=dev" -o build/last30days-pp-mcp ./cmd/last30days-pp-mcp
# Package as a .mcpb (requires the printing-press binary on PATH).
printing-press bundle . --skip-build --binary build/last30days-pp-mcp
```
The output `.mcpb` lands at `build/last30days-pp-mcp-<os>-<arch>.mcpb`. Drag it into Claude Desktop's Extensions panel to install.
## Runtime requirements
End users need Python 3.12+ on PATH. The bundle ships the engine source but relies on the host interpreter.
## Versioning
The MCPB `manifest.json` version is hand-bumped in the same PR that ships engine changes worth releasing. Release CI stamps the Go binary's `main.Version` from the tag.
mcp/cmd/last30days-pp-mcp/main.go
+39
View File
@@ -0,0 +1,39 @@
// Package main is the entry point for the last30days MCP server bundled
// as a .mcpb for Claude Desktop. The server registers a single research
// tool (see internal/tools) and serves it over stdio. See mcp/README.md
// for build and packaging instructions.
package main
import (
"fmt"
"os"
"github.com/mark3labs/mcp-go/server"
"github.com/mvanhorn/last30days-skill/mcp/internal/tools"
)
// Version is stamped at build time via -ldflags "-X main.Version=<tag>".
// It namespaces the per-user cache directory in internal/engine so multiple
// installed versions can coexist without clobbering each other.
var Version = "dev"
const (
serverName = "last30days"
serverVersion = "1"
)
func main() {
s := server.NewMCPServer(
serverName,
serverVersion,
server.WithToolCapabilities(false),
)
tools.Register(s, tools.Config{Version: Version})
if err := server.ServeStdio(s); err != nil {
fmt.Fprintf(os.Stderr, "last30days-pp-mcp: %v\n", err)
os.Exit(1)
}
}
mcp/go.mod
+14
View File
@@ -0,0 +1,14 @@
module github.com/mvanhorn/last30days-skill/mcp
go 1.25.5
require github.com/mark3labs/mcp-go v0.55.0
require (
github.com/google/jsonschema-go v0.4.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
golang.org/x/text v0.14.0 // indirect
)
mcp/go.sum
+34
View File
@@ -0,0 +1,34 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mark3labs/mcp-go v0.55.0 h1:lJfz2aoctiwK+sI991+uIYwmKNIBciI+O7zsyDsa4U8=
github.com/mark3labs/mcp-go v0.55.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
mcp/internal/engine/embed.go
+26
View File
@@ -0,0 +1,26 @@
// Package engine wraps the vendored Python last30days engine. The engine
// is embedded at build time via //go:embed and extracted into a per-user
// cache directory on first use, then invoked through python3 in a
// subprocess. Consumers should call EnsureUserCache to materialize the
// engine and Run to execute it.
package engine
import (
"embed"
"io/fs"
)
// EngineSourceDir is the embed root inside the binary. scripts/sync-engine.sh
// mirrors skills/last30days/scripts/ into this directory before each build.
// The all: prefix preserves files starting with "." or "_" so the .gitkeep
// anchor file survives - without it the embed would error before sync runs.
//
//go:embed all:vendored
var vendored embed.FS
// EngineFS returns the embedded engine as a filesystem rooted at the
// vendored/ directory contents (so callers see "last30days.py" at the
// root, not "vendored/last30days.py").
func EngineFS() (fs.FS, error) {
return fs.Sub(vendored, "vendored")
}
mcp/internal/engine/extract.go
+168
View File
@@ -0,0 +1,168 @@
package engine
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sync"
)
// SentinelFilename names the file Ensure writes inside the cache directory
// after a successful extraction. Its contents are compared to the requested
// version; a match short-circuits re-extraction on subsequent calls.
const SentinelFilename = ".version"
// cacheSubdir namespaces our cache under the OS user cache directory so
// multiple printing-press-style bundles can coexist.
const cacheSubdir = "last30days-pp-mcp"
// CacheEnvOverride lets users redirect the cache directory when the default
// OS cache location is read-only (locked-down corp images, ephemeral CI
// containers). Pointed at by extract errors via the documented escape hatch.
const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
// Ensure extracts src into baseDir/last30days-pp-mcp/<version> and returns
// the cache path. If the sentinel file already records the same version the
// directory is reused without rewriting. version must be non-empty so the
// cache layout always namespaces by version.
//
// Extraction writes to a sibling .tmp directory and renames it on success
// so a partial extraction can never be mistaken for a complete one. Concurrent
// callers within the same process serialize behind a per-cache-dir sync.Once
// so the rename happens exactly once.
func Ensure(src fs.FS, baseDir, version string) (string, error) {
if version == "" {
return "", errors.New("engine: version is required")
}
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
once := getOnce(cacheDir)
var extractErr error
once.Do(func() {
extractErr = ensureLocked(src, cacheDir, version)
})
if extractErr != nil {
// Reset the sync.Once so a follow-up call can retry rather than
// permanently caching the error. Retry is the right default when
// the failure is transient (e.g., disk full, parent dir restored).
resetOnce(cacheDir)
return "", extractErr
}
return cacheDir, nil
}
// EnsureUserCache wraps Ensure with the OS user cache dir (or the
// LAST30DAYS_CACHE_DIR override) as base. Production callers use this; tests
// use Ensure with an explicit temp dir.
func EnsureUserCache(src fs.FS, version string) (string, error) {
if override := os.Getenv(CacheEnvOverride); override != "" {
return Ensure(src, override, version)
}
base, err := os.UserCacheDir()
if err != nil {
return "", fmt.Errorf("engine: resolve user cache dir (set %s to override): %w", CacheEnvOverride, err)
}
return Ensure(src, base, version)
}
func ensureLocked(src fs.FS, cacheDir, version string) error {
if sentinelMatches(cacheDir, version) {
return nil
}
tmpDir := cacheDir + ".tmp"
if err := os.RemoveAll(tmpDir); err != nil {
return fmt.Errorf("engine: clean tmp cache: %w", err)
}
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
}
if err := extractAll(src, tmpDir); err != nil {
_ = os.RemoveAll(tmpDir)
return err
}
sentinel := filepath.Join(tmpDir, SentinelFilename)
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
_ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: write sentinel: %w", err)
}
if err := os.RemoveAll(cacheDir); err != nil {
_ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: clean old cache: %w", err)
}
if err := os.Rename(tmpDir, cacheDir); err != nil {
_ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: promote tmp cache: %w", err)
}
return nil
}
func sentinelMatches(cacheDir, version string) bool {
data, err := os.ReadFile(filepath.Join(cacheDir, SentinelFilename))
if err != nil {
return false
}
return string(data) == version
}
func extractAll(src fs.FS, dst string) error {
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if path == "." {
return nil
}
target := filepath.Join(dst, path)
if d.IsDir() {
return os.MkdirAll(target, 0o755)
}
return copyEmbeddedFile(src, path, target)
})
}
func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
in, err := src.Open(srcPath)
if err != nil {
return fmt.Errorf("engine: open %s: %w", srcPath, err)
}
defer func() { _ = in.Close() }()
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return fmt.Errorf("engine: ensure parent of %s: %w", dst, err)
}
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if err != nil {
return fmt.Errorf("engine: create %s: %w", dst, err)
}
defer func() { _ = out.Close() }()
if _, err := io.Copy(out, in); err != nil {
return fmt.Errorf("engine: write %s: %w", dst, err)
}
return nil
}
// onceRegistry serializes first-call extraction per cache directory so the
// rename in ensureLocked happens exactly once across goroutines.
var (
onceMu sync.Mutex
onceRegistry = map[string]*sync.Once{}
)
func getOnce(cacheDir string) *sync.Once {
onceMu.Lock()
defer onceMu.Unlock()
if o, ok := onceRegistry[cacheDir]; ok {
return o
}
o := &sync.Once{}
onceRegistry[cacheDir] = o
return o
}
func resetOnce(cacheDir string) {
onceMu.Lock()
defer onceMu.Unlock()
delete(onceRegistry, cacheDir)
}
mcp/internal/engine/extract_test.go
+167
View File
@@ -0,0 +1,167 @@
package engine
import (
"os"
"path/filepath"
"sync"
"testing"
"testing/fstest"
)
func newTestFS() fstest.MapFS {
return fstest.MapFS{
"last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
}
}
func TestEnsureExtractsEngine(t *testing.T) {
src := newTestFS()
base := t.TempDir()
cacheDir, err := Ensure(src, base, "v1")
if err != nil {
t.Fatalf("Ensure: %v", err)
}
if cacheDir != filepath.Join(base, cacheSubdir, "v1") {
t.Fatalf("cacheDir = %q, want %q", cacheDir, filepath.Join(base, cacheSubdir, "v1"))
}
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
mustReadFile(t, filepath.Join(cacheDir, "lib/env.py"), "# env helpers\n")
mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
}
func TestEnsureSkipsWhenSentinelMatches(t *testing.T) {
src := newTestFS()
base := t.TempDir()
cacheDir, err := Ensure(src, base, "v1")
if err != nil {
t.Fatalf("first Ensure: %v", err)
}
target := filepath.Join(cacheDir, "last30days.py")
info1, err := os.Stat(target)
if err != nil {
t.Fatalf("stat: %v", err)
}
// Reset the sync.Once so a second call would re-extract if not for the
// sentinel short-circuit. Without the reset, sync.Once would skip the
// extraction regardless of sentinel state.
resetOnce(cacheDir)
if _, err := Ensure(src, base, "v1"); err != nil {
t.Fatalf("second Ensure: %v", err)
}
info2, err := os.Stat(target)
if err != nil {
t.Fatalf("stat second: %v", err)
}
if !info2.ModTime().Equal(info1.ModTime()) {
t.Fatalf("expected file untouched on sentinel match; got mtime %v -> %v", info1.ModTime(), info2.ModTime())
}
}
func TestEnsureReExtractsOnVersionChange(t *testing.T) {
v1 := fstest.MapFS{
"last30days.py": &fstest.MapFile{Data: []byte("v1\n"), Mode: 0o644},
}
v2 := fstest.MapFS{
"last30days.py": &fstest.MapFile{Data: []byte("v2\n"), Mode: 0o644},
}
base := t.TempDir()
cache1, err := Ensure(v1, base, "v1")
if err != nil {
t.Fatalf("Ensure v1: %v", err)
}
cache2, err := Ensure(v2, base, "v2")
if err != nil {
t.Fatalf("Ensure v2: %v", err)
}
if cache1 == cache2 {
t.Fatalf("expected distinct cache dirs per version, got %q == %q", cache1, cache2)
}
mustReadFile(t, filepath.Join(cache1, "last30days.py"), "v1\n")
mustReadFile(t, filepath.Join(cache2, "last30days.py"), "v2\n")
}
func TestEnsureConcurrentFirstCall(t *testing.T) {
src := newTestFS()
base := t.TempDir()
const goroutines = 10
var wg sync.WaitGroup
wg.Add(goroutines)
results := make([]string, goroutines)
errs := make([]error, goroutines)
for i := 0; i < goroutines; i++ {
i := i
go func() {
defer wg.Done()
results[i], errs[i] = Ensure(src, base, "v1")
}()
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Fatalf("goroutine %d: %v", i, err)
}
}
for i := 1; i < goroutines; i++ {
if results[i] != results[0] {
t.Fatalf("goroutine 0 saw %q, goroutine %d saw %q", results[0], i, results[i])
}
}
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
}
func TestEnsureRejectsEmptyVersion(t *testing.T) {
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
t.Fatal("expected error for empty version")
}
}
func TestEnsureReturnsErrorWhenCacheUnwritable(t *testing.T) {
// Place the cache root at a path that cannot exist (a regular file).
// MkdirAll will refuse and Ensure must surface a wrapped error.
base := t.TempDir()
blocker := filepath.Join(base, "blocker")
if err := os.WriteFile(blocker, []byte("not a dir"), 0o644); err != nil {
t.Fatalf("setup: %v", err)
}
_, err := Ensure(newTestFS(), blocker, "v1")
if err == nil {
t.Fatal("expected error when cache parent is not a directory")
}
}
func TestEnsureUserCacheHonorsOverride(t *testing.T) {
override := t.TempDir()
t.Setenv(CacheEnvOverride, override)
src := newTestFS()
cacheDir, err := EnsureUserCache(src, "v1")
if err != nil {
t.Fatalf("EnsureUserCache: %v", err)
}
want := filepath.Join(override, cacheSubdir, "v1")
if cacheDir != want {
t.Fatalf("cacheDir = %q, want %q", cacheDir, want)
}
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
}
func mustReadFile(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
if string(data) != want {
t.Fatalf("%s: got %q, want %q", path, string(data), want)
}
}
mcp/internal/engine/run.go
+163
View File
@@ -0,0 +1,163 @@
package engine
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
)
// DefaultPythonBinary is the interpreter we look up unless RunOptions
// overrides it. Windows installs may expose only "python"; we surface a
// clear error in that case rather than silently picking the wrong binary.
const DefaultPythonBinary = "python3"
// MinPythonVersion mirrors the engine's MIN_PYTHON constant in
// last30days.py. Surfaced in errors so users know what they're missing.
const MinPythonVersion = "3.12"
// PythonInstallURL is included in the missing-interpreter error so users
// have a direct route from the failure to a fix.
const PythonInstallURL = "https://www.python.org/downloads/"
// DefaultTimeout caps a single research subprocess. The engine's deep mode
// can run several minutes; five minutes is a safe upper bound that still
// fails fast when something hangs.
const DefaultTimeout = 5 * time.Minute
// TimeoutEnvOverride lets operators override DefaultTimeout per install
// (seconds, integer). Honored by Run when RunOptions.Timeout is zero.
const TimeoutEnvOverride = "LAST30DAYS_MCP_TIMEOUT"
// RunOptions configures one invocation of the embedded Python engine.
// PythonPath is exposed so tests can substitute a stub interpreter without
// manipulating the process PATH.
type RunOptions struct {
PythonPath string // resolved python3 binary; empty means look up DefaultPythonBinary on PATH
CacheDir string // engine.Ensure result; lib/ here is added to PYTHONPATH
Args []string // arguments after last30days.py (topic, --emit=..., etc.)
ExtraEnv []string // appended to os.Environ() for the child process
Timeout time.Duration // zero means DefaultTimeout or TimeoutEnvOverride
}
// RunResult captures the engine's full output. Stdout is what we surface to
// the agent; Stderr is included in error messages so users can diagnose
// engine failures without leaving Claude Desktop.
type RunResult struct {
Stdout []byte
Stderr []byte
ExitCode int
TimedOut bool
}
// Run shells out to python3 with last30days.py inside cacheDir. The child
// receives the parent environment (so MCPB user_config env-injection
// reaches the engine) plus ExtraEnv and a PYTHONPATH that points at the
// cache so the engine's `from lib import ...` statements resolve.
//
// A missing interpreter, a non-zero exit, and a timeout each surface as
// distinct errors so the tool handler can map them to user-facing
// messages without re-parsing stderr.
func Run(ctx context.Context, opts RunOptions) (*RunResult, error) {
if opts.CacheDir == "" {
return nil, errors.New("engine: CacheDir is required")
}
pythonPath, err := resolvePython(opts.PythonPath)
if err != nil {
return nil, err
}
scriptPath := filepath.Join(opts.CacheDir, "last30days.py")
if _, err := os.Stat(scriptPath); err != nil {
return nil, fmt.Errorf("engine: last30days.py not found in cache %s: %w", opts.CacheDir, err)
}
timeout := resolveTimeout(opts.Timeout)
subCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
args := append([]string{scriptPath}, opts.Args...)
cmd := exec.CommandContext(subCtx, pythonPath, args...)
cmd.Env = buildEnv(opts.CacheDir, opts.ExtraEnv)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err = cmd.Run()
res := &RunResult{
Stdout: stdout.Bytes(),
Stderr: stderr.Bytes(),
ExitCode: 0,
TimedOut: errors.Is(subCtx.Err(), context.DeadlineExceeded),
}
if err == nil {
return res, nil
}
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
res.ExitCode = exitErr.ExitCode()
if res.TimedOut {
return res, fmt.Errorf("engine: subprocess exceeded %s timeout", timeout)
}
return res, fmt.Errorf("engine: subprocess exited with code %d", res.ExitCode)
}
return res, fmt.Errorf("engine: subprocess failed to start: %w", err)
}
// resolvePython returns an absolute path to the interpreter or an error
// naming the install URL. If the caller supplied a path we trust it - tests
// rely on this to inject a stub. Otherwise we look up python3 on PATH.
func resolvePython(override string) (string, error) {
if override != "" {
return override, nil
}
path, err := exec.LookPath(DefaultPythonBinary)
if err == nil {
return path, nil
}
return "", fmt.Errorf(
"engine: %s not found on PATH (need Python %s+, install from %s; current GOOS=%s)",
DefaultPythonBinary, MinPythonVersion, PythonInstallURL, runtime.GOOS,
)
}
func resolveTimeout(explicit time.Duration) time.Duration {
if explicit > 0 {
return explicit
}
if raw := os.Getenv(TimeoutEnvOverride); raw != "" {
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
return d
}
}
return DefaultTimeout
}
// buildEnv stitches PYTHONPATH onto os.Environ + ExtraEnv. Any pre-existing
// PYTHONPATH in the parent environment is dropped before appending the
// cache dir; otherwise the child sees two PYTHONPATH= entries and POSIX
// getenv returns the first one, so the user's value wins and the engine's
// `from lib import ...` fails with ModuleNotFoundError. The engine is
// self-contained and does not need the user's Python module search path.
func buildEnv(cacheDir string, extra []string) []string {
const pyKey = "PYTHONPATH="
parent := os.Environ()
base := make([]string, 0, len(parent)+1+len(extra))
for _, kv := range parent {
if strings.HasPrefix(kv, pyKey) {
continue
}
base = append(base, kv)
}
base = append(base, pyKey+cacheDir)
base = append(base, extra...)
return base
}
mcp/internal/engine/run_test.go
+281
View File
@@ -0,0 +1,281 @@
package engine
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// makeStubPython writes a shell script that simulates python3 and returns
// its absolute path. The script honors a small env-driven protocol so each
// test can shape its output:
//
// STUB_STDOUT - text printed to stdout
// STUB_STDERR - text printed to stderr
// STUB_EXIT_CODE - integer exit code (default 0)
// STUB_SLEEP_SECS - sleep before exiting (for timeout tests)
// STUB_ECHO_ENV - name of an env var; the stub prints "<NAME>=<VALUE>"
// STUB_ECHO_ARG - integer index; the stub prints "ARG<i>=<args[i]>"
//
// The stub ignores its first argument (the script path), matching how a
// real python3 invocation treats `python3 last30days.py ...`.
func makeStubPython(t *testing.T) string {
t.Helper()
if runtime.GOOS == "windows" {
t.Skip("stub-python tests rely on POSIX shell")
}
dir := t.TempDir()
path := filepath.Join(dir, "python3-stub.sh")
script := `#!/usr/bin/env bash
if [ -n "${STUB_SLEEP_SECS:-}" ]; then sleep "$STUB_SLEEP_SECS"; fi
if [ -n "${STUB_STDOUT:-}" ]; then printf "%s" "$STUB_STDOUT"; fi
if [ -n "${STUB_STDERR:-}" ]; then printf "%s" "$STUB_STDERR" >&2; fi
if [ -n "${STUB_ECHO_ENV:-}" ]; then echo "${STUB_ECHO_ENV}=${!STUB_ECHO_ENV:-<unset>}"; fi
if [ -n "${STUB_ECHO_ARG:-}" ]; then echo "ARG${STUB_ECHO_ARG}=${!STUB_ECHO_ARG:-<unset>}"; fi
exit "${STUB_EXIT_CODE:-0}"
`
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
t.Fatalf("write stub: %v", err)
}
return path
}
// stageCache materializes a fake CacheDir with a no-op last30days.py so
// the existence check in Run passes. The stub python3 ignores the script
// contents, so the file just has to exist.
func stageCache(t *testing.T) string {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "last30days.py"), []byte("# stub\n"), 0o644); err != nil {
t.Fatalf("stage cache: %v", err)
}
return dir
}
func TestRunHappyPath(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("STUB_STDOUT", "synthesis output\n")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
Args: []string{"my topic", "--emit=compact"},
})
if err != nil {
t.Fatalf("Run: %v", err)
}
if string(res.Stdout) != "synthesis output\n" {
t.Fatalf("stdout = %q, want %q", res.Stdout, "synthesis output\n")
}
if res.ExitCode != 0 {
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
}
if res.TimedOut {
t.Fatal("TimedOut = true, want false")
}
}
func TestRunForwardsEnv(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("OPENAI_API_KEY", "sk-test-value")
t.Setenv("STUB_ECHO_ENV", "OPENAI_API_KEY")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
if got := strings.TrimSpace(string(res.Stdout)); got != "OPENAI_API_KEY=sk-test-value" {
t.Fatalf("stdout = %q, want OPENAI_API_KEY=sk-test-value", got)
}
}
func TestRunSetsPythonPath(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
want := "PYTHONPATH=" + cache
if got := strings.TrimSpace(string(res.Stdout)); got != want {
t.Fatalf("stdout = %q, want %q", got, want)
}
}
// TestRunDropsPreExistingPythonPath guards the buildEnv dedup: when the
// parent already sets PYTHONPATH (common on dev machines and CI runners
// that touch Python), the child must NOT see two PYTHONPATH= entries.
// POSIX getenv returns the first match, so a duplicate from os.Environ
// would shadow our cache-dir entry and break `from lib import ...`.
func TestRunDropsPreExistingPythonPath(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("PYTHONPATH", "/users-stale-pythonpath")
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
got := strings.TrimSpace(string(res.Stdout))
want := "PYTHONPATH=" + cache
if got != want {
t.Fatalf("stdout = %q, want %q (stale parent value leaked through)", got, want)
}
}
func TestBuildEnvDropsAllPreExistingPythonPath(t *testing.T) {
// Direct unit test on buildEnv to catch the case where the parent has
// PYTHONPATH set: the returned slice must contain exactly one
// PYTHONPATH= entry, and it must be ours.
t.Setenv("PYTHONPATH", "/parent/one")
cache := "/cache/dir"
out := buildEnv(cache, []string{"EXTRA=1"})
var pythonPaths []string
for _, kv := range out {
if strings.HasPrefix(kv, "PYTHONPATH=") {
pythonPaths = append(pythonPaths, kv)
}
}
if len(pythonPaths) != 1 {
t.Fatalf("got %d PYTHONPATH entries, want 1: %v", len(pythonPaths), pythonPaths)
}
if pythonPaths[0] != "PYTHONPATH="+cache {
t.Fatalf("PYTHONPATH = %q, want %q", pythonPaths[0], "PYTHONPATH="+cache)
}
// Confirm ExtraEnv still rides along.
found := false
for _, kv := range out {
if kv == "EXTRA=1" {
found = true
break
}
}
if !found {
t.Fatal("EXTRA=1 missing from buildEnv output")
}
}
func TestRunSurfacesExitCode(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("STUB_STDERR", "engine boom\n")
t.Setenv("STUB_EXIT_CODE", "2")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
})
if err == nil {
t.Fatal("expected error for non-zero exit")
}
if res == nil {
t.Fatal("res is nil; want populated result alongside error")
}
if res.ExitCode != 2 {
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
}
if !strings.Contains(string(res.Stderr), "engine boom") {
t.Fatalf("stderr did not surface engine output: %q", res.Stderr)
}
}
func TestRunTimesOut(t *testing.T) {
stub := makeStubPython(t)
cache := stageCache(t)
t.Setenv("STUB_SLEEP_SECS", "3")
res, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
Timeout: 200 * time.Millisecond,
})
if err == nil {
t.Fatal("expected timeout error")
}
if !res.TimedOut {
t.Fatal("TimedOut = false, want true")
}
if !strings.Contains(err.Error(), "timeout") {
t.Fatalf("error %q lacks 'timeout' marker", err)
}
}
func TestRunMissingPython(t *testing.T) {
cache := stageCache(t)
// Empty PATH guarantees the lookup fails. PythonPath stays unset so Run
// falls through to exec.LookPath.
t.Setenv("PATH", "")
_, err := Run(context.Background(), RunOptions{CacheDir: cache})
if err == nil {
t.Fatal("expected lookup failure with empty PATH")
}
if !strings.Contains(err.Error(), DefaultPythonBinary) {
t.Fatalf("error %q does not mention %s", err, DefaultPythonBinary)
}
if !strings.Contains(err.Error(), PythonInstallURL) {
t.Fatalf("error %q does not include install URL", err)
}
}
func TestRunMissingScript(t *testing.T) {
stub := makeStubPython(t)
// CacheDir exists but contains no last30days.py.
cache := t.TempDir()
_, err := Run(context.Background(), RunOptions{
PythonPath: stub,
CacheDir: cache,
})
if err == nil {
t.Fatal("expected error when last30days.py missing")
}
if !strings.Contains(err.Error(), "last30days.py") {
t.Fatalf("error %q does not name missing script", err)
}
}
func TestRunRejectsEmptyCacheDir(t *testing.T) {
stub := makeStubPython(t)
_, err := Run(context.Background(), RunOptions{PythonPath: stub})
if err == nil {
t.Fatal("expected error for empty CacheDir")
}
if !errors.Is(err, err) || !strings.Contains(err.Error(), "CacheDir") {
t.Fatalf("error %q does not name CacheDir", err)
}
}
func TestResolveTimeoutHonorsEnv(t *testing.T) {
t.Setenv(TimeoutEnvOverride, "750ms")
if got := resolveTimeout(0); got != 750*time.Millisecond {
t.Fatalf("resolveTimeout = %v, want 750ms", got)
}
t.Setenv(TimeoutEnvOverride, "garbage")
if got := resolveTimeout(0); got != DefaultTimeout {
t.Fatalf("garbage value: got %v, want default %v", got, DefaultTimeout)
}
if got := resolveTimeout(time.Minute); got != time.Minute {
t.Fatalf("explicit value not honored: got %v", got)
}
}
mcp/internal/engine/vendored/.gitkeep
+2
View File
@@ -0,0 +1,2 @@
Populated at build time by scripts/sync-engine.sh.
Source of truth: skills/last30days/scripts/.
mcp/internal/manifest/manifest_test.go
+189
View File
@@ -0,0 +1,189 @@
// Package manifest holds tests for mcp/manifest.json. It contains no
// production code - the manifest itself is the artifact, and these tests
// guard structural invariants the bundling pipeline depends on.
package manifest
import (
"encoding/json"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
// envBinding is a minimal subset of the MCPB v0.3 manifest just covering
// the fields these tests assert on. We deliberately do not depend on the
// printing-press internal/pipeline types (that's an internal/ package and
// not importable across modules) - the structural invariants below are
// what actually matter for Claude Desktop install correctness.
type manifestShape struct {
ManifestVersion string `json:"manifest_version"`
Name string `json:"name"`
Version string `json:"version"`
Server struct {
Type string `json:"type"`
EntryPoint string `json:"entry_point"`
MCPConfig struct {
Command string `json:"command"`
Env map[string]string `json:"env"`
} `json:"mcp_config"`
} `json:"server"`
UserConfig map[string]struct {
Type string `json:"type"`
Title string `json:"title"`
Description string `json:"description"`
Sensitive bool `json:"sensitive"`
Required bool `json:"required"`
} `json:"user_config"`
Compatibility struct {
ClaudeDesktop string `json:"claude_desktop"`
Platforms []string `json:"platforms"`
} `json:"compatibility"`
}
// loadManifest reads mcp/manifest.json relative to this test file so the
// test passes regardless of where `go test` is invoked from.
func loadManifest(t *testing.T) manifestShape {
t.Helper()
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
// manifest_test.go is at mcp/internal/manifest/; manifest.json at mcp/.
manifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "manifest.json")
data, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var m manifestShape
if err := json.Unmarshal(data, &m); err != nil {
t.Fatalf("parse manifest: %v", err)
}
return m
}
func TestManifestRequiredFields(t *testing.T) {
m := loadManifest(t)
if m.ManifestVersion != "0.3" {
t.Errorf("manifest_version = %q, want 0.3", m.ManifestVersion)
}
if m.Name != "last30days-pp-mcp" {
t.Errorf("name = %q, want last30days-pp-mcp", m.Name)
}
if m.Version == "" {
t.Error("version is empty")
}
if m.Server.Type != "binary" {
t.Errorf("server.type = %q, want binary", m.Server.Type)
}
if m.Server.EntryPoint != "bin/last30days-pp-mcp" {
t.Errorf("server.entry_point = %q, want bin/last30days-pp-mcp", m.Server.EntryPoint)
}
if m.Compatibility.ClaudeDesktop == "" {
t.Error("compatibility.claude_desktop is empty")
}
}
// TestEnvAndUserConfigCrossReference is the key invariant: every
// ${user_config.<key>} substitution in server.mcp_config.env must point
// at a real user_config entry, and every declared user_config must be
// wired to an env var. A typo on either side silently disables a credential
// at install time without the binary or Claude Desktop noticing.
func TestEnvAndUserConfigCrossReference(t *testing.T) {
m := loadManifest(t)
if len(m.Server.MCPConfig.Env) == 0 {
t.Fatal("server.mcp_config.env is empty; expected user_config substitutions")
}
if len(m.UserConfig) == 0 {
t.Fatal("user_config is empty; expected per-key declarations")
}
for envName, value := range m.Server.MCPConfig.Env {
key, ok := parseUserConfigRef(value)
if !ok {
t.Errorf("env[%s] = %q is not a ${user_config.<key>} reference", envName, value)
continue
}
if _, declared := m.UserConfig[key]; !declared {
t.Errorf("env[%s] references user_config[%q], which is not declared", envName, key)
}
// The user_config key must be the lowercased env var so Claude
// Desktop's substitution rule matches PP's emitted shape.
if got := strings.ToLower(envName); key != got {
t.Errorf("env[%s] -> user_config[%q]; convention requires user_config[%q]", envName, key, got)
}
}
envValues := make(map[string]bool, len(m.Server.MCPConfig.Env))
for _, value := range m.Server.MCPConfig.Env {
if key, ok := parseUserConfigRef(value); ok {
envValues[key] = true
}
}
for key := range m.UserConfig {
if !envValues[key] {
t.Errorf("user_config[%q] is declared but never substituted into env", key)
}
}
}
func TestUserConfigShape(t *testing.T) {
m := loadManifest(t)
for key, slot := range m.UserConfig {
if slot.Type != "string" {
t.Errorf("user_config[%q].type = %q, want string", key, slot.Type)
}
if slot.Title == "" {
t.Errorf("user_config[%q].title is empty", key)
}
if slot.Description == "" {
t.Errorf("user_config[%q].description is empty", key)
}
if !slot.Sensitive {
// API keys must be flagged sensitive so Claude Desktop masks
// the input and prefers OS-keychain storage.
t.Errorf("user_config[%q].sensitive = false; want true for API credentials", key)
}
if slot.Required {
// The engine degrades to web-only mode without keys, so no
// key is install-blocking.
t.Errorf("user_config[%q].required = true; engine degrades without keys, so all keys are optional", key)
}
}
}
func TestPlatformsMatchShippingMatrix(t *testing.T) {
// compatibility.platforms must list exactly what the release CI
// actually packages. Listing a platform we don't ship would let
// Claude Desktop start an install that has no matching binary inside
// the bundle, producing a silent failure. The CI matrix in
// .github/workflows/release.yml currently covers darwin (arm64 +
// amd64) and linux/amd64; Windows is deferred.
m := loadManifest(t)
required := map[string]bool{"darwin": false, "linux": false}
forbidden := map[string]bool{"win32": true}
for _, p := range m.Compatibility.Platforms {
if _, ok := required[p]; ok {
required[p] = true
}
if forbidden[p] {
t.Errorf("compatibility.platforms contains %q but the release matrix does not ship that platform; add it to the matrix or remove from the manifest", p)
}
}
for p, found := range required {
if !found {
t.Errorf("compatibility.platforms missing %q", p)
}
}
}
func parseUserConfigRef(value string) (string, bool) {
const prefix = "${user_config."
const suffix = "}"
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
return "", false
}
return value[len(prefix) : len(value)-len(suffix)], true
}
mcp/internal/tools/preflight.go
+85
View File
@@ -0,0 +1,85 @@
package tools
import (
"context"
"errors"
"fmt"
mcplib "github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
)
func registerPreflightTool(s *server.MCPServer, cfg Config) {
s.AddTool(
mcplib.NewTool("preflight",
mcplib.WithDescription(
"Safely summarize what last30days would read, write, execute, and contact "+
"without running research, saving files, or reading browser cookies.",
),
mcplib.WithString("format", mcplib.Description("Output shape: 'text' (default) for a concise summary or 'json' for structured details.")),
mcplib.WithReadOnlyHintAnnotation(true),
mcplib.WithDestructiveHintAnnotation(false),
mcplib.WithOpenWorldHintAnnotation(false),
),
makePreflightHandler(cfg),
)
}
func makePreflightHandler(cfg Config) server.ToolHandlerFunc {
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
format, err := preflightFormatArgument(req.GetArguments())
if err != nil {
return mcplib.NewToolResultError(err.Error()), nil
}
src, err := engine.EngineFS()
if err != nil {
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
}
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
if err != nil {
return mcplib.NewToolResultError(fmt.Sprintf(
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
err, engine.CacheEnvOverride,
)), nil
}
res, runErr := engine.Run(ctx, engine.RunOptions{
CacheDir: cacheDir,
Args: preflightRunArgs(format),
})
if runErr != nil {
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
}
return mcplib.NewToolResultText(string(res.Stdout)), nil
}
}
func preflightRunArgs(format string) []string {
runArgs := []string{"--preflight", "--preflight-report-on-save-dir", mcpSaveDir()}
if format == "json" {
runArgs = append(runArgs, "--emit=json")
}
return runArgs
}
func preflightFormatArgument(args map[string]any) (string, error) {
raw, ok := args["format"]
if !ok {
return "text", nil
}
value, ok := raw.(string)
if !ok {
return "", errors.New("format must be a string")
}
switch value {
case "", "text":
return "text", nil
case "json":
return "json", nil
default:
return "", fmt.Errorf("format must be 'text' or 'json', got %q", value)
}
}
mcp/internal/tools/preflight_test.go
+60
View File
@@ -0,0 +1,60 @@
package tools
import (
"strings"
"testing"
)
func TestPreflightRunArgsDefaultTextIsSafe(t *testing.T) {
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
args := preflightRunArgs("text")
want := []string{
"--preflight",
"--preflight-report-on-save-dir",
"~/Documents/Last30Days",
}
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("args = %#v, want %#v", args, want)
}
}
func TestPreflightRunArgsJSONIsSafeAndStructured(t *testing.T) {
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
args := preflightRunArgs("json")
want := []string{
"--preflight",
"--preflight-report-on-save-dir",
"/tmp/last30days-reports",
"--emit=json",
}
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("args = %#v, want %#v", args, want)
}
}
func TestPreflightFormatArgumentDefaultsAndValidates(t *testing.T) {
cases := []struct {
name string
args map[string]any
want string
wantErr bool
}{
{"missing defaults to text", map[string]any{}, "text", false},
{"empty defaults to text", map[string]any{"format": ""}, "text", false},
{"text passes", map[string]any{"format": "text"}, "text", false},
{"json passes", map[string]any{"format": "json"}, "json", false},
{"invalid rejected", map[string]any{"format": "xml"}, "", true},
{"non-string rejected", map[string]any{"format": true}, "", true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := preflightFormatArgument(tc.args)
if (err != nil) != tc.wantErr {
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
}
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
})
}
}
mcp/internal/tools/research.go
+158
View File
@@ -0,0 +1,158 @@
// Package tools owns the MCP tool surface for last30days.
package tools
import (
"context"
"errors"
"fmt"
"os"
"strings"
mcplib "github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
)
// Config carries the version string used to namespace the per-user cache.
// main passes its ldflags-stamped Version here.
type Config struct {
Version string
}
// Register adds every tool this server exposes to s. The caller supplies a
// Config so test harnesses can pin a version without touching globals.
func Register(s *server.MCPServer, cfg Config) {
registerPreflightTool(s, cfg)
s.AddTool(
mcplib.NewTool("research",
mcplib.WithDescription(
"Research what people are actually saying about any topic in the last 30 days. "+
"Aggregates Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web, "+
"scored by upvotes, likes, transcripts, and real-money prediction-market odds. "+
"Returns the engine's compact output for the model to synthesize.",
),
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
mcplib.WithReadOnlyHintAnnotation(false),
mcplib.WithDestructiveHintAnnotation(false),
mcplib.WithOpenWorldHintAnnotation(true),
),
makeResearchHandler(cfg),
)
}
func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
args := req.GetArguments()
topic, err := requireString(args, "topic")
if err != nil {
return mcplib.NewToolResultError(err.Error()), nil
}
emit, err := emitArgument(args)
if err != nil {
return mcplib.NewToolResultError(err.Error()), nil
}
save, err := boolArgument(args, "save")
if err != nil {
return mcplib.NewToolResultError(err.Error()), nil
}
src, err := engine.EngineFS()
if err != nil {
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
}
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
if err != nil {
return mcplib.NewToolResultError(fmt.Sprintf(
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
err, engine.CacheEnvOverride,
)), nil
}
runArgs := researchRunArgs(topic, emit, save)
res, runErr := engine.Run(ctx, engine.RunOptions{
CacheDir: cacheDir,
Args: runArgs,
})
if runErr != nil {
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
}
return mcplib.NewToolResultText(string(res.Stdout)), nil
}
}
func researchRunArgs(topic, emit string, save bool) []string {
runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"}
if save {
runArgs = append(runArgs, "--save-dir", mcpSaveDir())
}
return runArgs
}
func mcpSaveDir() string {
saveDir := os.Getenv("LAST30DAYS_MEMORY_DIR")
if saveDir == "" {
return "~/Documents/Last30Days"
}
return saveDir
}
func requireString(args map[string]any, name string) (string, error) {
raw, ok := args[name]
if !ok {
return "", fmt.Errorf("%s is required", name)
}
value, ok := raw.(string)
if !ok || strings.TrimSpace(value) == "" {
return "", fmt.Errorf("%s must be a non-empty string", name)
}
return value, nil
}
func emitArgument(args map[string]any) (string, error) {
raw, ok := args["emit"]
if !ok {
return "compact", nil
}
value, ok := raw.(string)
if !ok {
return "", errors.New("emit must be a string")
}
switch value {
case "":
return "compact", nil
case "compact", "html":
return value, nil
default:
return "", fmt.Errorf("emit must be 'compact' or 'html', got %q", value)
}
}
func boolArgument(args map[string]any, name string) (bool, error) {
raw, ok := args[name]
if !ok {
return false, nil
}
value, ok := raw.(bool)
if !ok {
return false, fmt.Errorf("%s must be a boolean", name)
}
return value, nil
}
// formatRunError flattens engine.Run's distinct error shapes into a single
// user-facing message that includes the relevant stderr context.
func formatRunError(runErr error, res *engine.RunResult) string {
var msg strings.Builder
msg.WriteString(runErr.Error())
if res != nil && len(res.Stderr) > 0 {
msg.WriteString("\nengine stderr:\n")
msg.Write(res.Stderr)
}
return msg.String()
}
mcp/internal/tools/research_test.go
+175
View File
@@ -0,0 +1,175 @@
package tools
import (
"context"
"errors"
"strings"
"testing"
mcplib "github.com/mark3labs/mcp-go/mcp"
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
)
func newCallToolRequest(args map[string]any) mcplib.CallToolRequest {
var req mcplib.CallToolRequest
req.Params.Arguments = args
return req
}
// resultText pulls text content out of a tool result so tests can assert on
// the body Claude will see. Returns empty string when the result is nil or
// has no text content.
func resultText(res *mcplib.CallToolResult) string {
if res == nil {
return ""
}
var out strings.Builder
for _, item := range res.Content {
if tc, ok := item.(mcplib.TextContent); ok {
out.WriteString(tc.Text)
}
}
return out.String()
}
func TestRequireStringRejectsMissingAndBlank(t *testing.T) {
if _, err := requireString(map[string]any{}, "topic"); err == nil {
t.Fatal("expected error for missing topic")
}
if _, err := requireString(map[string]any{"topic": ""}, "topic"); err == nil {
t.Fatal("expected error for empty topic")
}
if _, err := requireString(map[string]any{"topic": " "}, "topic"); err == nil {
t.Fatal("expected error for whitespace-only topic")
}
if _, err := requireString(map[string]any{"topic": 42}, "topic"); err == nil {
t.Fatal("expected error for non-string topic")
}
v, err := requireString(map[string]any{"topic": "OpenAI"}, "topic")
if err != nil || v != "OpenAI" {
t.Fatalf("requireString ok = %q, %v", v, err)
}
}
func TestEmitArgumentDefaultsAndValidates(t *testing.T) {
cases := []struct {
name string
args map[string]any
want string
wantErr bool
}{
{"missing defaults to compact", map[string]any{}, "compact", false},
{"empty string defaults to compact", map[string]any{"emit": ""}, "compact", false},
{"compact passes through", map[string]any{"emit": "compact"}, "compact", false},
{"html passes through", map[string]any{"emit": "html"}, "html", false},
{"invalid value rejected", map[string]any{"emit": "json"}, "", true},
{"non-string rejected", map[string]any{"emit": 7}, "", true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := emitArgument(tc.args)
if (err != nil) != tc.wantErr {
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
}
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
})
}
}
func TestBoolArgument(t *testing.T) {
v, err := boolArgument(map[string]any{}, "save")
if err != nil || v {
t.Fatalf("missing: %v, %v", v, err)
}
v, err = boolArgument(map[string]any{"save": true}, "save")
if err != nil || !v {
t.Fatalf("true: %v, %v", v, err)
}
v, err = boolArgument(map[string]any{"save": false}, "save")
if err != nil || v {
t.Fatalf("false: %v, %v", v, err)
}
if _, err := boolArgument(map[string]any{"save": "true"}, "save"); err == nil {
t.Fatal("expected error for string value")
}
}
func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) {
args := researchRunArgs("OpenAI", "compact", false)
want := []string{"OpenAI", "--emit=compact", "--no-browser-cookies"}
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("args = %#v, want %#v", args, want)
}
}
func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) {
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
args := researchRunArgs("OpenAI", "html", true)
got := strings.Join(args, "\x00")
if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") {
t.Fatalf("args still include unsupported --save: %#v", args)
}
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "~/Documents/Last30Days"}
if got != strings.Join(want, "\x00") {
t.Fatalf("args = %#v, want %#v", args, want)
}
}
func TestResearchRunArgsSaveUsesMemoryDirEnvOverride(t *testing.T) {
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
args := researchRunArgs("OpenAI", "html", true)
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "/tmp/last30days-reports"}
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("args = %#v, want %#v", args, want)
}
}
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
// Validation failures are returned as MCP tool errors (not Go errors)
// so Claude sees a structured failure with a readable message rather
// than a transport-level fault.
handler := makeResearchHandler(Config{Version: "test"})
cases := []struct {
name string
args map[string]any
wantSub string
}{
{"missing topic", map[string]any{}, "topic is required"},
{"blank topic", map[string]any{"topic": " "}, "non-empty string"},
{"invalid emit", map[string]any{"topic": "OpenAI", "emit": "json"}, "must be 'compact' or 'html'"},
{"non-bool save", map[string]any{"topic": "OpenAI", "save": "yes"}, "save must be a boolean"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
res, err := handler(context.Background(), newCallToolRequest(tc.args))
if err != nil {
t.Fatalf("handler should not return Go error for validation; got %v", err)
}
if res == nil || !res.IsError {
t.Fatalf("expected IsError result, got %+v", res)
}
if !strings.Contains(resultText(res), tc.wantSub) {
t.Fatalf("result text %q missing substring %q", resultText(res), tc.wantSub)
}
})
}
}
func TestFormatRunErrorIncludesStderr(t *testing.T) {
res := &engine.RunResult{Stderr: []byte("engine exploded\n")}
msg := formatRunError(errors.New("boom"), res)
if !strings.Contains(msg, "boom") || !strings.Contains(msg, "engine exploded") {
t.Fatalf("formatRunError missed pieces: %q", msg)
}
}
func TestFormatRunErrorHandlesNilResult(t *testing.T) {
msg := formatRunError(errors.New("boom"), nil)
if msg != "boom" {
t.Fatalf("nil result: got %q, want %q", msg, "boom")
}
}
mcp/manifest.json
+151
View File
@@ -0,0 +1,151 @@
{
"manifest_version": "0.3",
"name": "last30days-pp-mcp",
"display_name": "Last30Days",
"version": "3.6.0",
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
"author": {
"name": "Matt Van Horn",
"url": "https://github.com/mvanhorn/last30days-skill"
},
"repository": {
"type": "git",
"url": "https://github.com/mvanhorn/last30days-skill"
},
"license": "MIT",
"keywords": [
"research",
"reddit",
"twitter",
"x",
"youtube",
"hacker-news",
"polymarket",
"github",
"search",
"synthesis"
],
"server": {
"type": "binary",
"entry_point": "bin/last30days-pp-mcp",
"mcp_config": {
"command": "${__dirname}/bin/last30days-pp-mcp",
"args": [],
"env": {
"OPENAI_API_KEY": "${user_config.openai_api_key}",
"XAI_API_KEY": "${user_config.xai_api_key}",
"BRAVE_API_KEY": "${user_config.brave_api_key}",
"EXA_API_KEY": "${user_config.exa_api_key}",
"SERPER_API_KEY": "${user_config.serper_api_key}",
"GOOGLE_API_KEY": "${user_config.google_api_key}",
"GEMINI_API_KEY": "${user_config.gemini_api_key}",
"GOOGLE_GENAI_API_KEY": "${user_config.google_genai_api_key}",
"APIFY_API_TOKEN": "${user_config.apify_api_token}",
"BSKY_APP_PASSWORD": "${user_config.bsky_app_password}",
"PARALLEL_API_KEY": "${user_config.parallel_api_key}",
"SCRAPECREATORS_API_KEY": "${user_config.scrapecreators_api_key}",
"OPENROUTER_API_KEY": "${user_config.openrouter_api_key}"
}
}
},
"user_config": {
"openai_api_key": {
"type": "string",
"title": "OPENAI_API_KEY",
"description": "OpenAI API key. Powers Reddit research via OpenAI's web_search tool. Get one at https://platform.openai.com/api-keys.",
"sensitive": true,
"required": false
},
"xai_api_key": {
"type": "string",
"title": "XAI_API_KEY",
"description": "xAI API key. Powers X / Twitter research via xAI's x_search tool. Get one at https://console.x.ai/.",
"sensitive": true,
"required": false
},
"brave_api_key": {
"type": "string",
"title": "BRAVE_API_KEY",
"description": "Brave Search API key. Used for grounded web search results. Get one at https://brave.com/search/api/.",
"sensitive": true,
"required": false
},
"exa_api_key": {
"type": "string",
"title": "EXA_API_KEY",
"description": "Exa search API key. Alternative web search backend with semantic ranking. Get one at https://exa.ai/.",
"sensitive": true,
"required": false
},
"serper_api_key": {
"type": "string",
"title": "SERPER_API_KEY",
"description": "Serper API key. Google search via API. Get one at https://serper.dev/.",
"sensitive": true,
"required": false
},
"google_api_key": {
"type": "string",
"title": "GOOGLE_API_KEY",
"description": "Google API key for YouTube transcript fetching and other Google services. Get one at https://console.cloud.google.com/apis/credentials.",
"sensitive": true,
"required": false
},
"gemini_api_key": {
"type": "string",
"title": "GEMINI_API_KEY",
"description": "Gemini API key. Used for synthesis fallback when other LLM providers are unavailable. Get one at https://aistudio.google.com/apikey.",
"sensitive": true,
"required": false
},
"google_genai_api_key": {
"type": "string",
"title": "GOOGLE_GENAI_API_KEY",
"description": "Alternative Google generative-AI API key. Same source as GEMINI_API_KEY; set whichever name your tooling expects.",
"sensitive": true,
"required": false
},
"apify_api_token": {
"type": "string",
"title": "APIFY_API_TOKEN",
"description": "Apify API token. Powers TikTok and Instagram Reels search via Apify actors. Get one at https://console.apify.com/account/integrations.",
"sensitive": true,
"required": false
},
"bsky_app_password": {
"type": "string",
"title": "BSKY_APP_PASSWORD",
"description": "Bluesky app password (not your main password). Powers AT Protocol post search. Create at https://bsky.app/settings/app-passwords.",
"sensitive": true,
"required": false
},
"parallel_api_key": {
"type": "string",
"title": "PARALLEL_API_KEY",
"description": "Parallel AI key. Powers parallel research runs across sources. Get one at https://parallel.ai/.",
"sensitive": true,
"required": false
},
"scrapecreators_api_key": {
"type": "string",
"title": "SCRAPECREATORS_API_KEY",
"description": "ScrapeCreators API key. Powers creator-focused social search across TikTok, Instagram, and YouTube. Get one at https://scrapecreators.com/.",
"sensitive": true,
"required": false
},
"openrouter_api_key": {
"type": "string",
"title": "OPENROUTER_API_KEY",
"description": "OpenRouter API key. Alternative LLM provider gateway for synthesis. Get one at https://openrouter.ai/keys.",
"sensitive": true,
"required": false
}
},
"compatibility": {
"claude_desktop": ">=1.0.0",
"platforms": [
"darwin",
"linux"
]
}
}
mcp/scripts/sync-engine.sh
Executable
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Mirrors skills/last30days/scripts/{last30days.py,lib/} into mcp/vendored/
# so the Go binary's embed.FS captures the engine at build time.
#
# Source of truth: skills/last30days/scripts/. Never edit mcp/vendored/ directly.
# Run before `go build` locally and in CI before `printing-press bundle`.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MCP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
REPO_ROOT="$(cd "${MCP_DIR}/.." && pwd)"
ENGINE_SRC="${REPO_ROOT}/skills/last30days/scripts"
# Embed path must live inside the consuming package (Go //go:embed cannot
# reach outside its own directory tree), so vendored/ sits under engine/.
VENDORED="${MCP_DIR}/internal/engine/vendored"
if [ ! -f "${ENGINE_SRC}/last30days.py" ]; then
echo "sync-engine: ${ENGINE_SRC}/last30days.py not found" >&2
exit 1
fi
mkdir -p "${VENDORED}"
# Clear stale content while keeping the .gitkeep that anchors the embed path.
find "${VENDORED}" -mindepth 1 -not -name ".gitkeep" -delete
# Copy the entry script and the lib/ tree (modules + lib/vendor/).
cp "${ENGINE_SRC}/last30days.py" "${VENDORED}/last30days.py"
cp -R "${ENGINE_SRC}/lib" "${VENDORED}/lib"
# Strip caches so the embed.FS stays deterministic.
find "${VENDORED}" -type d -name "__pycache__" -prune -exec rm -rf {} +
find "${VENDORED}" -type f -name "*.pyc" -delete
echo "sync-engine: vendored engine at ${VENDORED}"
media/pr-assets/gogcli-589-zoom-demo.gif
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.4 MiB

media/pr-assets/last30days-ad.gif
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.6 MiB

pyproject.toml
+42
View File
@@ -0,0 +1,42 @@
[project]
name = "last30days-skill"
version = "3.13.0"
description = "Multi-source last-30-days research skill"
readme = "README.md"
requires-python = ">=3.12"
dependencies = []
[dependency-groups]
dev = [
"pytest>=9.1.0,<10",
"pytest-cov>=7,<8",
]
[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
addopts = [
"-q",
"--tb=short",
]
[tool.coverage.run]
branch = true
source = ["skills/last30days/scripts", "tests"]
omit = [
"skills/last30days/scripts/lib/vendor/*",
"dist/*",
]
[tool.coverage.report]
skip_empty = true
show_missing = true
# Coverage gate (issue #254). Floor intended to rise over time, not a ceiling.
# Baseline measured 2026-07-03 on main before feat/hosted-api-mode
# (source = scripts + tests): TOTAL 84.06%. Gate pinned at that baseline.
# Do not lower without documenting why in the PR (see AGENTS.md Rules).
fail_under = 84
omit = [
"skills/last30days/scripts/lib/vendor/*",
"dist/*",
]
skills/last30days/.skillignore
+13
View File
@@ -0,0 +1,13 @@
# Hermes scans from this skill directory, not the repository root.
# Keep non-runtime packaging/dev/eval artifacts out of install-time security scans.
assets/
agents/
scripts/build-skill.sh
scripts/compare.sh
scripts/evaluate_search_quality.py
scripts/test_device_auth.py
scripts/test-v1-vs-v2.sh
scripts/verify_v3.py
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
scripts/lib/vendor/
skills/last30days/SKILL.md
+2144
View File
File diff suppressed because it is too large Load Diff
skills/last30days/agents/openai.yaml
+8
View File
@@ -0,0 +1,8 @@
interface:
display_name: "Last 30 Days"
short_description: "Research any topic across Reddit, X, YouTube, and the web from the last 30 days. Returns synthesized expert answers and copy-paste prompts."
default_prompt: "Research this topic from the last 30 days across Reddit, X, YouTube, and web. Synthesize what people are actually saying, upvoting, and sharing right now."
brand_color: "#FF6B35"
policy:
allow_implicit_invocation: true
skills/last30days/assets/aging-portrait.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.7 MiB

skills/last30days/assets/claude-code-rap.mp3
BIN
View File
Binary file not shown.
skills/last30days/assets/dog-as-human.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 MiB

skills/last30days/assets/dog-original.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.8 MiB

skills/last30days/assets/swimmom-mockup.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.6 MiB

skills/last30days/references/save-html-brief.md
+204
View File
@@ -0,0 +1,204 @@
# Save shareable HTML brief
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either through an HTML-looking prompt argument like `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "give it to me in HTML", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here. Those prompt arguments are user intent signals for the skill; they are not the full Python CLI contract.
The contract has two modes:
- **HTML as the requested deliverable** (`--emit=html`, `--emit:html`, `--html`, or prose like "give it to me in HTML"): the HTML artifact is the primary output. Write the synthesis to the temp file, render the HTML, then give a concise artifact handoff in chat instead of pasting the full Markdown report again.
- **Normal report plus HTML copy** (the user asks for the normal report and also wants an HTML copy): the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
## When to fire this flow
- For normal-report-plus-HTML mode: after you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
- For HTML-as-deliverable mode: after you have drafted the synthesis that will go into the HTML, before emitting the final chat response.
- BEFORE the WAIT FOR USER'S RESPONSE pause.
- ONLY if the user asked. Do NOT save HTML when the user didn't ask for it.
## How to fire it
```bash
# 1. Write your synthesis prose VERBATIM to a temp file. The synthesis is the
# "What I learned:" prose label, the bold-lead-in paragraphs with their
# inline citations, and the "KEY PATTERNS from the research:" numbered list.
# Do NOT include the badge or the engine footer in the temp file - the engine
# adds those when it renders the HTML.
# - HTML-as-deliverable mode: use the exact synthesis draft you prepared for
# the artifact. Do not paste it to chat first.
# - Normal-report-plus-HTML mode: use the exact synthesis text you already
# wrote in chat.
# In both modes, do not paraphrase, summarize, or reorder. The HTML must read
# identically to the intended report in voice and citations.
SYNTHESIS_FILE="/tmp/last30days-synthesis-${CLAUDE_SESSION_ID}.md"
# >| not >: fixed path may already exist on a same-session re-run; a plain >
# is refused under `set -o noclobber`.
cat >| "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
What I learned:
**{First headline}** - {body with [name](url) inline citations}
**{Second headline}** - {body}
**{Third headline}** - {body}
KEY PATTERNS from the research:
1. {pattern} - per [@handle](url)
2. {pattern} - per [r/sub](url)
3. {pattern} - per [@handle](url)
SYNTHESIS_EOF
# 2. Convert the synthesis to a self-contained HTML file via the engine.
# REPLAY THE SAME SCOPE FLAGS as your original run (--plan, --hiring-signals,
# resolved --x-handle/--subreddits/etc). On a same-topic follow-up, the
# engine reuses the structured last-report cache at
# ~/.config/last30days/last-report.json to build badge metadata and footer
# without re-running source fetchers. That cache is intentionally short-lived
# (default: one hour; tune with LAST30DAYS_REPORT_CACHE_TTL_SECONDS, or set
# it to 0 to disable reuse). If the cache is stale, missing, or for a
# different topic, stderr says "No matching cached report data" and the
# engine falls back to a fresh run; the same scope flags keep that fallback
# aligned with the synthesis body.
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//;s/-$//')
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html"
# Collision guard: the `> "$HTML_PATH"` redirect below OVERWRITES - the engine
# does NOT auto-date the brief (its date-suffix logic applies only to --save-dir
# raw files, not to this redirected --emit=html stream). So if the clean name
# already exists, date-suffix it here to avoid clobbering a prior brief.
if [ -f "$HTML_PATH" ]; then
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief-$(date +%F).html"
fi
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
--emit=html \
--synthesis-file "$SYNTHESIS_FILE" \
"${SCOPE_FLAGS[@]}" \
>| "$HTML_PATH" # >| not >: noclobber-safe write to the collision-guarded path
# where SCOPE_FLAGS is the same array you passed the first time, e.g.
# SCOPE_FLAGS=(--hiring-signals --plan "$QUERY_PLAN_FILE" --x-handle=acme).
# For a scoped --hiring-signals brief, --hiring-signals MUST be here too so
# the footer reflects the jobs-scoped board, not a generic crawl.
# 3. Finish with the artifact handoff described below. Do not print the saved
# path from the shell block; the chat handoff is the single user-visible
# completion message.
```
## Optional hosted publishing
Only publish after the local HTML file has already been saved and the user chooses a publish option. The local HTML save is always first, and its absolute path is always shown before any publish/upload step.
Respect any existing user, project, or host preference for HTML publishing first. If the user already has a preferred publisher or internal sharing workflow, include that option. If multiple publishing options are available, show each as its own choice and include `ht-ml.app` as one option; label `ht-ml.app` as supporting optional password protection. If no preference exists, use `ht-ml.app` as the fallback publishing option.
Use this decision flow:
- Save the local HTML file.
- Show the absolute saved path.
- Then proactively present next-step choices:
1. Open HTML file
2. Publish to `<preferred/configured service>`; if `ht-ml.app` is shown, say password protection is available
3. Done for now
- Do not upload until the user chooses a publishing option.
When publishing to `ht-ml.app`, ask a second question:
- **Public link** - publish without a password.
- **Password-protected link** - ask the user to type the shared password in free form, then publish with that password.
Before the `ht-ml.app` choice, tell the user that public pages may be crawled or indexed, and that password protection is available. If the user chooses password protection, use a unique shared password they provide for this report; do not use their own account password.
Agents should discover the current publishing mechanics for the selected service when needed, including by visiting the service site, rather than hard-coding detailed service-specific instructions in chat. For the built-in `ht-ml.app` path, the engine supports `--publish-html`; on the password-protected branch, pass the shared password through `LAST30DAYS_PUBLISH_PASSWORD` rather than command-line arguments.
When the user chooses the built-in `ht-ml.app` path, add `--publish-html` to the same `--emit=html` command. Use `--output "$HTML_PATH"` rather than shell redirection so the engine can write the `.publish.json` companion metadata next to the local HTML file. On the password-protected branch, set `LAST30DAYS_PUBLISH_PASSWORD` in the subprocess environment instead of passing `--publish-password` in the shell command.
```bash
LAST30DAYS_PUBLISH_PASSWORD="${PUBLISH_PASSWORD:-}" \
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
--emit=html \
--synthesis-file "$SYNTHESIS_FILE" \
--output "$HTML_PATH" \
--publish-html \
"${SCOPE_FLAGS[@]}" \
>/dev/null
```
The hosted URL appears on stderr as `[last30days] Published HTML to https://...`. Confirm the result with the hosted URL. If the user chose password protection, also repeat the shared password they selected so they can send the URL and password together. The engine writes URL metadata to `<HTML_PATH>.publish.json`. The provider may return an `update_key`; treat it as secret. The engine deliberately does not write the update key to stdout, the HTML artifact, or `.publish.json` companion metadata.
## Chat handoff after saving
Use the mode that matches the request.
### HTML as the requested deliverable
When HTML is the requested deliverable - whether by `--emit=html`, `--emit:html`, `--html`, or natural-language phrasing - do **not** paste the full Markdown report back into chat after saving the artifact. The user asked for an HTML deliverable; repeating the Markdown makes the run feel like a normal report with an attachment bolted on.
Respond with a concise handoff that includes the next-step choices:
```text
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
📎 Shareable brief saved to <absolute HTML path>
What do you want to do next?
1. Open HTML file
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
3. Done for now
```
If the user chooses open, open the HTML file when the host can safely open local files, leave the saved-path line in chat, and add `Opened locally.` Let the host choose the correct OS-specific mechanism; do not print a menu of shell commands. If opening fails or the host is headless, do not treat that as a failed report; show the path and say the file is ready to open in a browser.
### Normal report plus HTML copy
When the user asked for a normal `/last30days` report and also asked for an HTML copy, keep the full chat synthesis and append this artifact block after the invitation:
```text
📎 Shareable brief saved to <absolute HTML path>
What do you want to do next?
1. Open HTML file
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
3. Done for now
```
If the user chooses open, open it when the host can safely open local files; otherwise the saved-path line is enough. Do not upload in this flow unless the user chooses a publishing option.
## What ends up in the HTML file
The engine's `--emit=html` renderer combines:
- The badge (`🌐 last30days vX.Y.Z · synced YYYY-MM-DD`) at the top
- A single inline metadata line (`{date range} · {active sources}`) below the badge
- Your synthesis verbatim, with prose labels promoted to `<h2>` and bold lead-ins preserved
- All `[name](url)` citations rendered as `<a>` tags
- The engine footer (`✅ All agents reported back!` tree) preserved verbatim in monospace
- A colophon with the topic and a re-run hint
The renderer strips engine-internal noise that doesn't belong in a shareable artifact: the `# last30days vX.Y.Z: TOPIC` debug file header, the model-facing `> Safety note:` blockquote, and the `I'm now an expert on X` invitation block. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs - they never leak into the share-ready file.
## Comparison mode
Same flow when the topic is `X vs Y` (or `X vs Y vs Z`). The engine routes through `render_for_html_comparison` internally; you don't need to do anything special. The synthesis temp file should still contain the comparison-shaped synthesis you wrote in chat (`## Quick Verdict`, `## {Entity}` per entity, `## Head-to-Head` table, `## The Bottom Line`, `## The emerging stack` per LAW 4 comparison exception).
## Follow-up turn
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN explicitly refers back to that visible synthesis ("save that as HTML", "make this shareable", "turn the above into HTML"), do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`, then use the normal-report-plus-HTML artifact block.
If the follow-up instead asks for a new HTML deliverable ("give it to me in HTML", `--emit=html`, `--html`) rather than referring back to an already-visible report, treat it as HTML-as-deliverable mode.
The engine will try to reuse `~/.config/last30days/last-report.json` for that second invocation when it is still within `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` (default: one hour). If stderr says it is reusing cached report data, continue normally. If stderr says no matching cache exists, the cache may be stale; let the command finish only if you supplied the same scope flags as the original run. Otherwise stop and re-run with the original flags so the HTML footer does not describe a different dataset.
## What NOT to do
- Do NOT save HTML if the user didn't ask. The sparse mode (no synthesis) produces a thin file; not useful as a shareable.
- Do NOT add content to the temp file beyond your synthesis prose. The badge / footer / colophon come from the engine.
- Do NOT change the file path convention. `${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html` is the canonical location.
- Do NOT silently overwrite an existing file. The `--emit=html` output is written via a shell redirect (`>| "$HTML_PATH"`), which OVERWRITES the collision-guarded path — use `>|` not `>` because `set -o noclobber` refuses plain `>` when the file already exists. The collision guard in step 2 handles same-topic re-runs: if `{slug}-brief.html` already exists it date-suffixes to `{slug}-brief-YYYY-MM-DD.html`. Always report whichever path the redirect actually used in the chat handoff.
- Do NOT include the data quality warning text in the temp file or in your final chat line. Warnings are an engine-stderr concern, not an artifact concern.
- Do NOT publish, upload, or send the HTML to a third-party service as part of the local save flow.
- Do NOT publish to any service merely because HTML was requested. Show the saved path and next-step choices first; publishing requires the user to choose a publish option.
- Do NOT block a local HTML export on a hosting decision unless the user explicitly asked for a hosted URL.
- Do NOT paste or store the `update_key` in chat, Markdown, HTML, raw output, or companion metadata.
## Edge cases
- **Topic with shell-special characters** (quotes, ampersands): the temp filename uses a slugified version, but the engine receives the raw topic. The `cat <<'SYNTHESIS_EOF'` quoted heredoc form handles arbitrary content without expansion. Your synthesis text can include any character.
- **Very long synthesis**: no upper bound. The engine handles long markdown bodies. Just paste verbatim.
- **Synthesis with images or non-ASCII**: emoji and Unicode pass through. Image tags pass through as raw HTML; the renderer doesn't transform them. If you didn't include images in chat, don't add them here.
- **No `${LAST30DAYS_MEMORY_DIR}` set**: defaults to `~/Documents/Last30Days/` per the SKILL.md `Configuration` section.
skills/last30days/scripts/briefing.py
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python3
"""Morning briefing generator for last30days.
Synthesizes accumulated findings into formatted briefings.
The Python script collects the data; the agent (via SKILL.md) does the
beautiful synthesis. This script provides the structured data.
Usage:
python3 briefing.py generate # Daily briefing data
python3 briefing.py generate --weekly # Weekly digest data
python3 briefing.py show [--date DATE] # Show saved briefing
"""
import argparse
import json
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
SCRIPT_DIR = Path(__file__).parent.resolve()
sys.path.insert(0, str(SCRIPT_DIR))
import store
BRIEFS_DIR = Path.home() / ".local" / "share" / "last30days" / "briefs"
def _parse_sqlite_utc_timestamp(value: str) -> datetime:
return datetime.strptime(value, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
def generate_daily(since: str = None) -> dict:
"""Generate daily briefing data.
Returns structured data for the agent to synthesize into a beautiful briefing.
"""
store.init_db()
topics = store.list_topics()
if not topics:
return {
"status": "no_topics",
"message": "No watchlist topics yet. Add one with: last30days watch add \"your topic\"",
}
enabled = [t for t in topics if t["enabled"]]
if not enabled:
return {
"status": "no_enabled",
"message": "All topics are paused. Enable a topic to generate briefings.",
}
# Default: findings since yesterday
if not since:
since = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
briefing_topics = []
total_new = 0
for topic in enabled:
findings = store.get_new_findings(topic["id"], since)
last_run = topic.get("last_run")
last_status = topic.get("last_status", "unknown")
# Calculate staleness
stale = False
hours_ago = None
if last_run:
try:
run_dt = _parse_sqlite_utc_timestamp(last_run)
hours_ago = (datetime.now(timezone.utc) - run_dt).total_seconds() / 3600
stale = hours_ago > 36 # Stale if > 36 hours
except (ValueError, TypeError):
stale = True
topic_data = {
"name": topic["name"],
"findings": findings,
"new_count": len(findings),
"last_run": last_run,
"last_status": last_status,
"stale": stale,
"hours_ago": round(hours_ago, 1) if hours_ago else None,
}
# Extract top finding by engagement
if findings:
top = max(findings, key=lambda f: f.get("engagement_score", 0))
topic_data["top_finding"] = {
"title": top.get("source_title", ""),
"source": top.get("source", ""),
"author": top.get("author", ""),
"engagement": top.get("engagement_score", 0),
"content": top.get("content", "")[:300],
}
briefing_topics.append(topic_data)
total_new += len(findings)
# Cost info
daily_cost = store.get_daily_cost()
budget = float(store.get_setting("daily_budget", "5.00"))
# Find the single top finding across all topics (for TL;DR)
all_findings = []
for t in briefing_topics:
for f in t["findings"]:
f["_topic"] = t["name"]
all_findings.append(f)
top_overall = None
if all_findings:
top_overall = max(all_findings, key=lambda f: f.get("engagement_score", 0))
result = {
"status": "ok",
"date": datetime.now().strftime("%Y-%m-%d"),
"since": since,
"topics": briefing_topics,
"total_new": total_new,
"total_topics": len(briefing_topics),
"top_finding": {
"title": top_overall.get("source_title", ""),
"topic": top_overall.get("_topic", ""),
"engagement": top_overall.get("engagement_score", 0),
} if top_overall else None,
"cost": {
"daily": daily_cost,
"budget": budget,
},
"failed_topics": [
t["name"] for t in briefing_topics if t["last_status"] == "failed"
],
}
# Save briefing data
_save_briefing(result)
return result
def generate_weekly() -> dict:
"""Generate weekly digest data with trend analysis."""
store.init_db()
week_ago = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
two_weeks_ago = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
topics = store.list_topics()
if not topics:
return {"status": "no_topics", "message": "No watchlist topics."}
weekly_topics = []
for topic in topics:
if not topic["enabled"]:
continue
# This week's findings
this_week = store.get_new_findings(topic["id"], week_ago)
# Last week's findings (for comparison)
conn = store._connect()
try:
last_week_rows = conn.execute(
"""SELECT * FROM findings
WHERE topic_id = ? AND first_seen >= ? AND first_seen < ? AND dismissed = 0
ORDER BY engagement_score DESC""",
(topic["id"], two_weeks_ago, week_ago),
).fetchall()
last_week = [dict(r) for r in last_week_rows]
finally:
conn.close()
this_engagement = sum(f.get("engagement_score", 0) for f in this_week)
last_engagement = sum(f.get("engagement_score", 0) for f in last_week)
# Trend calculation
if last_engagement > 0:
engagement_change = ((this_engagement - last_engagement) / last_engagement) * 100
else:
engagement_change = 100 if this_engagement > 0 else 0
weekly_topics.append({
"name": topic["name"],
"this_week_count": len(this_week),
"last_week_count": len(last_week),
"this_week_engagement": this_engagement,
"last_week_engagement": last_engagement,
"engagement_change_pct": round(engagement_change, 1),
# get_new_findings returns first_seen DESC, so sort by engagement
# before slicing — otherwise the digest headlines the most recent
# items, not the highest-engagement ones (the daily path keys on
# engagement too).
"top_findings": sorted(
this_week,
key=lambda f: f.get("engagement_score", 0),
reverse=True,
)[:5],
})
result = {
"status": "ok",
"type": "weekly",
"week_of": week_ago,
"topics": weekly_topics,
}
_save_briefing(result, suffix="-weekly")
return result
def show_briefing(date: str = None) -> dict:
"""Load a saved briefing by date."""
if not date:
date = datetime.now().strftime("%Y-%m-%d")
path = BRIEFS_DIR / f"{date}.json"
if not path.exists():
# Try weekly
path = BRIEFS_DIR / f"{date}-weekly.json"
if not path.exists():
return {"status": "not_found", "message": f"No briefing found for {date}."}
with open(path, encoding="utf-8") as f:
return json.load(f)
def _save_briefing(data: dict, suffix: str = ""):
"""Save briefing data to local archive."""
BRIEFS_DIR.mkdir(parents=True, exist_ok=True)
date = datetime.now().strftime("%Y-%m-%d")
path = BRIEFS_DIR / f"{date}{suffix}.json"
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, default=str)
def main():
parser = argparse.ArgumentParser(description="Generate last30days briefings")
sub = parser.add_subparsers(dest="command")
# generate
g = sub.add_parser("generate", help="Generate a briefing")
g.add_argument("--weekly", action="store_true", help="Weekly digest")
g.add_argument("--since", help="Findings since date (YYYY-MM-DD)")
# show
s = sub.add_parser("show", help="Show a saved briefing")
s.add_argument("--date", help="Date (YYYY-MM-DD, default: today)")
args = parser.parse_args()
if args.command == "generate":
if args.weekly:
result = generate_weekly()
else:
result = generate_daily(since=args.since)
print(json.dumps(result, indent=2, default=str))
elif args.command == "show":
result = show_briefing(date=args.date)
print(json.dumps(result, indent=2, default=str))
else:
parser.print_help()
sys.exit(1)
if __name__ == "__main__":
main()
skills/last30days/scripts/build-skill.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
#
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
# See
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
cd "$REPO_ROOT"
if ! git diff --quiet || ! git diff --cached --quiet; then
echo "error: working tree is dirty; commit or stash before building" >&2
exit 1
fi
mkdir -p dist
OUT="dist/last30days.skill"
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
SIZE=$(du -h "$OUT" | cut -f1)
if [ "$COUNT" -gt 200 ]; then
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2
exit 1
fi
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
exit 1
fi
echo "built $OUT ($COUNT files, $SIZE)"
echo "upload via the claude.ai skill UI"
skills/last30days/scripts/compare.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
# A/B test runner: public release vs private beta
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
#
# Runs /last30days (public release) and /last30days-beta (private beta)
# sequentially with a 30s gap, saves raw results with distinct suffixes,
# prints file paths for comparison.
set -e
if [ $# -eq 0 ]; then
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
exit 1
fi
TOPIC="$*"
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
DIR="$LAST30DAYS_MEMORY_DIR"
DATE=$(date +%Y-%m-%d)
echo "=============================================="
echo " A/B Test: $TOPIC"
echo " Date: $DATE"
echo "=============================================="
echo ""
# Run 1: public release
echo "[1/2] Running /last30days (public release)..."
echo " This takes 2-4 minutes..."
claude -p "/last30days $TOPIC" > /dev/null 2>&1 || true
RELEASE_FILE="$DIR/${SLUG}-raw.md"
[ -f "$RELEASE_FILE" ] && echo " Done: $RELEASE_FILE" || echo " FAILED: no output file"
echo ""
echo " Waiting 30s for API rate limits..."
sleep 30
# Run 2: private beta
echo "[2/2] Running /last30days-beta (private beta)..."
echo " This takes 2-4 minutes..."
claude -p "/last30days-beta $TOPIC" > /dev/null 2>&1 || true
BETA_FILE="$DIR/${SLUG}-raw-beta.md"
[ -f "$BETA_FILE" ] && echo " Done: $BETA_FILE" || echo " FAILED: no output file"
echo ""
echo "=============================================="
echo " Both complete. Raw files:"
echo "=============================================="
echo ""
ls -la "$DIR/${SLUG}-raw"*.md 2>/dev/null || echo " (no files found - check if skills saved correctly)"
echo ""
echo "To compare, run in Claude Code:"
echo " Read and compare these raw research files, produce a detailed report:"
echo " $RELEASE_FILE"
echo " $BETA_FILE"
echo ""
echo "Beta output should start with a line like:"
echo " 🧪 last30days-beta · branch <name> · synced $DATE"
echo "If that line is missing, the beta badge regressed. See docs/plans/2026-04-17-005-*-plan.md."
echo ""
skills/last30days/scripts/evaluate_search_quality.py
+588
View File
@@ -0,0 +1,588 @@
#!/usr/bin/env python3
"""Compare two last30days revisions on the v3 ranked candidate output."""
from __future__ import annotations
import argparse
import json
import math
import os
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
sys.path.insert(0, str(Path(__file__).parent))
from lib import env as envlib
from lib import schema
from lib.providers import GEMINI_FLASH_LITE
SKILL_ROOT = Path(__file__).resolve().parents[1]
REPO_ROOT = Path(__file__).resolve().parents[3]
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
def _load_default_topics() -> list[tuple[str, str]]:
if EVAL_TOPICS_FILE.exists():
rows = json.loads(EVAL_TOPICS_FILE.read_text())
return [(row["topic"], row["query_type"]) for row in rows]
return [
("nano banana pro prompting", "product"),
("codex vs claude code", "comparison"),
("openclaw vs nanoclaw vs ironclaw", "comparison"),
("anthropic odds", "prediction"),
("kanye west", "breaking_news"),
("remotion animations for Claude Code", "how_to"),
]
DEFAULT_TOPICS = _load_default_topics()
DEFAULT_SEARCH = ""
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
EVAL_CREDENTIAL_ENV_KEYS = (
"GOOGLE_API_KEY",
"GEMINI_API_KEY",
"GOOGLE_GENAI_API_KEY",
"OPENAI_API_KEY",
"XAI_API_KEY",
"SCRAPECREATORS_API_KEY",
"BSKY_HANDLE",
"BSKY_APP_PASSWORD",
"TRUTHSOCIAL_TOKEN",
"AUTH_TOKEN",
"CT0",
)
def stable_item_key(item: dict[str, Any]) -> str:
return str(item.get("candidate_id") or item.get("url") or item.get("title") or "")
def row_sources(row: dict[str, Any]) -> list[str]:
candidate = schema.candidate_from_dict(row)
return schema.candidate_sources(candidate)
def row_best_date(row: dict[str, Any]) -> str | None:
candidate = schema.candidate_from_dict(row)
return schema.candidate_best_published_at(candidate)
V2_SOURCE_KEYS = [
("reddit", "title"),
("x", "text"),
("youtube", "title"),
("tiktok", "text"),
("instagram", "text"),
("hackernews", "title"),
("bluesky", "text"),
("truthsocial", "text"),
("polymarket", "question"),
("web", "title"),
]
def build_ranked_items(report: dict[str, Any], limit: int) -> list[dict[str, Any]]:
# v3 format: ranked_candidates list
if report.get("ranked_candidates"):
ranked = []
for row in report["ranked_candidates"][:limit]:
candidate_sources = row_sources(row)
ranked.append({
"key": stable_item_key(row),
"source": ", ".join(candidate_sources),
"sources": candidate_sources,
"url": str(row.get("url") or ""),
"text": str(row.get("title") or ""),
"date": row_best_date(row),
"score": float(row.get("final_score") or 0.0),
})
return ranked
# v2 format: per-source lists (reddit, x, youtube, etc.)
all_items = []
for source_key, text_field in V2_SOURCE_KEYS:
for item in report.get(source_key) or []:
if not isinstance(item, dict):
continue
all_items.append({
"key": str(item.get("url") or item.get("id") or item.get(text_field) or ""),
"source": source_key,
"sources": [source_key],
"url": str(item.get("url") or ""),
"text": str(item.get(text_field) or item.get("title") or ""),
"date": item.get("date"),
"score": float(item.get("score") or 0.0),
})
all_items.sort(key=lambda x: x["score"], reverse=True)
return all_items[:limit]
def source_sets(report: dict[str, Any], limit: int) -> dict[str, set[str]]:
grouped: dict[str, set[str]] = {}
for item in build_ranked_items(report, limit):
for source in item["sources"]:
grouped.setdefault(source, set()).add(item["key"])
return grouped
def jaccard(left: set[str], right: set[str]) -> float:
if not left and not right:
return 1.0
union = left | right
if not union:
return 1.0
return len(left & right) / len(union)
def retention(left: set[str], right: set[str]) -> float:
if not left:
return 1.0
return len(left & right) / len(left)
def precision_at_k(ranking: list[dict[str, Any]], judgments: dict[str, int], k: int) -> float:
top = ranking[:k]
if not top:
return 0.0
return sum(1 for item in top if judgments.get(item["key"], 0) >= 2) / len(top)
def ndcg_at_k(ranking: list[dict[str, Any]], judgments: dict[str, int], k: int, judged_pool: list[dict[str, Any]]) -> float:
top = ranking[:k]
if not top:
return 0.0
def dcg(grades: list[int]) -> float:
total = 0.0
for index, grade in enumerate(grades, start=1):
total += (2**grade - 1) / math.log2(index + 1)
return total
actual = [judgments.get(item["key"], 0) for item in top]
ideal = sorted((judgments.get(item["key"], 0) for item in judged_pool), reverse=True)[: len(top)]
ideal_score = dcg(ideal)
if ideal_score == 0:
return 0.0
return dcg(actual) / ideal_score
def source_coverage_recall(ranking: list[dict[str, Any]], judged_pool: list[dict[str, Any]], judgments: dict[str, int]) -> float:
good_sources = {
source
for item in judged_pool
if judgments.get(item["key"], 0) >= 2
for source in item["sources"]
}
if not good_sources:
return 1.0
hit_sources = {
source
for item in ranking
if judgments.get(item["key"], 0) >= 2
for source in item["sources"]
}
return len(hit_sources & good_sources) / len(good_sources)
def resolve_google_judge_api_key(config: dict[str, Any]) -> str | None:
return (
os.environ.get("GOOGLE_API_KEY")
or config.get("GOOGLE_API_KEY")
or os.environ.get("GEMINI_API_KEY")
or config.get("GEMINI_API_KEY")
or os.environ.get("GOOGLE_GENAI_API_KEY")
or config.get("GOOGLE_GENAI_API_KEY")
)
def extract_gemini_text(payload: dict[str, Any]) -> str:
for candidate in payload.get("candidates") or []:
content = candidate.get("content") or {}
for part in content.get("parts") or []:
if part.get("text"):
return part["text"]
raise ValueError("Gemini response did not contain text.")
def call_gemini_judge(api_key: str, model: str, prompt: str) -> dict[str, Any]:
body = {
"contents": [{"parts": [{"text": prompt}]}],
"generationConfig": {"temperature": 0, "responseMimeType": "application/json"},
}
request = Request(
GEMINI_API_URL.format(model=model, api_key=api_key),
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urlopen(request, timeout=120) as response:
payload = json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(f"Gemini HTTP {exc.code}: {detail}") from exc
except URLError as exc:
raise RuntimeError(f"Gemini request failed: {exc}") from exc
return json.loads(extract_gemini_text(payload))
def build_judge_prompt(topic: str, query_type: str, items: list[dict[str, Any]]) -> str:
item_lines = []
for item in items:
item_lines.append(
"\n".join([
f"- id: {item['key']}",
f" source: {item['source']}",
f" title: {item['text'][:220]}",
f" url: {item['url']}",
f" date: {item.get('date') or 'unknown'}",
])
)
return f"""
Judge search-result relevance for a last-30-days research tool.
Topic: {topic}
Query type: {query_type}
Score each item on this 0-3 scale:
- 0 = off-topic or clearly bad
- 1 = weak or tangential
- 2 = relevant and useful
- 3 = highly relevant, one of the best results
Return JSON only:
{{
"judgments": [
{{"id": "ITEM_ID", "grade": 0}}
]
}}
Items:
{chr(10).join(item_lines)}
""".strip()
def get_judgments(
*,
output_dir: Path,
slug: str,
topic: str,
query_type: str,
items: list[dict[str, Any]],
judge_model: str,
gemini_api_key: str | None,
) -> dict[str, int]:
cache_file = output_dir / "judgments" / f"{slug}.json"
cache_file.parent.mkdir(parents=True, exist_ok=True)
stale_cache = False
if cache_file.exists():
payload = json.loads(cache_file.read_text())
# The cache key is the topic slug alone, but judgments are model-
# specific. Only reuse the cache when it was produced by the same judge
# model; otherwise re-judge, so a --judge-model change cannot return
# stale grades that silently skew precision@k / nDCG. Caches written
# before judge_model was recorded miss here and get refreshed once.
if payload.get("judge_model") == judge_model:
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
stale_cache = True
if not gemini_api_key or not items:
if stale_cache:
# Discarded a different-model cache but can't re-judge. Returning {}
# scores every item as ungraded (zero precision@k / nDCG); say so
# rather than letting the run report silently wrong numbers.
sys.stderr.write(
f"[Eval] Cached judgments for {slug!r} were graded by a different "
f"judge model and no Gemini API key is set to re-judge; returning "
f"no grades (metrics for this topic will be zero).\n"
)
return {}
payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items))
payload["judge_model"] = judge_model
cache_file.write_text(json.dumps(payload, indent=2))
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
def create_eval_env() -> dict[str, str]:
config = envlib.get_config()
passthrough = {
"PATH": os.environ.get("PATH", ""),
"LANG": os.environ.get("LANG", "en_US.UTF-8"),
"LC_ALL": os.environ.get("LC_ALL", ""),
"TMPDIR": os.environ.get("TMPDIR", ""),
"PYTHONUTF8": "1",
"LAST30DAYS_CONFIG_DIR": "",
}
for key in EVAL_CREDENTIAL_ENV_KEYS:
value = os.environ.get(key) or config.get(key)
if value:
passthrough[key] = value
return passthrough
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
if not engine.exists():
engine = repo_dir / "scripts" / "last30days.py"
cmd = [sys.executable, str(engine), topic, "--emit=json"]
# Current engines default to the stable agent export, while older revisions
# used by the evaluator implicitly emit the raw report and do not recognize
# --json-profile. Request raw explicitly whenever the checked-out engine
# supports the selector.
if not engine.exists() or "--json-profile" in engine.read_text(encoding="utf-8"):
cmd.append("--json-profile=raw")
if search:
cmd.extend(["--search", search])
if quick:
cmd.append("--quick")
if mock:
cmd.append("--mock")
result = subprocess.run(
cmd,
cwd=repo_dir,
env=env,
capture_output=True,
text=True,
timeout=timeout_seconds,
check=False,
)
if result.returncode != 0:
raise RuntimeError(f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}")
payload = json.loads(result.stdout)
# Shape guard: the evaluator compares raw Report fields. If the engine
# emitted the agent profile anyway (flag detection missed a future
# spelling), fail loudly instead of scoring empty ranked_candidates.
if "schema_version" in payload and "ranked_candidates" not in payload:
raise RuntimeError(
f"{repo_dir.name} emitted the agent JSON profile; the evaluator "
"requires the raw Report (--json-profile=raw)."
)
return payload
def create_worktree(rev: str) -> Path:
worktree_dir = Path(tempfile.mkdtemp(prefix="last30days-eval-"))
subprocess.run(
["git", "worktree", "add", "--detach", str(worktree_dir), rev],
cwd=REPO_ROOT,
check=True,
capture_output=True,
text=True,
)
return worktree_dir
def resolve_repo_dir(label: str) -> tuple[Path, bool]:
"""Resolve a benchmark label into a repo directory and whether it is temporary."""
if label == "WORKTREE":
return REPO_ROOT, False
return create_worktree(label), True
def remove_worktree(path: Path) -> None:
subprocess.run(
["git", "worktree", "remove", "--force", str(path)],
cwd=REPO_ROOT,
check=False,
capture_output=True,
text=True,
)
try:
os.rmdir(path)
except OSError:
pass
def summarize_topic(topic: str, query_type: str, baseline_report: dict[str, Any], candidate_report: dict[str, Any], judgments: dict[str, int], judged_pool: list[dict[str, Any]], limit: int) -> dict[str, Any]:
baseline_ranked = build_ranked_items(baseline_report, limit)
candidate_ranked = build_ranked_items(candidate_report, limit)
baseline_sets = source_sets(baseline_report, limit)
candidate_sets = source_sets(candidate_report, limit)
overall_left = set().union(*baseline_sets.values()) if baseline_sets else set()
overall_right = set().union(*candidate_sets.values()) if candidate_sets else set()
sources = sorted(set(baseline_sets) | set(candidate_sets))
return {
"topic": topic,
"query_type": query_type,
"baseline": {
"precision_at_5": precision_at_k(baseline_ranked, judgments, 5),
"ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5, judged_pool),
"source_coverage_recall": source_coverage_recall(baseline_ranked, judged_pool, judgments),
},
"candidate": {
"precision_at_5": precision_at_k(candidate_ranked, judgments, 5),
"ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5, judged_pool),
"source_coverage_recall": source_coverage_recall(candidate_ranked, judged_pool, judgments),
},
"stability": {
"overall_jaccard": jaccard(overall_left, overall_right),
"overall_retention_vs_baseline": retention(overall_left, overall_right),
"per_source": {
source: {
"baseline_count": len(baseline_sets.get(source, set())),
"candidate_count": len(candidate_sets.get(source, set())),
"jaccard": jaccard(baseline_sets.get(source, set()), candidate_sets.get(source, set())),
"retention_vs_baseline": retention(baseline_sets.get(source, set()), candidate_sets.get(source, set())),
}
for source in sources
},
},
}
def write_summary(output_dir: Path, baseline_label: str, candidate_label: str, summaries: list[dict[str, Any]]) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
payload = {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"baseline": baseline_label,
"candidate": candidate_label,
"topics": summaries,
}
(output_dir / "metrics.json").write_text(json.dumps(payload, indent=2))
lines = [
"# Search Quality Evaluation",
"",
f"- Baseline: `{baseline_label}`",
f"- Candidate: `{candidate_label}`",
f"- Generated: {payload['generated_at']}",
"",
"| Topic | Base P@5 | Cand P@5 | Base nDCG@5 | Cand nDCG@5 | Jaccard | Retention |",
"|---|---:|---:|---:|---:|---:|---:|",
]
for row in summaries:
lines.append(
"| {topic} | {bp:.2f} | {cp:.2f} | {bn:.2f} | {cn:.2f} | {jac:.2f} | {ret:.2f} |".format(
topic=row["topic"],
bp=row["baseline"]["precision_at_5"],
cp=row["candidate"]["precision_at_5"],
bn=row["baseline"]["ndcg_at_5"],
cn=row["candidate"]["ndcg_at_5"],
jac=row["stability"]["overall_jaccard"],
ret=row["stability"]["overall_retention_vs_baseline"],
)
)
(output_dir / "summary.md").write_text("\n".join(lines) + "\n")
def write_failure_summary(
output_dir: Path,
baseline_label: str,
candidate_label: str,
summaries: list[dict[str, Any]],
failures: list[dict[str, Any]],
) -> None:
write_summary(output_dir, baseline_label, candidate_label, summaries)
metrics_path = output_dir / "metrics.json"
payload = json.loads(metrics_path.read_text()) if metrics_path.exists() else {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"baseline": baseline_label,
"candidate": candidate_label,
"topics": [],
}
payload["failures"] = failures
metrics_path.write_text(json.dumps(payload, indent=2))
summary_path = output_dir / "summary.md"
lines = summary_path.read_text().splitlines() if summary_path.exists() else ["# Search Quality Evaluation", ""]
if failures:
lines.extend([
"",
"## Failures",
"",
])
for failure in failures:
lines.append(f"- `{failure['topic']}`: {failure['error']}")
summary_path.write_text("\n".join(lines).rstrip() + "\n")
def parse_topics_file(path: Path) -> list[tuple[str, str]]:
rows = json.loads(path.read_text())
return [(str(row["topic"]), str(row.get("query_type") or "general")) for row in rows]
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Compare two last30days revisions on ranked candidate quality")
parser.add_argument("--baseline", default="HEAD~1")
parser.add_argument("--candidate", default="WORKTREE")
parser.add_argument("--search", default=DEFAULT_SEARCH)
parser.add_argument("--output-dir", default="tmp/search-quality")
parser.add_argument("--judge-model", default=DEFAULT_JUDGE_MODEL)
parser.add_argument("--timeout", type=int, default=240)
parser.add_argument("--limit", type=int, default=20)
parser.add_argument("--mock", action="store_true")
parser.add_argument("--quick", action="store_true")
parser.add_argument("--topics-file")
return parser
def main() -> int:
args = build_parser().parse_args()
topics = parse_topics_file(Path(args.topics_file)) if args.topics_file else DEFAULT_TOPICS
output_dir = Path(args.output_dir).resolve()
config = envlib.get_config()
gemini_api_key = resolve_google_judge_api_key(config)
run_env = create_eval_env()
baseline_dir, baseline_temp = resolve_repo_dir(args.baseline)
candidate_dir, candidate_temp = resolve_repo_dir(args.candidate)
try:
summaries = []
failures = []
for topic, query_type in topics:
try:
baseline_report = run_last30days(
baseline_dir,
topic,
search=args.search,
timeout_seconds=args.timeout,
quick=args.quick,
mock=args.mock,
env=run_env,
)
candidate_report = run_last30days(
candidate_dir,
topic,
search=args.search,
timeout_seconds=args.timeout,
quick=args.quick,
mock=args.mock,
env=run_env,
)
judged_pool_map = {
item["key"]: item
for item in build_ranked_items(baseline_report, args.limit) + build_ranked_items(candidate_report, args.limit)
}
judged_pool = list(judged_pool_map.values())
judgments = get_judgments(
output_dir=output_dir,
slug="".join(char.lower() if char.isalnum() else "-" for char in topic).strip("-"),
topic=topic,
query_type=query_type,
items=judged_pool,
judge_model=args.judge_model,
gemini_api_key=gemini_api_key,
)
summaries.append(summarize_topic(topic, query_type, baseline_report, candidate_report, judgments, judged_pool, args.limit))
except Exception as exc:
failures.append({"topic": topic, "query_type": query_type, "error": str(exc)})
write_failure_summary(output_dir, args.baseline, args.candidate, summaries, failures)
finally:
if baseline_temp:
remove_worktree(baseline_dir)
if candidate_temp:
remove_worktree(candidate_dir)
result = {"output_dir": str(output_dir), "topics": len(topics), "failures": len(failures)}
print(json.dumps(result, indent=2))
return 1 if failures else 0
if __name__ == "__main__":
raise SystemExit(main())
skills/last30days/scripts/last30days.py
+2655
View File
File diff suppressed because it is too large Load Diff
skills/last30days/scripts/lib/__init__.py
+1
View File
@@ -0,0 +1 @@
# last30days library modules
skills/last30days/scripts/lib/arxiv.py
+290
View File
@@ -0,0 +1,290 @@
"""arXiv research-paper source for last30days.
Shells out to ``arxiv-pp-cli`` (open Atom API, no auth) to surface recent
research papers relevant to a topic. arXiv carries no engagement signal, so
ranking leans on relevance (the CLI's own relevance sort plus token overlap)
and recency.
Activation gate: this source is only available when ``arxiv-pp-cli`` is on
PATH. ``pipeline.available_sources`` checks ``shutil.which`` before including
``arxiv``. The functions below also detect the missing-binary case defensively.
Default-on safety (two gates, both required):
1. Query construction. arXiv is queried with a *quoted* phrase and
``--sort-by relevance``. Sorting by submitted-date instead returns the
newest cs.* papers regardless of topic -- topic-blind noise.
2. Recency cutoff. Entries older than ``RECENCY_DAYS`` are dropped. Research
does not trend on a 30-day clock, so this window is wider than the social
sources' 30 days; it keeps arXiv current while dropping stale keyword
matches (e.g. a 2017 sports-statistics paper that an off-topic query like
"Golden State Warriors" would otherwise surface).
"""
from __future__ import annotations
import json
import shutil
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from . import log, subproc
from .relevance import token_overlap_relevance
CLI_BIN = "arxiv-pp-cli"
# Per-depth result counts.
DEPTH_CONFIG = {
"quick": 5,
"default": 10,
"deep": 20,
}
# Recency window for arXiv specifically. Papers do not trend daily; a year keeps
# the source current (the off-topic 2017 paper still drops) without discarding
# the genuinely-relevant work from the last few months.
RECENCY_DAYS = 365
SEARCH_TIMEOUT = 30
def _log(msg: str) -> None:
log.source_log("arXiv", msg, tty_only=False)
def _is_available() -> bool:
"""True when the arxiv-pp-cli binary is on PATH."""
return shutil.which(CLI_BIN) is not None
def _today() -> datetime:
return datetime.now(timezone.utc)
def _build_search_query(topic: str) -> str:
"""Quote the topic so arXiv treats it as a phrase across all fields.
Inner double-quotes are stripped (arXiv has no phrase-escaping); the outer
quotes plus ``all:`` give a phrase-scoped relevance search.
"""
return f'all:"{_clean_phrase(topic)}"'
def _clean_phrase(topic: str) -> str:
"""Strip quotes and collapse whitespace into a phrase for the query."""
return " ".join(topic.replace('"', " ").split())
def _build_search_args(topic: str, limit: int) -> List[str]:
return [
CLI_BIN,
"query",
"--search-query",
_build_search_query(topic),
"--sort-by",
"relevance",
"--max-results",
str(limit),
"--agent",
]
def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
"""Invoke arxiv-pp-cli and parse the JSON envelope.
arXiv returns ``{"meta": ..., "results": {"entries": [...]}}``. This
normalizes to ``{"results": [...entries...]}`` so the parse step sees a
flat list, matching the other sources' shape. Never raises.
"""
if not _is_available():
return {"results": [], "error": f"{CLI_BIN} not on PATH"}
try:
result = subproc.run_with_timeout(cmd, timeout=timeout)
except subproc.SubprocTimeout as exc:
_log(f"Timeout: {exc}")
return {"results": [], "error": str(exc)}
except FileNotFoundError as exc:
_log(f"Binary missing: {exc}")
return {"results": [], "error": str(exc)}
except OSError as exc:
_log(f"Spawn failed: {exc}")
return {"results": [], "error": str(exc)}
if result.returncode != 0:
snippet = (result.stderr or "").strip().splitlines()[:1]
first = snippet[0] if snippet else f"exit {result.returncode}"
_log(f"CLI exit {result.returncode}: {first}")
return {"results": [], "error": first}
stdout = result.stdout or ""
if not stdout.strip():
return {"results": []}
try:
data = json.loads(stdout)
except json.JSONDecodeError as exc:
_log(f"JSON decode failed: {exc}")
return {"results": [], "error": f"json decode: {exc}"}
return {"results": _extract_entries(data)}
def _extract_entries(data: Any) -> List[Dict[str, Any]]:
"""Pull the entries list out of arXiv's nested envelope.
Tolerates ``{"results": {"entries": [...]}}`` (current shape),
``{"entries": [...]}``, and a bare list.
"""
if isinstance(data, list):
return [e for e in data if isinstance(e, dict)]
if isinstance(data, dict):
results = data.get("results")
if isinstance(results, dict):
entries = results.get("entries")
if isinstance(entries, list):
return [e for e in entries if isinstance(e, dict)]
if isinstance(results, list):
return [e for e in results if isinstance(e, dict)]
entries = data.get("entries")
if isinstance(entries, list):
return [e for e in entries if isinstance(e, dict)]
return []
def search_arxiv(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> Dict[str, Any]:
"""Search arXiv via arxiv-pp-cli using a quoted, relevance-sorted query.
Returns a dict with a flat ``results`` list of entry dicts. On failure,
``results`` is empty and an ``error`` key carries a one-line description.
"""
if not topic or not topic.strip():
return {"results": []}
# A topic of only quote characters cleans to an empty phrase (all:""),
# which is a topic-blind query; bail rather than search for nothing.
if not _clean_phrase(topic):
return {"results": []}
limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
cmd = _build_search_args(topic, limit)
_log(f"query '{topic}' (relevance, max={limit})")
response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
_log(f"found {len(response.get('results') or [])} entries")
return response
def _parse_published(published: Optional[str]) -> Optional[datetime]:
"""Parse an arXiv ``published`` timestamp (ISO 8601, e.g.
'2026-06-25T17:59:48Z') into an aware datetime. Returns None on failure."""
if not published or not isinstance(published, str):
return None
text = published.strip().replace("Z", "+00:00")
try:
dt = datetime.fromisoformat(text)
except ValueError:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
def _alternate_url(entry: Dict[str, Any]) -> str:
"""Return the human-facing abstract URL (rel=alternate), not the PDF."""
links = entry.get("links")
if isinstance(links, list):
for link in links:
if isinstance(link, dict) and link.get("rel") == "alternate":
href = str(link.get("href") or "").strip()
if href:
return href
# Fall back to the abstract URL derived from the entry id.
entry_id = str(entry.get("id") or "").strip()
if entry_id.startswith("http"):
return entry_id
return ""
def _author_names(entry: Dict[str, Any]) -> List[str]:
authors = entry.get("authors")
out: List[str] = []
if isinstance(authors, list):
for a in authors:
if isinstance(a, dict):
name = str(a.get("name") or "").strip()
if name:
out.append(name)
return out
def parse_arxiv_response(
response: Dict[str, Any],
query: str = "",
today: Optional[datetime] = None,
) -> List[Dict[str, Any]]:
"""Parse an arXiv envelope into normalized item dicts.
Applies the recency cutoff (drops entries older than ``RECENCY_DAYS`` and
entries with an unparseable date) and computes a token-overlap relevance
hint. Returns dicts ready for ``normalize._normalize_arxiv``.
"""
raw = response.get("results") if isinstance(response, dict) else None
if not isinstance(raw, list):
return []
now = today or _today()
items: List[Dict[str, Any]] = []
for i, entry in enumerate(raw):
if not isinstance(entry, dict):
continue
title = " ".join(str(entry.get("title") or "").split()).strip()
if not title:
continue
published = _parse_published(entry.get("published") or entry.get("updated"))
if published is None:
# No usable date -> cannot honor the recency contract; drop.
continue
age_days = (now - published).days
# Allow a one-day grace on the future side: a paper announced later in
# the same UTC day yields age_days == -1 (timedelta.days floors toward
# negative); dropping it as "future" would discard the freshest work.
if age_days > RECENCY_DAYS or age_days < -1:
continue
summary = " ".join(str(entry.get("summary") or "").split()).strip()
authors = _author_names(entry)
url = _alternate_url(entry)
rank_decay = max(0.3, 1.0 - (i * 0.03))
if query:
content_score = token_overlap_relevance(query, f"{title} {summary}".strip())
else:
content_score = 0.5
relevance = min(1.0, 0.6 * rank_decay + 0.4 * content_score)
primary_author = authors[0] if authors else ""
author_label = primary_author
if len(authors) > 1:
author_label = f"{primary_author} et al."
items.append(
{
"id": str(entry.get("id") or url or f"AX{i + 1}"),
"title": title,
"url": url,
"summary": summary,
"author": author_label,
"authors": authors,
"date": published.date().isoformat(),
"engagement": {},
"relevance": round(relevance, 2),
"why_relevant": (
f"arXiv paper ({primary_author}, {published.date().isoformat()})"
if primary_author
else f"arXiv paper ({published.date().isoformat()})"
),
}
)
return items
skills/last30days/scripts/lib/backends.py
+607
View File
@@ -0,0 +1,607 @@
"""Backend-chain descriptors with predicted selection (doctor, R4).
Chained sources declare their routing here ONCE — imported from the
definitions ``lib/env.py`` already owns (chain order, pin var names) — and
``resolve()`` turns side-effect-free probes into a truthful prediction of
what the next run will do.
Two resolution modes:
- ``alternative`` (X, YouTube, web search): the pipeline tries genuinely
interchangeable backends in a declared order. Resolution probes ALL
candidates first, then picks (collect-then-pick): the first fully-usable
backend wins the "will use" prediction; otherwise the best degraded
candidate resolves with a warn tier; otherwise the source is an error
carrying the highest-priority backend's prescription. Collecting before
picking prevents an installed-but-unauthenticated preferred backend from
shadowing a fully working fallback.
- ``conditional`` (Reddit): routing is per-query and outcome-dependent —
public keyless composite by default, ScrapeCreators backfill only when
results fall below the configured thinness floor (see the gating in
``lib/pipeline.py``). No probe can pick one winner, so resolution renders
honest conditional wording instead of an ``active_backend``. Reddit's
internal keyless lanes (rss/listing/arctic/shreddit) are sub-probe detail
inside the public composite, never chain entries.
``active_backend`` semantics: a PREDICTION — "the first backend the probes
say the next run will try" — rendered as "will use". It is not an
observation of what served a past run, and runtime failover can still
diverge mid-run (a present-but-expired paid key passes a presence probe).
Paid lanes (xai, xquik, serper, and every other API-key backend, including
ScrapeCreators) probe KEY PRESENCE ONLY: a dict lookup, never a network
call or credential spend. Binary-backed lanes reuse the U1 dependency
probe layer (``health.probe_dependency``) so a stale shim reads as BROKEN,
not available (#692).
This module observes and predicts only. It must never alter which backend
the pipeline actually uses; parity with the pipeline's pre-failover
selection is asserted in ``tests/test_backend_descriptors.py``.
"""
from __future__ import annotations
from dataclasses import dataclass
from shutil import which
from typing import Any, Callable, Dict, List, Optional, Tuple
from . import env, health, prescriptions
# Resolution modes.
MODE_ALTERNATIVE = "alternative" # probe-ordered chain, first-usable wins
MODE_CONDITIONAL = "conditional" # per-query routing; wording, never a winner
# Rollup tiers for a resolved chain (doctor maps these into its R1 table).
TIER_OK = "ok"
TIER_WARN = "warn"
TIER_ERROR = "error"
# Web search backend order. grounding.web_search's auto branch owns the
# runtime behavior (brave -> exa -> serper -> parallel -> keyless floor);
# there is no importable constant there, so this declaration is guarded by
# the grounding-auto parity test rather than an import.
WEB_BACKEND_ORDER: Tuple[str, ...] = ("brave", "exa", "serper", "parallel", "keyless")
# YouTube backend order (pipeline: yt-dlp first, ScrapeCreators search
# fallback when yt-dlp is absent or fails — see lib/pipeline.py).
YOUTUBE_BACKEND_ORDER: Tuple[str, ...] = ("yt-dlp", "scrapecreators")
# Chain-failure fixes embed the registry's CLI forms (KTD 7): the command a
# backend finding prescribes and the one doctor/quality-nudge render for the
# same failure mode come from one entry and cannot drift.
_SC_PRESCRIPTION = (
"set SCRAPECREATORS_API_KEY (free 10,000-call signup: "
f"{prescriptions.get('scrapecreators', 'key_missing').fix_cli})"
)
_X_COOKIES_PRESCRIPTION = (
"run setup with browser-cookie consent: "
f"{prescriptions.get('x', 'cookies_missing').fix_cli}"
)
@dataclass
class BackendFinding:
"""Side-effect-free probe outcome for one backend of a chained source.
``status`` uses the ``lib.health`` vocabulary (OK/DEGRADED/MISSING/
BROKEN/TIMEOUT/ERROR). ``prescription`` is the fix when non-OK.
``requires`` is the backend's requirement note for report rendering.
"""
name: str
status: str
detail: str = ""
prescription: str = ""
requires: str = ""
@property
def usable(self) -> bool:
"""Fully or partially usable (OK/DEGRADED) — eligible for selection."""
return self.status in (health.OK, health.DEGRADED)
@dataclass(frozen=True)
class BackendSpec:
"""One backend in a chain: name, probe, requirement note, paid flag.
``probe`` must be side-effect-free. When ``paid`` is True the probe is
key-presence only: no subprocess, no network, no credential spend.
"""
name: str
requires: str
probe: Callable[[Dict[str, Any]], "BackendFinding"]
paid: bool = False
@dataclass(frozen=True)
class ChainDescriptor:
"""A chained source's declared routing: backends, mode, and pin knob."""
source: str
mode: str
backends: Tuple[BackendSpec, ...]
pin_var: Optional[str] = None # env var pin (X, Reddit)
pin_flag: Optional[str] = None # CLI flag pin (web: --web-backend)
@dataclass
class BackendResolution:
"""Resolved routing for one chained source.
``active_backend`` is the will-use PREDICTION for alternative chains
and always None for conditional mode (Reddit never gets a computed
winner — ``conditional`` carries the honest wording instead).
"""
source: str
mode: str
chain: List[str]
findings: List[BackendFinding]
active_backend: Optional[str] = None
tier: str = TIER_OK
pinned: bool = False
pin: Optional[str] = None
prescription: str = ""
conditional: str = ""
@property
def summary(self) -> str:
"""One-line rendering: will-use prediction or conditional wording."""
if self.mode == MODE_CONDITIONAL:
return self.conditional
if self.active_backend is None:
line = f"no usable backend (chain: {' -> '.join(self.chain)})"
if self.prescription:
line += f"; fix: {self.prescription}"
return line
line = f"will use: {self.active_backend}"
if self.pinned:
line += f" (pinned via {self._pin_origin()})"
return line
def _pin_origin(self) -> str:
d = DESCRIPTORS.get(self.source)
if d is None:
return "pin"
return d.pin_var or d.pin_flag or "pin"
# ---------------------------------------------------------------------------
# Probes. All side-effect-free; paid lanes are pure dict lookups.
# ---------------------------------------------------------------------------
def _key_probe(name: str, key_var: str, requires: str, note: str = "") -> Callable:
"""Key-presence probe for a paid API lane. Never touches the network."""
def probe(config: Dict[str, Any]) -> BackendFinding:
if config.get(key_var):
return BackendFinding(
name=name,
status=health.OK,
detail=f"{key_var} present",
requires=requires,
)
prescription = note or f"set {key_var} in ~/.config/last30days/.env"
return BackendFinding(
name=name,
status=health.MISSING,
detail=f"{key_var} not set",
prescription=prescription,
requires=requires,
)
return probe
def _probe_bird(config: Dict[str, Any]) -> BackendFinding:
"""Bird = vendored X GraphQL client (node script) + browser-cookie creds.
Cookie presence is checked FIRST, mirroring ``env._x_backend_available``'s
gating (``has_bird_creds and is_bird_installed()``): without cookies bird
is unconfigured regardless of node/script state, and the fix is the
cookie-consent flow — a broken node runtime must not turn an unconfigured
backend into an error carrying a node prescription.
"""
from . import bird_x
requires = "X browser cookies (AUTH_TOKEN/CT0) + node"
if not (config.get("AUTH_TOKEN") and config.get("CT0")):
return BackendFinding(
name="bird",
status=health.MISSING,
detail="X browser cookies (AUTH_TOKEN/CT0) not configured",
prescription=_X_COOKIES_PRESCRIPTION,
requires=requires,
)
if not bird_x.is_bird_installed():
# Distinguish a missing/broken node runtime from a missing script.
node = health.probe_dependency("node")
if node.status != health.OK:
return BackendFinding(
name="bird",
status=node.status,
detail=node.detail,
prescription=node.prescription,
requires=requires,
)
return BackendFinding(
name="bird",
status=health.MISSING,
detail="vendored bird-search client not found",
prescription="reinstall the skill (npx skills add . -g -y) to restore lib/vendor/bird-search",
requires=requires,
)
node = health.probe_dependency("node")
if node.status != health.OK:
# Resolvable-but-broken node (stale shim) must not read as usable.
return BackendFinding(
name="bird",
status=node.status,
detail=node.detail,
prescription=node.prescription,
requires=requires,
)
return BackendFinding(
name="bird",
status=health.OK,
detail="browser-cookie auth (AUTH_TOKEN/CT0) configured",
requires=requires,
)
def _probe_xurl(config: Dict[str, Any]) -> BackendFinding:
"""xurl = official X API v2 CLI (OAuth2). Free lane; LOCAL-ONLY probe.
Doctor's no-network guarantee forbids the live ``xurl whoami`` check
(``xurl_x.is_available()`` — an authenticated X API call, reserved for
research time). This probe keys on local evidence instead: the binary
on PATH plus xurl's on-disk token store (~/.xurl). Stored credentials
read as OK with an explicit "not live-verified" caveat; an unreadable
token store is a typed ERROR (broken, not unconfigured).
"""
from . import xurl_x
requires = "xurl CLI installed + OAuth2 login"
if which("xurl") is None:
return BackendFinding(
name="xurl",
status=health.MISSING,
detail="xurl CLI not found on PATH",
prescription="npm install -g xurl && xurl auth oauth2 login",
requires=requires,
)
store_status, store_detail = xurl_x.stored_auth_status()
if store_status == xurl_x.AUTH_OK:
return BackendFinding(
name="xurl",
status=health.OK,
detail=(
"installed; stored OAuth2 credentials present; "
"auth not live-verified (no network)"
),
requires=requires,
)
if store_status == xurl_x.AUTH_ERROR:
return BackendFinding(
name="xurl",
status=health.ERROR,
detail=store_detail,
prescription="xurl auth oauth2 login",
requires=requires,
)
return BackendFinding(
name="xurl",
status=health.MISSING,
detail="xurl installed but not authenticated",
prescription="xurl auth oauth2 login",
requires=requires,
)
def _probe_ytdlp(config: Dict[str, Any]) -> BackendFinding:
"""yt-dlp via the U1 dependency-probe layer (missing/broken/timeout)."""
dep = health.probe_dependency("yt-dlp")
return BackendFinding(
name="yt-dlp",
status=dep.status,
detail=dep.detail,
prescription=dep.prescription,
requires="yt-dlp on the agent-subprocess PATH",
)
def _probe_web_keyless(config: Dict[str, Any]) -> BackendFinding:
"""The keyless web-search floor: works keyless, but degraded quality."""
requires = "no key; suppressed on native-search hosts"
if env.keyless_web_allowed(config):
return BackendFinding(
name="keyless",
status=health.DEGRADED,
detail="keyless search floor (no paid key; lower quality)",
requires=requires,
)
return BackendFinding(
name="keyless",
status=health.MISSING,
detail="keyless floor suppressed: host has native web search",
prescription="",
requires=requires,
)
def _probe_reddit_public(config: Dict[str, Any]) -> BackendFinding:
"""Public keyless Reddit composite; internal lanes are sub-probe detail."""
return BackendFinding(
name="public",
status=health.OK,
detail="public keyless composite (lanes: rss, listing, arctic, shreddit)",
requires="none (public endpoints)",
)
# ---------------------------------------------------------------------------
# Registry: routing declared once, from env.py's definitions where they exist.
# ---------------------------------------------------------------------------
_X_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
"xai": _key_probe("xai", "XAI_API_KEY", "XAI_API_KEY (xAI/Grok live search)"),
"bird": _probe_bird,
"xurl": _probe_xurl,
"xquik": _key_probe("xquik", "XQUIK_API_KEY", "XQUIK_API_KEY (xquik.com)"),
}
_X_PAID = {"xai", "xquik"}
_WEB_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
"brave": _key_probe("brave", "BRAVE_API_KEY", "BRAVE_API_KEY"),
"exa": _key_probe("exa", "EXA_API_KEY", "EXA_API_KEY"),
"serper": _key_probe("serper", "SERPER_API_KEY", "SERPER_API_KEY"),
"parallel": _key_probe("parallel", "PARALLEL_API_KEY", "PARALLEL_API_KEY"),
"keyless": _probe_web_keyless,
}
_WEB_KEYED = {"brave", "exa", "serper", "parallel"}
_SC_SPEC = BackendSpec(
name="scrapecreators",
requires="SCRAPECREATORS_API_KEY",
probe=_key_probe(
"scrapecreators", "SCRAPECREATORS_API_KEY", "SCRAPECREATORS_API_KEY",
note=_SC_PRESCRIPTION,
),
paid=True,
)
DESCRIPTORS: Dict[str, ChainDescriptor] = {
# X: chain order and pin var imported from env.py (single source of truth).
"x": ChainDescriptor(
source="x",
mode=MODE_ALTERNATIVE,
backends=tuple(
BackendSpec(
name=name,
requires={
"xai": "XAI_API_KEY (xAI/Grok live search)",
"bird": "X browser cookies (AUTH_TOKEN/CT0) + node",
"xurl": "xurl CLI installed + OAuth2 login",
"xquik": "XQUIK_API_KEY (xquik.com)",
}[name],
probe=_X_PROBES[name],
paid=name in _X_PAID,
)
for name in env.X_BACKEND_ORDER
),
pin_var=env.X_BACKEND_PIN_VAR,
),
"youtube": ChainDescriptor(
source="youtube",
mode=MODE_ALTERNATIVE,
backends=(
BackendSpec(
name="yt-dlp",
requires="yt-dlp on the agent-subprocess PATH",
probe=_probe_ytdlp,
),
_SC_SPEC,
),
pin_var=None, # no YouTube pin knob exists
),
"web": ChainDescriptor(
source="web",
mode=MODE_ALTERNATIVE,
backends=tuple(
BackendSpec(
name=name,
requires=(f"{name.upper()}_API_KEY" if name in _WEB_KEYED
else "no key; suppressed on native-search hosts"),
probe=_WEB_PROBES[name],
paid=name in _WEB_KEYED,
)
for name in WEB_BACKEND_ORDER
),
pin_var=None, # pinned per-run via --web-backend, not an env var
pin_flag="--web-backend",
),
"reddit": ChainDescriptor(
source="reddit",
mode=MODE_CONDITIONAL,
backends=(
BackendSpec(
name="public",
requires="none (public endpoints)",
probe=_probe_reddit_public,
),
_SC_SPEC,
),
pin_var=env.REDDIT_BACKEND_PIN_VAR,
),
}
def get_descriptor(source: str) -> ChainDescriptor:
"""Return the declared routing descriptor for ``source`` (KeyError if none)."""
return DESCRIPTORS[source]
# ---------------------------------------------------------------------------
# Resolution
# ---------------------------------------------------------------------------
def resolve(
source: str,
config: Dict[str, Any],
pin: Optional[str] = None,
) -> BackendResolution:
"""Resolve a chained source's routing into a truthful prediction.
``pin`` is an explicit per-run pin (the ``--web-backend`` flag); it
takes precedence over the descriptor's env pin var. ``"auto"``/None
mean unpinned. Probing is side-effect-free and collect-then-pick.
Time budget: backends are probed sequentially, so a chain's budget is
ADDITIVE across its backends — each binary-backed probe is bounded by
``health.PROBE_TIMEOUT`` and paid/key lanes are dict lookups that cost
nothing, giving a worst case of roughly (binary probes in the chain) x
``health.PROBE_TIMEOUT``. Deliberately no intra-chain concurrency:
probes are memoized per process and the worst case only occurs when
multiple binaries are simultaneously hung.
"""
descriptor = get_descriptor(source)
findings = [
_run_probe(spec, config) for spec in descriptor.backends
]
if descriptor.mode == MODE_CONDITIONAL:
return _resolve_conditional(descriptor, config, findings)
return _resolve_alternative(descriptor, config, findings, pin)
def _run_probe(spec: BackendSpec, config: Dict[str, Any]) -> BackendFinding:
"""Run one probe, isolating failures so one bad probe can't blank a chain."""
try:
finding = spec.probe(config)
except Exception as exc: # a probe bug must not take the report down
finding = BackendFinding(
name=spec.name,
status=health.ERROR,
detail=f"probe failed: {type(exc).__name__}: {exc}",
requires=spec.requires,
)
if not finding.requires:
finding.requires = spec.requires
return finding
def _resolve_alternative(
descriptor: ChainDescriptor,
config: Dict[str, Any],
findings: List[BackendFinding],
pin: Optional[str],
) -> BackendResolution:
names = [spec.name for spec in descriptor.backends]
by_name = {f.name: f for f in findings}
res = BackendResolution(
source=descriptor.source,
mode=MODE_ALTERNATIVE,
chain=list(names),
findings=findings,
)
pin_name: Optional[str] = None
if pin and pin not in ("auto", "none") and pin in by_name:
pin_name = pin
elif descriptor.pin_var:
raw = (config.get(descriptor.pin_var) or "").lower()
if raw in by_name:
pin_name = raw
if pin_name:
# A pin forces a single backend (no failover) — mirror
# env.x_backend_chain's pin semantics exactly.
res.pinned = True
res.pin = pin_name
finding = by_name[pin_name]
if finding.status == health.OK:
res.active_backend = pin_name
res.tier = TIER_OK
elif finding.status == health.DEGRADED:
res.active_backend = pin_name
res.tier = TIER_WARN
else:
res.tier = TIER_ERROR
res.prescription = finding.prescription or (
f"unpin {descriptor.pin_var or descriptor.pin_flag} or fix {pin_name}"
)
return res
# Collect-then-pick: first fully-usable wins; else best degraded; else
# error carrying the highest-priority backend's prescription.
for finding in findings:
if finding.status == health.OK:
res.active_backend = finding.name
res.tier = TIER_OK
return res
for finding in findings:
if finding.status == health.DEGRADED:
res.active_backend = finding.name
res.tier = TIER_WARN
return res
res.tier = TIER_ERROR
res.prescription = findings[0].prescription if findings else ""
return res
def _reddit_sc_min_items(config: Dict[str, Any]) -> int:
"""The thinness floor, parsed exactly as the pipeline parses it
(lib/pipeline.py reddit fetch: int(... or 0), malformed -> 0)."""
try:
return int(config.get(env.REDDIT_SC_MIN_ITEMS_VAR) or 0)
except (TypeError, ValueError):
return 0
def _resolve_conditional(
descriptor: ChainDescriptor,
config: Dict[str, Any],
findings: List[BackendFinding],
) -> BackendResolution:
"""Reddit: render the real per-query semantics, never a computed winner."""
res = BackendResolution(
source=descriptor.source,
mode=MODE_CONDITIONAL,
chain=[spec.name for spec in descriptor.backends],
findings=findings,
active_backend=None, # conditional mode never picks a winner
tier=TIER_OK, # the public keyless composite is always reachable
)
has_key = bool(config.get("SCRAPECREATORS_API_KEY"))
raw_pin = (config.get(descriptor.pin_var) or "").lower() if descriptor.pin_var else ""
pinned_sc = has_key and raw_pin == "scrapecreators"
floor = _reddit_sc_min_items(config)
if pinned_sc:
res.pinned = True
res.pin = "scrapecreators"
res.conditional = (
f"ScrapeCreators primary (pinned via {descriptor.pin_var}); "
"public keyless composite fallback"
)
return res
if has_key:
if floor > 0:
backfill = (
f"ScrapeCreators backfill when results fall below the "
f"{floor}-item floor"
)
else:
backfill = "ScrapeCreators backfill when the free path returns nothing"
res.conditional = f"public keyless composite (default); {backfill}"
return res
res.conditional = "public keyless composite (default); no ScrapeCreators key for backfill"
if raw_pin == "scrapecreators":
# The pipeline ignores the pin without a key; say so honestly.
res.conditional += (
f" ({descriptor.pin_var} pin ignored: SCRAPECREATORS_API_KEY not set)"
)
return res
skills/last30days/scripts/lib/bird_x.py
+652
View File
@@ -0,0 +1,652 @@
"""Bird X search client for the v3.0.0 last30days pipeline.
Uses a vendored subset of @steipete/bird v0.8.0 (MIT License) to search X
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js.
See scripts/lib/vendor/bird-search/package.json for authoritative version.
"""
import json
import os
import shutil
import sys
import time
from pathlib import Path
from . import env, health, http, log, subproc
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from .relevance import token_overlap_relevance as _compute_relevance
# How many times to retry the bird-search subprocess when stdout is non-JSON
# (typically an HTML anti-bot interstitial from Twitter's edge).
MAX_JSON_DECODE_RETRIES = 2
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
def _leading_mentions(text: str) -> list:
"""Leading-run @mention parse, shared with other X-shaped sources (xquik).
Thin wrapper over ``query.leading_mentions`` so bird and xquik share one
implementation; kept here for existing call sites and tests.
"""
from .query import leading_mentions
return leading_mentions(text)
def _first_of(*values):
"""Return first value that is not None."""
for v in values:
if v is not None:
return v
return None
# Path to the vendored bird-search wrapper
_BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
# Depth configurations: number of results to request
DEPTH_CONFIG = {
"quick": 12,
"default": 30,
"deep": 60,
}
# Module-level credentials injected from .env config
_credentials: Dict[str, str] = {}
def set_credentials(auth_token: Optional[str], ct0: Optional[str]):
"""Inject AUTH_TOKEN/CT0 from .env config so Node subprocesses can use them."""
if auth_token:
_credentials['AUTH_TOKEN'] = auth_token
if ct0:
_credentials['CT0'] = ct0
def _has_injected_credentials() -> bool:
"""Return True when both X session cookies were injected from config."""
return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0'))
def _has_process_credentials() -> bool:
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
return bool(env.read_secret_env("AUTH_TOKEN") and env.read_secret_env("CT0"))
def _subprocess_env() -> Dict[str, str]:
"""Build env dict for Node subprocesses, merging injected credentials."""
env = os.environ.copy()
env.update(_credentials)
# Hard-disable browser-cookie fallback so normal pipeline runs never hit
# Safari/Chrome Keychain prompts during source detection or search.
env["BIRD_DISABLE_BROWSER_COOKIES"] = "1"
return env
def _log(msg: str):
log.source_log("Bird", msg, tty_only=False)
def classify_run_failure(detail: str) -> str:
"""Map Bird's subprocess-only failure shapes to run outcome states."""
text = detail.lower()
if any(marker in text for marker in ("interstitial", "non-json", "invalid json")):
return health.SCHEMA_DRIFT
if any(
marker in text
for marker in ("cookie expired", "expired cookie", "unauthorized", "forbidden", "login required")
):
return health.AUTH_FAILED
return http.classify_failure(message=detail)
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for X search.
X search is literal keyword AND matching — all words must appear.
Aggressively strip question/meta/research words to keep only the
core product/concept name (max 5 words).
"""
from .query import extract_core_subject
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
def is_bird_installed() -> bool:
"""Check if vendored Bird search module is available.
Returns:
True if bird-search.mjs exists and Node.js is in PATH.
"""
if not _BIRD_SEARCH_MJS.exists():
return False
return shutil.which("node") is not None
def is_bird_authenticated() -> Optional[str]:
"""Check if explicit X credentials are available.
Returns:
Auth source string if authenticated, None otherwise.
"""
if not is_bird_installed():
return None
if _has_injected_credentials():
return "env AUTH_TOKEN"
if _has_process_credentials():
return "env AUTH_TOKEN"
return None
_probe_cache: Optional[Optional[bool]] = "unset" # "unset" | True | False | None
def probe_works(timeout: int = 8) -> Optional[bool]:
"""Cheap runtime check that X auth actually returns data.
Returns True when a 1-result probe comes back without an error, False on a
clear failure (auth error / generic search failure), and None when the
result is inconclusive (network timeout) so callers can fail open and keep
the static credential-presence status rather than reporting a false-down.
Cached per process so repeated diagnose calls don't re-probe.
"""
global _probe_cache
if _probe_cache != "unset":
return _probe_cache # type: ignore[return-value]
if not (_has_injected_credentials() or _has_process_credentials()):
_probe_cache = False
return False
from datetime import datetime, timedelta, timezone
since = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%d")
# @x (the platform's own account) posts frequently, so a no-error response
# means auth works even if this particular window is quiet.
resp = _run_bird_search(f"from:x since:{since}", count=1, timeout=timeout)
if isinstance(resp, dict) and resp.get("error"):
err = str(resp.get("error")).lower()
if "timed out" in err or "timeout" in err:
_probe_cache = None # inconclusive — don't downgrade on a transient timeout
return None
_probe_cache = False
return False
_probe_cache = True
return True
def check_npm_available() -> bool:
"""Check if npm is available (kept for API compatibility).
Returns:
True if 'npm' command is available in PATH, False otherwise.
"""
return shutil.which("npm") is not None
def install_bird() -> Tuple[bool, str]:
"""No-op. Bird search is vendored in v3.0.0, no installation needed.
Returns:
Tuple of (success, message).
"""
if is_bird_installed():
return True, "Bird search is bundled with /last30days v3.0.0 - no installation needed."
if not shutil.which("node"):
return False, "Node.js 22+ is required for X search. Install Node.js first."
return False, f"Vendored bird-search.mjs not found at {_BIRD_SEARCH_MJS}"
def get_bird_status() -> Dict[str, Any]:
"""Get comprehensive Bird search status.
Returns:
Dict with keys: installed, authenticated, username, can_install
"""
installed = is_bird_installed()
auth_source = is_bird_authenticated() if installed else None
return {
"installed": installed,
"authenticated": auth_source is not None,
"username": auth_source, # Now returns auth source (e.g., "Safari", "env AUTH_TOKEN")
"can_install": True, # Always vendored in v3.0.0
}
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
"""Invoke the vendored bird-search.mjs subprocess once.
Returns (result, error_dict). If error_dict is non-None, treat it as the
final result and do not retry — those errors are terminal (timeout,
spawn failure). If error_dict is None, the subprocess ran to completion
and `result` is the SubprocResult; the caller decides whether to retry
based on the result.stdout content.
"""
cmd = [
"node", str(_BIRD_SEARCH_MJS),
query,
"--count", str(count),
"--json",
]
pid_holder: list[int] = []
def _register(pid: int) -> None:
pid_holder.append(pid)
try:
from last30days import register_child_pid
register_child_pid(pid)
except ImportError:
pass
try:
result = subproc.run_with_timeout(
cmd,
timeout=timeout,
env=_subprocess_env(),
on_pid=_register,
)
except subproc.SubprocTimeout:
return None, {"error": f"Search timed out after {timeout}s", "items": []}
except Exception as e:
return None, {"error": str(e), "items": []}
finally:
if pid_holder:
try:
from last30days import unregister_child_pid
unregister_child_pid(pid_holder[0])
except Exception:
pass
return result, None
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
"""Run a search using the vendored bird-search.mjs module.
Retries the subprocess on JSON-decode failure (typically a Twitter
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
errors (subprocess timeout, non-zero return code) are returned
immediately without retry.
Args:
query: Full search query string (including since: filter)
count: Number of results to request
timeout: Timeout in seconds (per attempt)
Returns:
Raw Bird JSON response or error dict.
"""
last_decode_error: Optional[str] = None
for attempt in range(MAX_JSON_DECODE_RETRIES):
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
if terminal_error is not None:
return terminal_error
if result.returncode != 0:
error = result.stderr.strip() or "Bird search failed"
return {"error": error, "items": []}
output = result.stdout.strip()
if not output:
return {"items": []}
try:
parsed = json.loads(output)
except json.JSONDecodeError as e:
# Twitter's edge sometimes serves an HTML anti-bot interstitial
# in place of JSON. Tag the failure shape so it's distinguishable
# from "no results" in logs, then retry the subprocess.
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
attempt_num = attempt + 1
log_msg = (
f"Bird search returned non-JSON stdout "
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
f"first 80 chars: {output[:80]!r})"
)
last_decode_error = str(e)
if attempt_num < MAX_JSON_DECODE_RETRIES:
log.source_log(
"X/bird",
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
tty_only=False,
)
time.sleep(JSON_DECODE_RETRY_DELAY)
continue
log.source_log("X/bird", log_msg, tty_only=False)
return {
"error": (
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
f"(likely Twitter anti-bot interstitial): {e}"
),
"items": [],
}
if isinstance(parsed, list):
return {"items": parsed}
return parsed
# Defensive fallthrough — loop should always return above.
return {
"error": f"Bird search exhausted retries: {last_decode_error}",
"items": [],
}
def search_x(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> Dict[str, Any]:
"""Search X using Bird CLI with automatic retry on 0 results.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD) - unused but kept for API compatibility
depth: Research depth - "quick", "default", or "deep"
Returns:
Raw Bird JSON response or error dict.
"""
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
timeout = 30 if depth == "quick" else 45 if depth == "default" else 60
# Extract core subject - X search is literal, not semantic
core_topic = _extract_core_subject(topic)
query = f"{core_topic} since:{from_date}"
_log(f"Searching: {query}")
response = _run_bird_search(query, count, timeout)
# Check if we got results
items = parse_bird_response(response, query=core_topic)
# Retry with OR groups for multi-word queries (X supports OR operator)
core_words = core_topic.split()
if not items and len(core_words) >= 2:
from .query import extract_compound_terms
compounds = extract_compound_terms(topic)
if compounds:
# Build OR-group query: ("multi-agent" OR "agent simulation") since:DATE
or_parts = ' OR '.join(f'"{t}"' for t in compounds[:3])
_log(f"0 results for '{core_topic}', retrying with OR groups: {or_parts}")
query = f"({or_parts}) since:{from_date}"
response = _run_bird_search(query, count, timeout)
items = parse_bird_response(response, query=core_topic)
# Retry with fewer keywords if still 0 results and query has 3+ words
if not items and len(core_words) > 2:
shorter = ' '.join(core_words[:2])
_log(f"0 results for '{core_topic}', retrying with '{shorter}'")
query = f"{shorter} since:{from_date}"
response = _run_bird_search(query, count, timeout)
items = parse_bird_response(response, query=core_topic)
# Last-chance retry: use strongest remaining token (often the product name)
if not items and core_words:
low_signal = {
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
'best', 'top', 'latest', 'new', 'plugin', 'plugins',
'skill', 'skills', 'tool', 'tools',
}
candidates = [w for w in core_words if w not in low_signal]
if candidates:
# Keep an entity anchor (the first distinctive topic token) in the
# retry so it can't collapse to a bare generic token like "compound"
# and flood the X pool with off-topic noise. Add the strongest
# (longest) distinctive token when it differs from the anchor;
# otherwise query the anchor alone. Better to return 0 than to
# over-broaden to an unanchored generic term.
anchor = candidates[0]
strongest = max(candidates, key=len)
retry_terms = anchor if strongest == anchor else f"{anchor} {strongest}"
_log(f"0 results for '{core_topic}', retrying anchored on '{retry_terms}'")
query = f"{retry_terms} since:{from_date}"
response = _run_bird_search(query, count, timeout)
return response
def search_handles(
handles: List[str],
topic: Optional[str],
from_date: str,
count_per: int = 5,
) -> List[Dict[str, Any]]:
"""Search specific X handles for topic-related content.
Pulls each handle's actual timeline via `from:handle since:` — the FROM
lane (tweets BY the person), engagement-weighted downstream. The topic is
used for relevance RANKING, never AND'd into the query: X search is literal,
so `from:handle <their name>` only matched tweets where they wrote their own
name and returned ~0. Used in Phase 2 after entity extraction.
Args:
handles: List of X handles to search (without @)
topic: Search topic — used for relevance ranking only, not the query
from_date: Start date (YYYY-MM-DD)
count_per: Results to request per handle
Returns:
List of raw item dicts (same format as parse_bird_response output).
"""
core_topic = _extract_core_subject(topic) if topic else None
def _search_one_handle(handle: str) -> List[Dict[str, Any]]:
handle = handle.lstrip("@")
# Always unfiltered: pull the timeline, rank by topic relevance below.
query = f"from:{handle} since:{from_date}"
cmd = [
"node", str(_BIRD_SEARCH_MJS),
query,
"--count", str(count_per),
"--json",
]
try:
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
except subproc.SubprocTimeout:
_log(f"Handle search timed out for @{handle}")
return []
except OSError as e:
_log(f"Handle search error for @{handle}: {e}")
return []
if result.returncode != 0:
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
return []
output = result.stdout.strip()
if not output:
return []
try:
response = json.loads(output)
except json.JSONDecodeError:
_log(f"Invalid JSON from handle search for @{handle}")
return []
items = parse_bird_response(response, query=core_topic)
# Log on success/empty too (not only on failure): a silent handle search
# made the from: query look like it never ran and caused wrong diagnoses.
_log(f"Searching: {query} -> {len(items)} results")
return items
from concurrent.futures import ThreadPoolExecutor, as_completed
all_items: List[Dict[str, Any]] = []
with ThreadPoolExecutor(max_workers=min(5, len(handles))) as executor:
futures = {executor.submit(_search_one_handle, h): h for h in handles}
for future in as_completed(futures):
all_items.extend(future.result())
return all_items
def search_mentions(
handles: List[str],
from_date: str,
count_per: int = 5,
) -> List[Dict[str, Any]]:
"""Search for tweets ABOUT/TO each handle — the mention lane.
Queries `@handle since:` (tweets that mention the account) and excludes the
handle's OWN tweets (those belong to the FROM lane via search_handles), so
this surfaces what OTHERS are saying about the person. Engagement-weighted
downstream; deduped against the FROM lane by URL at normalize time.
Args:
handles: List of X handles (without @)
from_date: Start date (YYYY-MM-DD)
count_per: Results to request per handle
Returns:
List of raw item dicts (same format as parse_bird_response output).
"""
def _search_one(handle: str) -> List[Dict[str, Any]]:
handle = handle.lstrip("@")
query = f"@{handle} since:{from_date}"
cmd = [
"node", str(_BIRD_SEARCH_MJS),
query,
"--count", str(count_per),
"--json",
]
try:
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
except subproc.SubprocTimeout:
_log(f"Mention search timed out for @{handle}")
return []
except OSError as e:
_log(f"Mention search error for @{handle}: {e}")
return []
if result.returncode != 0:
_log(f"Mention search failed for @{handle}: {result.stderr.strip()}")
return []
output = result.stdout.strip()
if not output:
return []
try:
response = json.loads(output)
except json.JSONDecodeError:
_log(f"Invalid JSON from mention search for @{handle}")
return []
items = parse_bird_response(response, query=None)
# ABOUT lane = OTHERS mentioning the handle. Drop the handle's own tweets
# (the FROM lane already covers those); identify by the status URL author.
hl = handle.lower()
# The Bird API may return either x.com or twitter.com permalinks, so
# match both when excluding the handle's own tweets.
def _is_own(url):
u = (url or "").lower()
return f"x.com/{hl}/status" in u or f"twitter.com/{hl}/status" in u
about = [it for it in items if not _is_own(it.get("url"))]
_log(f"Searching: {query} -> {len(about)} mentions")
return about
from concurrent.futures import ThreadPoolExecutor, as_completed
all_items: List[Dict[str, Any]] = []
with ThreadPoolExecutor(max_workers=min(5, len(handles))) as executor:
futures = {executor.submit(_search_one, h): h for h in handles}
for future in as_completed(futures):
all_items.extend(future.result())
return all_items
def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
"""Parse Bird response to match xai_x output format.
Args:
response: Raw Bird JSON response
query: Original search query for relevance scoring
Returns:
List of normalized item dicts matching xai_x.parse_x_response() format.
"""
items = []
# Check for errors
if "error" in response and response["error"]:
_log(f"Bird error: {response['error']}")
return items
# Bird returns a list of tweets directly or under a key
raw_items = response if isinstance(response, list) else response.get("items", response.get("tweets", []))
if not isinstance(raw_items, list):
return items
for i, tweet in enumerate(raw_items):
if not isinstance(tweet, dict):
continue
# Extract URL - Bird uses permanent_url or we construct from id
url = tweet.get("permanent_url") or tweet.get("url", "")
if not url and tweet.get("id"):
# Try different field structures Bird might use
author = tweet.get("author", {}) or tweet.get("user", {})
screen_name = author.get("username") or author.get("screen_name", "")
if screen_name:
url = f"https://x.com/{screen_name}/status/{tweet['id']}"
if not url:
continue
# Parse date from created_at/createdAt (e.g., "Wed Jan 15 14:30:00 +0000 2026")
date = None
created_at = tweet.get("createdAt") or tweet.get("created_at", "")
if created_at:
try:
# Try ISO format first (e.g., "2026-02-03T22:33:32Z")
# Check for ISO date separator, not just "T" (which appears in "Tue")
if len(created_at) > 10 and created_at[10] == "T":
dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
else:
# Twitter format: "Wed Jan 15 14:30:00 +0000 2026"
dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
date = dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
# Extract user info (Bird uses author.username, older format uses user.screen_name)
author = tweet.get("author", {}) or tweet.get("user", {})
author_handle = author.get("username") or author.get("screen_name", "") or tweet.get("author_handle", "")
# Build engagement dict (Bird uses camelCase: likeCount, retweetCount, etc.)
engagement = {
"likes": _first_of(tweet.get("likeCount"), tweet.get("like_count"), tweet.get("favorite_count")),
"reposts": _first_of(tweet.get("retweetCount"), tweet.get("retweet_count")),
"replies": _first_of(tweet.get("replyCount"), tweet.get("reply_count")),
"quotes": _first_of(tweet.get("quoteCount"), tweet.get("quote_count")),
}
# Convert to int where possible
for key in engagement:
if engagement[key] is not None:
try:
engagement[key] = int(engagement[key])
except (ValueError, TypeError):
engagement[key] = None
# Build normalized item
text = str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500]
item = {
"id": f"X{i+1}",
"text": text,
"url": url,
"author_handle": author_handle.lstrip("@"),
# Leading @mentions parsed from the post text identify who a reply is
# directed at (X replies open with the target handle(s)). Used by the
# interaction-signal classifier in rerank.
"mentioned_handles": _leading_mentions(text),
"date": date,
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
"why_relevant": "", # Bird doesn't provide relevance explanations
"relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
}
items.append(item)
return items
skills/last30days/scripts/lib/bluesky.py
+323
View File
@@ -0,0 +1,323 @@
"""Bluesky search via AT Protocol (requires app password).
Uses bsky.social for auth and api.bsky.app for post search (the canonical
authenticated AppView). The previous default `public.api.bsky.app` is the
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
regardless of auth header (verified 2026-05-04). Override the search host
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
The createSession endpoint accepts main-account passwords too, but they're
bad hygiene (no scope, can't revoke individually).
"""
import math
import os
import re
import sys
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from . import http, log
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
.env file. The project's env.py loads .env into config but not into
os.environ, so check both — same hybrid pattern as last30days.py for
LAST30DAYS_STORE.
Hardens user-supplied host values against three common mis-configurations:
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
we log a warning and fall back to the default rather than building an
invalid URL with an opaque downstream error.
"""
config = config or {}
raw = (
os.environ.get("BSKY_SEARCH_HOST")
or config.get("BSKY_SEARCH_HOST")
or _DEFAULT_BSKY_SEARCH_HOST
)
host = raw.strip().rstrip("/")
# Strip embedded scheme so users who paste full URLs do not break the f-string.
for prefix in ("https://", "http://"):
if host.lower().startswith(prefix):
host = host[len(prefix):]
break
if not host or "/" in host or " " in host:
# Embedded path or whitespace remains — don't trust it. Default + log.
if raw != _DEFAULT_BSKY_SEARCH_HOST:
_log(
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
)
host = _DEFAULT_BSKY_SEARCH_HOST
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
# with three hyphens at fixed positions).
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
def _validate_app_password_format(value) -> bool:
"""Return True if value matches Bluesky's 19-char app-password format.
False for non-strings (None, int, list) so callers passing config dict
values directly don't crash. Detect-but-not-gate: the createSession
endpoint also accepts main-account passwords, so failing this check is
a hygiene smell, not a hard error.
"""
if not isinstance(value, str):
return False
return bool(_APP_PASSWORD_RE.fullmatch(value))
DEPTH_CONFIG = {
"quick": 15,
"default": 30,
"deep": 60,
}
# Module-level token cache (valid for the lifetime of a single research run)
_cached_token: Optional[str] = None
_token_created_at: float = 0.0
_session_error: Optional[str] = None
_TOKEN_MAX_AGE_SECONDS = 5400 # 90 minutes (conservative, tokens last ~2 hours)
def _log(msg: str):
log.source_log("Bluesky", msg, tty_only=False)
def _create_session(handle: str, app_password: str) -> Optional[str]:
"""Create an AT Protocol session and return the access token.
Args:
handle: Bluesky handle (e.g. user.bsky.social)
app_password: App password from bsky.app/settings/app-passwords
Returns:
Access JWT string, or None on failure. Sets _session_error on failure.
"""
global _cached_token, _token_created_at, _session_error
if _cached_token and (time.monotonic() - _token_created_at < _TOKEN_MAX_AGE_SECONDS):
return _cached_token
if _cached_token:
_log("Session token expired, re-authenticating")
_cached_token = None
_token_created_at = 0.0
try:
response = http.request(
"POST",
BSKY_SESSION_URL,
json_data={"identifier": handle, "password": app_password},
timeout=15,
)
token = response.get("accessJwt")
if token:
_cached_token = token
_token_created_at = time.monotonic()
_session_error = None
_log("Session created successfully")
return token
_log("No accessJwt in session response")
_session_error = "No accessJwt in session response"
return None
except http.HTTPError as e:
if e.status_code == 403 and e.body and "cloudflare" in e.body.lower():
_session_error = "Cloudflare blocked the request (403 Forbidden). This is a network-level block, not an auth issue. Try a different network or VPN."
elif e.status_code == 401:
_session_error = "Invalid credentials (401 Unauthorized). Check BSKY_HANDLE and BSKY_APP_PASSWORD."
else:
_session_error = f"Session request failed: {e}"
_log(f"Session creation failed: {_session_error}")
return None
except Exception as e:
_session_error = f"Session request failed: {type(e).__name__}: {e}"
_log(f"Session creation failed: {_session_error}")
return None
def _reset_session_cache() -> None:
global _cached_token, _token_created_at, _session_error
_cached_token = None
_token_created_at = 0.0
_session_error = None
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Bluesky search."""
from .query import SOCIAL_NOISE, extract_core_subject
return extract_core_subject(topic, noise=SOCIAL_NOISE)
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from Bluesky post to YYYY-MM-DD.
AT Protocol uses ISO 8601 format in indexedAt and createdAt fields.
"""
for key in ("indexedAt", "createdAt"):
val = item.get(key)
if val and isinstance(val, str):
try:
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
return None
def search_bluesky(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Search Bluesky via AT Protocol API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
config: Config dict with BSKY_HANDLE and BSKY_APP_PASSWORD
Returns:
Dict with 'posts' list from AT Protocol response.
"""
config = config or {}
handle = config.get("BSKY_HANDLE", "")
app_password = config.get("BSKY_APP_PASSWORD", "")
if not handle or not app_password:
return {"posts": [], "error": "Bluesky credentials not configured"}
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
# form. createSession accepts main-account passwords too — but main
# passwords have no scope (full account access), can't be revoked
# individually, and rotating them breaks every service that holds them.
# We warn but do not gate, matching the project's detect-don't-block
# philosophy elsewhere.
if not _validate_app_password_format(app_password):
_log(
"BSKY_APP_PASSWORD does not look like an app password "
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
"account password — those work but are bad hygiene. Generate "
"an app password at https://bsky.app/settings/app-passwords"
)
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching for '{core_topic}' (depth={depth}, limit={count})")
from urllib.parse import urlencode
params = {
"q": core_topic,
"limit": str(min(count, 100)),
"sort": "top",
}
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
token = _create_session(handle, app_password)
if not token:
error_msg = _session_error or "Bluesky session creation failed (unknown error)"
return None, error_msg
try:
response = http.request(
"GET", url,
headers={"Authorization": f"Bearer {token}"},
timeout=30,
)
return response, None
except http.HTTPError as e:
_log(f"Search failed: {e}")
if e.status_code == 401:
_reset_session_cache()
return None, "refresh"
if e.status_code == 403 and e.body and "cloudflare" in e.body.lower():
return None, "Bluesky search blocked by Cloudflare (403). This is a network-level block - try a different network or VPN."
return None, f"Bluesky search failed: {e}"
except Exception as e:
_log(f"Search failed: {e}")
return None, f"Bluesky search failed: {type(e).__name__}: {e}"
response, error_msg = _auth_and_search()
if error_msg == "refresh":
_log("Session expired; recreating token and retrying once")
response, error_msg = _auth_and_search()
if error_msg:
return {"posts": [], "error": error_msg}
if response is None:
return {"posts": [], "error": "Bluesky search failed (unknown error)"}
posts = response.get("posts", [])
_log(f"Found {len(posts)} posts")
return response
def parse_bluesky_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse AT Protocol response into normalized item dicts.
Returns:
List of item dicts ready for normalization.
"""
posts = response.get("posts", [])
items = []
for i, post in enumerate(posts):
record = post.get("record") or {}
text = record.get("text") or ""
author = post.get("author") or {}
handle = author.get("handle") or ""
display_name = author.get("displayName") or handle
# Post URI -> URL
# URI format: at://did:plc:xxx/app.bsky.feed.post/rkey
uri = post.get("uri") or ""
rkey = uri.rsplit("/", 1)[-1] if uri else ""
url = f"https://bsky.app/profile/{handle}/post/{rkey}" if handle and rkey else ""
likes = post.get("likeCount") or 0
reposts = post.get("repostCount") or 0
replies = post.get("replyCount") or 0
quotes = post.get("quoteCount") or 0
date_str = _parse_date(post) or _parse_date(record)
# Relevance: position-based (AT Protocol sorts by relevance with sort=top)
rank_score = max(0.3, 1.0 - (i * 0.02))
engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
items.append({
"handle": handle,
"display_name": display_name,
"text": text,
"url": url,
"date": date_str,
"engagement": {
"likes": likes,
"reposts": reposts,
"replies": replies,
"quotes": quotes,
},
"relevance": round(relevance, 2),
"why_relevant": f"Bluesky: @{handle}: {text[:60]}" if text else f"Bluesky: {handle}",
})
return items
skills/last30days/scripts/lib/categories.py
+289
View File
@@ -0,0 +1,289 @@
"""Category-peer subreddit map for Step 0.55 community resolution.
When a topic is a product in a known category (AI image generation, AI coding
agents, SaaS screen recording, etc.), brand-specific subreddits returned by
WebSearch are insufficient: cross-product technique discussion lives in
category-peer subs. This module classifies a topic into a category by matching
compound-term patterns against the lowercased topic string, then returns the
priority-ordered peer subreddit list for that category.
The map is intentionally small, curated, and code-reviewed. Adding a new
category is a code change; there is no user-editable override surface.
False-positive guard: every pattern is either a multi-word compound (e.g.
"image generation", "text to image") or a domain-specific single word
(e.g. "midjourney", "stablediffusion"). Bare common nouns like "image",
"ai", or "model" are never used as patterns.
First-match-wins: categories are evaluated in declared order. Entries are
sorted from most-specific to least-specific so narrower categories claim a
topic before broader ones. For example, `ai_image_generation` appears
before `ai_chat_model` so "gpt image 2" matches the image-gen category.
"""
from __future__ import annotations
import re
from typing import List, Optional, TypedDict
class _CategoryEntry(TypedDict):
patterns: List[str]
peer_subs: List[str]
CATEGORY_PEERS: dict[str, _CategoryEntry] = {
"ai_image_generation": {
"patterns": [
"image generation",
"image gen",
"text to image",
"text-to-image",
"gpt image",
"gpt-image",
"nano banana",
"midjourney",
"stable diffusion",
"stablediffusion",
"dall-e",
"dalle",
"flux.1",
"flux schnell",
"imagen",
"seedance",
"ideogram",
"recraft",
],
"peer_subs": [
"StableDiffusion",
"midjourney",
"dalle2",
"aiArt",
"PromptEngineering",
"MediaSynthesis",
],
},
"ai_video_generation": {
"patterns": [
"video generation",
"text to video",
"text-to-video",
"sora",
"veo 3",
"veo3",
"runway gen",
"kling",
"pika labs",
"luma dream machine",
"hailuo",
],
"peer_subs": [
"aivideo",
"StableDiffusion",
"runwayml",
"singularity",
"MediaSynthesis",
],
},
"ai_music_generation": {
"patterns": [
"music generation",
"ai music",
"suno",
"udio",
"riffusion",
"stable audio",
],
"peer_subs": [
"SunoAI",
"udiomusic",
"aimusic",
"artificial",
],
},
"ai_coding_agent": {
"patterns": [
"claude code",
"cursor ide",
"github copilot",
"windsurf",
"aider",
"cline",
"openclaw",
"hermes agent",
"continue.dev",
"codeium",
"sweep ai",
"devin ai",
"coding agent",
"coding assistant",
],
"peer_subs": [
"ChatGPTCoding",
"LocalLLaMA",
"singularity",
"PromptEngineering",
],
},
"ai_agent_framework": {
"patterns": [
"ai agent",
"ai agents",
"agent framework",
"agentic framework",
"langchain",
"langgraph",
"crewai",
"autogen",
"llamaindex",
"dspy",
"smolagents",
],
"peer_subs": [
"LangChain",
"LocalLLaMA",
"AI_Agents",
"MachineLearning",
],
},
"ai_chat_model": {
"patterns": [
"gpt-5",
"gpt-4",
"claude opus",
"claude sonnet",
"claude haiku",
"gemini pro",
"gemini flash",
"llama 3",
"llama 4",
"deepseek",
"qwen",
"mistral large",
"grok",
],
"peer_subs": [
"LocalLLaMA",
"ChatGPT",
"ClaudeAI",
"singularity",
"artificial",
],
},
"saas_screen_recording": {
"patterns": [
"screen recording",
"screen recorder",
"loom video",
"tella screen",
"vidyard",
"screen capture tool",
],
"peer_subs": [
"SaaS",
"screenrecording",
"productivity",
"Entrepreneur",
],
},
"saas_productivity": {
"patterns": [
"notion app",
"obsidian plugin",
"obsidian app",
"linear app",
"asana",
"clickup",
"productivity app",
],
"peer_subs": [
"productivity",
"SaaS",
"ObsidianMD",
"Notion",
],
},
"prediction_markets": {
"patterns": [
"polymarket",
"kalshi",
"prediction market",
"event contracts",
"manifold markets",
],
"peer_subs": [
"Polymarket",
"Kalshi",
"predictionmarkets",
],
},
"crypto_defi": {
"patterns": [
"defi protocol",
"yield farming",
"liquidity pool",
"stablecoin",
"ethereum layer",
"layer 2",
"l2 rollup",
],
"peer_subs": [
"defi",
"ethfinance",
"CryptoCurrency",
"ethereum",
],
},
"dev_tool_cli": {
"patterns": [
"cli tool",
"command line tool",
"terminal app",
"dev tool",
],
"peer_subs": [
"commandline",
"programming",
"webdev",
],
},
}
def detect_category(topic: Optional[str]) -> Optional[str]:
"""Classify a topic into a known category by compound-term match.
Returns the category id (e.g. "ai_image_generation") or None if no
category's patterns match. Matching is case-insensitive substring over
the lowercased topic. Declaration order wins (first-match-wins), so the
map is ordered from most-specific to least-specific.
A None or empty topic returns None. Classification never raises on
normal string inputs; callers do not need to wrap in try/except for
typical paths, though defensive callers may.
"""
if not topic:
return None
lowered = topic.lower()
for category_id, entry in CATEGORY_PEERS.items():
for pattern in entry["patterns"]:
# Word-boundary match: "ai agent" must not fire on "Dubai agents"
# or "Thai agents". Substring matching classified those as
# ai_agent_framework and routed discovery to LangChain subreddits.
if re.search(rf"(?<![a-z0-9]){re.escape(pattern)}(?![a-z0-9])", lowered):
return category_id
return None
def peer_subs_for(category_id: Optional[str]) -> List[str]:
"""Return the priority-ordered peer subreddit list for a category.
Returns an empty list for None or unknown category ids. The returned
list is a fresh copy; callers may safely mutate it.
"""
if not category_id:
return []
entry = CATEGORY_PEERS.get(category_id)
if not entry:
return []
return list(entry["peer_subs"])
skills/last30days/scripts/lib/chrome_cookies.py
+414
View File
@@ -0,0 +1,414 @@
"""Chromium-family cookie extraction for macOS.
Extracts cookies from Chromium-based browser SQLite databases using only
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
dependencies.
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
Every Chromium-based browser (Chrome, Brave, Edge, Vivaldi, Opera, Arc,
Chromium) shares the same algorithm; only the profile directory and Keychain
service name differ, so they all run through the same decryption core.
This is NOT affected by Windows App-Bound Encryption (v20).
"""
import hashlib
import logging
import os
import shutil
import sqlite3
import subprocess
import tempfile
from pathlib import Path
from typing import Optional
logger = logging.getLogger(__name__)
def _lock_temp_cookie_copy(path: str) -> None:
"""Restrict copied cookie DB temp files to the current user on POSIX."""
if os.name == "nt":
return
Path(path).chmod(0o600)
# Cookie DB locations on macOS
_APP_SUPPORT = Path.home() / "Library" / "Application Support"
CHROME_BASE_DIR = _APP_SUPPORT / "Google" / "Chrome"
# Kept for backward compatibility; resolution now goes through the profile
# finder (which also handles the modern Network/Cookies layout).
CHROME_COOKIES_DB = CHROME_BASE_DIR / "Default" / "Cookies"
BRAVE_BASE_DIR = _APP_SUPPORT / "BraveSoftware" / "Brave-Browser"
# Other Chromium-based browsers, keyed by FROM_BROWSER name. Each maps to
# (profile base directory, macOS Keychain service name). Chrome and Brave keep
# their dedicated helpers below for backward compatibility; everything here is
# resolved generically by extract_chromium_browser_cookies_macos(). Keychain
# service names follow Chromium's "<Browser> Safe Storage" convention.
CHROMIUM_BROWSER_PROFILES: dict[str, tuple[Path, str]] = {
"edge": (_APP_SUPPORT / "Microsoft Edge", "Microsoft Edge Safe Storage"),
"vivaldi": (_APP_SUPPORT / "Vivaldi", "Vivaldi Safe Storage"),
"opera": (_APP_SUPPORT / "com.operasoftware.Opera", "Opera Safe Storage"),
"arc": (_APP_SUPPORT / "Arc" / "User Data", "Arc Safe Storage"),
"chromium": (_APP_SUPPORT / "Chromium", "Chromium Safe Storage"),
}
# Chromium v10 encryption constants (shared by Chrome and Brave)
CHROME_SALT = b"saltysalt"
CHROME_PBKDF2_ITERATIONS = 1003
CHROME_KEY_LENGTH = 16
# IV is 16 space characters (0x20)
CHROME_IV_HEX = "20" * 16
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
Calls `security find-generic-password` which may trigger a system dialog
on first access.
Returns the raw passphrase bytes, or None on failure.
"""
try:
result = subprocess.run(
["security", "find-generic-password", "-w", "-s", service_name],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode != 0:
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
return None
passphrase = result.stdout.strip()
if not passphrase:
logger.info("%s Keychain returned empty passphrase", service_name)
return None
return passphrase.encode("utf-8")
except FileNotFoundError:
logger.info("'security' command not found — not on macOS?")
return None
except subprocess.TimeoutExpired:
logger.info("%s Keychain access timed out", service_name)
return None
except Exception as e:
logger.info("Failed to get %s encryption key: %s", service_name, e)
return None
def _get_chrome_encryption_key() -> Optional[bytes]:
return _get_chromium_encryption_key("Chrome Safe Storage")
def _derive_aes_key(passphrase: bytes) -> bytes:
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
return hashlib.pbkdf2_hmac(
"sha1",
passphrase,
CHROME_SALT,
CHROME_PBKDF2_ITERATIONS,
dklen=CHROME_KEY_LENGTH,
)
def _decrypt_v10_value(encrypted_value: bytes, aes_key: bytes, db_version: int) -> Optional[str]:
"""Decrypt a Chrome v10-encrypted cookie value.
Uses system openssl CLI for AES-128-CBC decryption (zero pip deps).
For Chrome 130+ (db_version >= 24), strips 32-byte SHA-256 prefix after decryption.
Returns decrypted string or None on failure.
"""
# Strip the 'v10' prefix
ciphertext = encrypted_value[3:]
if not ciphertext:
return None
hex_key = aes_key.hex()
try:
result = subprocess.run(
[
"openssl", "enc", "-aes-128-cbc", "-d",
"-K", hex_key,
"-iv", CHROME_IV_HEX,
"-nopad",
],
input=ciphertext,
capture_output=True,
timeout=5,
)
if result.returncode != 0:
logger.debug("openssl decryption failed: %s", result.stderr.decode(errors="replace").strip())
return None
decrypted = result.stdout
if not decrypted:
return None
# Remove PKCS7 padding
decrypted = _remove_pkcs7_padding(decrypted)
if decrypted is None:
return None
# Chrome 130+ (db version >= 24): strip 32-byte SHA-256 prefix
if db_version >= 24 and len(decrypted) > 32:
decrypted = decrypted[32:]
return decrypted.decode("utf-8", errors="replace")
except FileNotFoundError:
logger.info("openssl not found — cannot decrypt Chrome cookies")
return None
except subprocess.TimeoutExpired:
logger.info("openssl decryption timed out")
return None
except Exception as e:
logger.debug("Chrome cookie decryption error: %s", e)
return None
def _remove_pkcs7_padding(data: bytes) -> Optional[bytes]:
"""Remove PKCS7 padding from decrypted data.
The last byte indicates the number of padding bytes added.
All padding bytes must have the same value.
Returns unpadded data or None if padding is invalid.
"""
if not data:
return None
pad_len = data[-1]
if pad_len < 1 or pad_len > 16:
return None
# Verify all padding bytes match
if data[-pad_len:] != bytes([pad_len]) * pad_len:
return None
return data[:-pad_len]
def _get_db_version(cursor: sqlite3.Cursor) -> int:
"""Get Chrome cookie database version from the meta table.
Returns 0 if meta table doesn't exist or version can't be read.
"""
try:
cursor.execute("SELECT value FROM meta WHERE key = 'version'")
row = cursor.fetchone()
if row:
return int(row[0])
except Exception:
pass
return 0
def _extract_chromium_cookies_macos(
db_path: Path,
keychain_service: str,
domain: str,
cookie_names: list[str],
) -> Optional[dict[str, str]]:
"""Extract cookies from any Chromium-based browser on macOS.
Copies the locked Cookies database to a temp file, reads specified cookies,
and decrypts v10-encrypted values using the Keychain-stored key.
Args:
db_path: Path to the browser's Cookies SQLite file.
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
cookie_names: List of cookie names to extract.
Returns:
Dict mapping cookie name to decrypted value, or None on failure.
Only includes cookies that were successfully found and decrypted.
"""
if not db_path.exists():
logger.info("%s cookies database not found at %s", keychain_service, db_path)
return None
# Copy DB to temp file (browser locks the original while running)
tmp_fd = None
tmp_path = None
try:
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
shutil.copy2(str(db_path), tmp_path)
_lock_temp_cookie_copy(tmp_path)
except Exception as e:
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
if tmp_path:
try:
Path(tmp_path).unlink(missing_ok=True)
except Exception:
pass
return None
finally:
if tmp_fd is not None:
import os
os.close(tmp_fd)
try:
conn = sqlite3.connect(tmp_path)
cursor = conn.cursor()
db_version = _get_db_version(cursor)
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
placeholders = ",".join("?" for _ in cookie_names)
query = (
f"SELECT name, value, encrypted_value FROM cookies "
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
)
params = [f"%{domain}"] + list(cookie_names)
cursor.execute(query, params)
results: dict[str, str] = {}
aes_key = None
key_fetched = False
for name, value, encrypted_value in cursor.fetchall():
if value:
results[name] = value
continue
if encrypted_value and encrypted_value[:3] == b"v10":
if not key_fetched:
# Fetch the Keychain key lazily — only once we actually have
# an encrypted cookie to decrypt. This avoids a macOS
# Keychain prompt for browsers that don't hold the requested
# cookie, which matters for FROM_BROWSER=auto across several
# installed Chromium browsers.
passphrase = _get_chromium_encryption_key(keychain_service)
aes_key = _derive_aes_key(passphrase) if passphrase else None
key_fetched = True
if aes_key is None:
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
continue
decrypted = _decrypt_v10_value(encrypted_value, aes_key, db_version)
if decrypted:
results[name] = decrypted
else:
logger.debug("Failed to decrypt cookie %s", name)
elif encrypted_value:
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
conn.close()
if not results:
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
return None
return results
except sqlite3.Error as e:
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
return None
except Exception as e:
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
return None
finally:
try:
Path(tmp_path).unlink(missing_ok=True)
except Exception:
pass
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
"""Extract cookies from Chrome on macOS.
Resolves the cookie DB through the shared profile finder so Chrome gets the
same modern ``Default/Network/Cookies`` (Chromium >= 96) and legacy
``Default/Cookies`` probing as the rest of the Chromium family.
"""
db_path = _find_chromium_cookies_db(CHROME_BASE_DIR)
if db_path is None:
logger.info("Chrome cookies database not found under %s", CHROME_BASE_DIR)
return None
return _extract_chromium_cookies_macos(
db_path, "Chrome Safe Storage", domain, cookie_names
)
def _profile_cookie_db(profile_dir: Path) -> Optional[Path]:
"""Return the Cookies DB inside a profile dir, or None.
Prefers the modern ``Network/Cookies`` location (Chromium >= 96 moved the
cookie store into a per-profile ``Network/`` subdirectory) and falls back
to the legacy flat ``Cookies`` file. Different browsers and versions use
different layouts, so both are probed.
"""
for rel in ("Network/Cookies", "Cookies"):
candidate = profile_dir / rel
if candidate.exists():
return candidate
return None
def _find_chromium_cookies_db(base_dir: Path) -> Optional[Path]:
"""Find a Chromium-based browser's Cookies database under base_dir.
Checks the Default profile first, then the base dir itself (Opera's flat
layout), then numbered "Profile N" directories by most-recently-modified.
Each location is probed for both the modern ``Network/Cookies`` and legacy
``Cookies`` paths (see _profile_cookie_db). Chromium browsers create extra
profiles as "Profile 1", "Profile 2", etc. alongside Default; the most
recently used one is the likeliest to hold current cookies. Lexicographic
sort would visit "Profile 10" before "Profile 2", which can return the
wrong profile, so we sort by mtime.
"""
found = _profile_cookie_db(base_dir / "Default")
if found:
return found
found = _profile_cookie_db(base_dir)
if found:
return found
try:
candidates = [
child for child in base_dir.iterdir()
if child.is_dir() and child.name.startswith("Profile ")
]
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
found = _profile_cookie_db(child)
if found:
return found
except OSError:
pass
return None
def _find_brave_cookies_db() -> Optional[Path]:
"""Find Brave's Cookies database on macOS (Default, then Profile N)."""
return _find_chromium_cookies_db(BRAVE_BASE_DIR)
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
"""Extract cookies from Brave on macOS.
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
path and Keychain service name differ.
"""
db_path = _find_brave_cookies_db()
if db_path is None:
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
return None
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
def extract_chromium_browser_cookies_macos(
browser: str, domain: str, cookie_names: list[str]
) -> Optional[dict[str, str]]:
"""Extract cookies from a registry-defined Chromium browser on macOS.
Covers every browser in CHROMIUM_BROWSER_PROFILES (Edge, Vivaldi, Opera,
Arc, Chromium). They all reuse Chrome's v10 AES-128-CBC encryption; only
the profile directory and Keychain service name differ.
"""
spec = CHROMIUM_BROWSER_PROFILES.get(browser)
if spec is None:
logger.debug("Unknown Chromium browser: %s", browser)
return None
base_dir, keychain_service = spec
db_path = _find_chromium_cookies_db(base_dir)
if db_path is None:
logger.info("%s cookies database not found under %s", keychain_service, base_dir)
return None
return _extract_chromium_cookies_macos(db_path, keychain_service, domain, cookie_names)

Some files were not shown because too many files have changed in this diff Show More