name: Publish to PyPI # SECURITY: Only triggered via repository_dispatch from release.yml # (after security gate passes). We intentionally do NOT support # workflow_dispatch because that would bypass security checks. # To re-publish, trigger a new release through release.yml. # # REPRODUCIBILITY: repository_dispatch runs on default-branch HEAD at # dispatch time — NOT the release commit. The `release` env approval is # human-paced, so main can move between the release commit and the # dispatch (on v1.9.0 it was 4 PRs ahead, and PyPI shipped code absent # from the tag/Docker image). Both build jobs therefore check out # `client_payload.sha` — the exact commit every gate ran against — and # fail closed if the dispatcher didn't pin it. on: repository_dispatch: types: [publish-pypi] permissions: {} # Minimal top-level for OSSF Scorecard Token-Permissions jobs: # Build frontend if package.json exists (isolated, no secrets) build-frontend: runs-on: ubuntu-latest outputs: has-frontend: ${{ steps.check.outputs.has-frontend }} container: # Node 24 to match `package.json`'s `engines: { node: ">=24.0.0" }`. # Was previously node:20 — npm could resolve dependencies that target # APIs missing on 20 and the wheel-building publish path could ship # frontend assets that break at runtime on the Node-24 Docker image. image: node:24-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f # node:24-alpine # Note: Network is needed for npm ci to work, but no secrets are available options: --user 1001 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Validate pinned release commit # POSIX sh (this job's container is Alpine — no bash). Fail closed: # without a full 40-hex SHA, checkout would silently fall back to # default-branch HEAD and reintroduce the drift this pin prevents. # The length test matters: grep -E matches PER LINE, so a crafted # multiline value ("<40-hex>\n") would pass the regex alone; # ${#VAR} counts bytes in POSIX sh, so -eq 40 also excludes any # embedded newline. Ancestry (sha actually on main) is enforced in # build-package — this container has no python3/gh; this job holds # no secrets and its output only reaches PyPI through build-package, # which checks out the same validated sha. env: RELEASE_SHA: ${{ github.event.client_payload.sha }} run: | if [ "${#RELEASE_SHA}" -ne 40 ] || ! printf '%s' "$RELEASE_SHA" | grep -qE '^[0-9a-f]{40}$'; then echo "::error::client_payload.sha is missing or not a full commit SHA — refusing to build from default-branch HEAD. The dispatcher (release.yml trigger-pypi) must pin the release commit." exit 1 fi echo "Building from pinned release commit $RELEASE_SHA" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.client_payload.sha }} persist-credentials: false - name: Check for frontend assets id: check run: | if [ -f "package.json" ]; then echo "has-frontend=true" >> "$GITHUB_OUTPUT" echo "Found package.json - will build frontend" else echo "has-frontend=false" >> "$GITHUB_OUTPUT" echo "ERROR: No package.json found - frontend build is required for PyPI releases" exit 1 fi - name: Build frontend if: steps.check.outputs.has-frontend == 'true' run: | echo "=== Frontend Build Process Starting ===" echo "Current directory: $(pwd)" echo "Node version: $(node --version)" echo "NPM version: $(npm --version)" # Install dependencies from root package.json echo "" echo "📦 Installing dependencies..." # Use npm ci for reproducible builds from lockfile npm ci --ignore-scripts --no-audit --no-fund echo "✅ Dependencies installed" # Show pre-build directory structure echo "" echo "📁 Pre-build directory structure:" echo "Contents of src/local_deep_research/web/static/:" find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "Directory not found" # Build with Vite (outputs to src/local_deep_research/web/static/dist) echo "" echo "🔨 Running Vite build..." npm run build echo "✅ Vite build completed" # Show post-build directory structure echo "" echo "📁 Post-build directory structure:" echo "Contents of src/local_deep_research/web/static/:" find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null echo "" echo "Contents of src/local_deep_research/web/static/dist/:" find src/local_deep_research/web/static/dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null echo "" echo "Looking for manifest.json:" find src/local_deep_research/web/static/dist -name "manifest.json" -type f 2>/dev/null || echo "No manifest.json found" # Verify the build completed successfully if [ ! -f "src/local_deep_research/web/static/dist/.vite/manifest.json" ]; then echo "❌ ERROR: Build failed - manifest.json not created at expected location" echo "Expected location: src/local_deep_research/web/static/dist/.vite/manifest.json" echo "Actual dist contents:" find src/local_deep_research/web/static/dist -type f | head -20 exit 1 fi echo "✅ Found manifest.json at: src/local_deep_research/web/static/dist/.vite/manifest.json" echo "Manifest contents (first 10 lines):" head -10 src/local_deep_research/web/static/dist/.vite/manifest.json # Create build marker echo "" echo "📝 Creating build marker..." echo "{\"status\":\"complete\",\"built\":\"$(date -Iseconds)\"}" > src/local_deep_research/web/static/.frontend-built echo "✅ Build marker created" - name: Upload frontend artifacts if: steps.check.outputs.has-frontend == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: frontend-assets path: src/local_deep_research/web/static/ retention-days: 1 include-hidden-files: true # Ensure .vite directory is included # Build Python package (isolated, no PyPI access) build-package: needs: build-frontend runs-on: ubuntu-latest container: image: python:3.12-slim@sha256:971f04b358cf483ec445a8d388fb55267451f080d90fb136c8e69684a02a9604 # python:3.12-slim permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Validate pinned release commit # Same fail-closed shape guard as build-frontend (length test: see # comment there), plus the ANCESTRY check that only this job can # run (python3 in the container): the pinned sha must be reachable # from main. Anyone able to send repository_dispatch (repo write / # the PAT holder) chooses the payload — without this check they # could point the build at ANY commit object in the repo network # (scratch branches, fork-PR heads) that never passed review or the # release gates. release.yml's own build job verifies its sha is on # main, but a forged dispatch bypasses release.yml entirely; the # `release` environment approval on the publish job is then the # only barrier, and an approver has no reason to suspect a # legitimate-looking pending deployment. Enforced here (not in # build-frontend) because every byte that reaches PyPI flows # through this job's checkout. env: RELEASE_SHA: ${{ github.event.client_payload.sha }} GH_TOKEN: ${{ github.token }} run: | if [ "${#RELEASE_SHA}" -ne 40 ] || ! printf '%s' "$RELEASE_SHA" | grep -qE '^[0-9a-f]{40}$'; then echo "::error::client_payload.sha is missing or not a full commit SHA — refusing to build from default-branch HEAD. The dispatcher (release.yml trigger-pypi) must pin the release commit." exit 1 fi python3 - <<'PYEOF' import json import os import sys import urllib.request sha = os.environ["RELEASE_SHA"] repo = os.environ["GITHUB_REPOSITORY"] req = urllib.request.Request( f"https://api.github.com/repos/{repo}/compare/{sha}...heads/main", headers={ "Authorization": f"Bearer {os.environ['GH_TOKEN']}", "Accept": "application/vnd.github+json", }, ) try: with urllib.request.urlopen(req) as resp: status = json.load(resp)["status"] except urllib.error.HTTPError as exc: # 404 = sha unknown to the repo network. Fail closed either way. print(f"::error::compare API rejected sha {sha}: HTTP {exc.code}") sys.exit(1) # base=sha, head=main: main being "ahead" of (or "identical" to) # the sha means the sha is an ancestor of main. "diverged" or # "behind" means it is not. if status not in ("ahead", "identical"): print( f"::error::pinned sha {sha} is not an ancestor of main " f"(compare status: {status}) — refusing to build unreviewed code" ) sys.exit(1) print(f"Pinned sha is on main (compare status: {status})") PYEOF echo "Building from pinned release commit $RELEASE_SHA" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.client_payload.sha }} persist-credentials: false - name: Install build dependencies run: | apt-get update && apt-get install -y libsqlcipher-dev build-essential pip install pdm==2.26.2 - name: Download frontend artifacts if: needs.build-frontend.outputs.has-frontend == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true with: name: frontend-assets path: src/local_deep_research/web/static/ - name: Verify frontend build artifacts run: | echo "=== Frontend Artifacts Verification ===" echo "Current directory: $(pwd)" echo "" echo "📁 Checking downloaded artifacts structure:" echo "Contents of src/:" find src/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/ directory not found" echo "" echo "Contents of src/local_deep_research/:" find src/local_deep_research/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/ not found" echo "" echo "Contents of src/local_deep_research/web/:" find src/local_deep_research/web/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/web/ not found" echo "" echo "Contents of src/local_deep_research/web/static/:" find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/web/static/ not found" # Check if dist directory exists echo "" echo "🔍 Checking for dist directory..." if [ ! -d "src/local_deep_research/web/static/dist" ]; then echo "❌ ERROR: Frontend dist directory not found!" echo "Expected location: src/local_deep_research/web/static/dist" echo "Actual structure of static directory:" find src/local_deep_research/web/static -type d 2>/dev/null | head -20 echo "The frontend build artifacts are missing." exit 1 fi echo "✅ Found dist directory at: src/local_deep_research/web/static/dist" # Show dist directory structure echo "" echo "📁 Dist directory structure:" echo "Contents of dist/:" find src/local_deep_research/web/static/dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null echo "" echo "All files in dist (recursive):" find src/local_deep_research/web/static/dist -type f | head -20 # Check for critical files echo "" echo "🔍 Checking for manifest.json..." # Check both possible locations for manifest.json MANIFEST_PATH="" if [ -f "src/local_deep_research/web/static/dist/.vite/manifest.json" ]; then MANIFEST_PATH="src/local_deep_research/web/static/dist/.vite/manifest.json" elif [ -f "src/local_deep_research/web/static/dist/manifest.json" ]; then MANIFEST_PATH="src/local_deep_research/web/static/dist/manifest.json" fi if [ -z "$MANIFEST_PATH" ]; then echo "❌ ERROR: Vite manifest.json not found!" echo "Checked locations:" echo " - src/local_deep_research/web/static/dist/.vite/manifest.json" echo " - src/local_deep_research/web/static/dist/manifest.json" echo "Looking for any manifest.json files:" find src/local_deep_research/web/static -name "manifest.json" -type f 2>/dev/null || echo "None found" echo "Checking if .vite directory exists:" find src/local_deep_research/web/static/dist/ -maxdepth 1 -name ".*" -type d || echo "No hidden directories found" echo "" echo "This is likely due to artifact transfer losing the hidden .vite directory." echo "The include-hidden-files option should fix this." echo "The frontend build appears to be incomplete - cannot continue." exit 1 fi echo "✅ Found manifest.json at: $MANIFEST_PATH" echo "Manifest size: $(wc -c < "$MANIFEST_PATH") bytes" # Check for JS files echo "" echo "🔍 Checking for JavaScript files..." JS_COUNT=$(find src/local_deep_research/web/static/dist -name "*.js" -type f | wc -l) if [ "$JS_COUNT" -eq 0 ]; then echo "❌ ERROR: No JavaScript files found!" echo "Expected JS files in dist/" echo "The frontend build appears to be incomplete." exit 1 fi echo "✅ Found $JS_COUNT JavaScript file(s)" echo "JS files:" find src/local_deep_research/web/static/dist -name "*.js" -type f | head -5 # Check for CSS files echo "" echo "🔍 Checking for CSS files..." CSS_COUNT=$(find src/local_deep_research/web/static/dist -name "*.css" -type f | wc -l) if [ "$CSS_COUNT" -eq 0 ]; then echo "❌ ERROR: No CSS files found!" echo "Expected CSS files in dist/" echo "The frontend build appears to be incomplete." exit 1 fi echo "✅ Found $CSS_COUNT CSS file(s)" echo "CSS files:" find src/local_deep_research/web/static/dist -name "*.css" -type f | head -5 # Check for build marker echo "" echo "🔍 Checking for build marker..." if [ ! -f "src/local_deep_research/web/static/.frontend-built" ]; then echo "⚠️ WARNING: Frontend build marker not found (non-critical)" else echo "✅ Found build marker" echo "Build marker contents:" cat src/local_deep_research/web/static/.frontend-built fi echo "" echo "=== ✅ Frontend build artifacts verified successfully ===" echo "Summary:" echo " - Dist directory: ✓" echo " - Manifest.json: ✓" echo " - JS files: $JS_COUNT file(s)" echo " - CSS files: $CSS_COUNT file(s)" - name: Build Python package run: | echo "=== Python Package Build ===" echo "Current directory: $(pwd)" echo "" echo "📦 Building Python package with PDM..." pdm build echo "" echo "✅ Package build completed" echo "" echo "📁 Package contents:" find dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null echo "" echo "Package sizes:" du -h dist/* - name: Upload package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: python-dist path: dist/ retention-days: 1 # Publish (ONLY job with PyPI access) publish: needs: build-package runs-on: ubuntu-latest environment: release permissions: id-token: write attestations: write # Required for generating attestations steps: - name: Harden Runner uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: python-dist path: dist/ - name: List packages run: | echo "=== Downloaded Package Artifacts ===" echo "📦 Packages in dist/:" ls -lh dist/ echo "" echo "Package details:" for file in dist/*; do echo " - $(basename "$file"): $(du -h "$file" | cut -f1)" done - name: Verify package contents run: | echo "=== Package Content Verification ===" echo "Verifying package contains frontend assets..." echo "" # Install wheel to inspect package echo "📦 Installing wheel for package inspection..." pip install wheel==0.46.2 # Find the wheel file WHEEL_FILE=$(find dist -name "*.whl" -type f 2>/dev/null | head -1) TAR_FILE=$(find dist -name "*.tar.gz" -type f 2>/dev/null | head -1) if [ -z "$WHEEL_FILE" ]; then echo "❌ ERROR: No wheel file found in dist/" echo "Found files:" find dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null exit 1 fi echo "✅ Found wheel file: $WHEEL_FILE" echo "Wheel size: $(du -h "$WHEEL_FILE" | cut -f1)" echo "" if [ -n "$TAR_FILE" ]; then echo "✅ Found source distribution: $TAR_FILE" echo "Source dist size: $(du -h "$TAR_FILE" | cut -f1)" fi echo "" echo "🔍 Inspecting wheel contents..." # Extract and check for frontend files python -m zipfile -l "$WHEEL_FILE" > wheel_contents.txt TOTAL_FILES=$(wc -l < wheel_contents.txt) echo "Total files in wheel: $TOTAL_FILES" echo "" # Count different file types echo "📊 File type breakdown:" echo " Python files: $(grep -c '\.py' wheel_contents.txt || echo 0)" echo " JavaScript files: $(grep -c '\.js' wheel_contents.txt || echo 0)" echo " CSS files: $(grep -c '\.css' wheel_contents.txt || echo 0)" echo " JSON files: $(grep -c '\.json' wheel_contents.txt || echo 0)" echo " HTML files: $(grep -c '\.html' wheel_contents.txt || echo 0)" echo "" # Check for critical frontend directories/files echo "🔍 Checking for frontend assets..." echo "Looking for dist directory..." if ! grep -q "local_deep_research/web/static/dist/" wheel_contents.txt; then echo "❌ ERROR: Frontend dist directory not found in package!" echo "Package is missing frontend build artifacts." echo "Static files found in package:" grep "static" wheel_contents.txt | head -10 || echo "No static files found" exit 1 fi echo "✅ Found dist directory" echo "" echo "Looking for JavaScript files..." JS_IN_WHEEL=$(grep -c "local_deep_research/web/static/dist.*\.js" wheel_contents.txt || echo 0) if [ "$JS_IN_WHEEL" -eq 0 ]; then echo "❌ ERROR: Frontend JS files not found in package!" echo "JavaScript files in package:" grep "\.js" wheel_contents.txt | head -10 || echo "No JS files found" exit 1 fi echo "✅ Found $JS_IN_WHEEL JavaScript file(s) in dist" echo "" echo "Looking for CSS files..." CSS_IN_WHEEL=$(grep -c "local_deep_research/web/static/dist.*\.css" wheel_contents.txt || echo 0) if [ "$CSS_IN_WHEEL" -eq 0 ]; then echo "❌ ERROR: Frontend CSS files not found in package!" echo "CSS files in package:" grep "\.css" wheel_contents.txt | head -10 || echo "No CSS files found" exit 1 fi echo "✅ Found $CSS_IN_WHEEL CSS file(s) in dist" echo "" echo "Looking for manifest.json..." if ! grep -q "local_deep_research/web/static/dist/.vite/manifest.json" wheel_contents.txt; then echo "⚠️ WARNING: manifest.json not found in package (may be okay)" else echo "✅ Found manifest.json" fi echo "" echo "=== ✅ Package verification successful ===" echo "Summary:" echo " - Wheel file: ✓" echo " - Frontend dist: ✓" echo " - JS files: $JS_IN_WHEEL file(s)" echo " - CSS files: $CSS_IN_WHEEL file(s)" echo "" echo "Sample frontend assets in package:" grep "local_deep_research/web/static/dist/" wheel_contents.txt | head -10 - name: Generate attestations uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: 'dist/*' - name: Pre-publish summary env: EVENT_NAME: ${{ github.event_name }} DISPATCH_TAG: ${{ github.event.client_payload.tag }} DISPATCH_PRERELEASE: ${{ github.event.client_payload.prerelease }} RELEASE_TAG: ${{ github.event.release.tag_name }} RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} run: | echo "=== 🚀 Ready to Publish ===" echo "" echo "📦 Package Information:" # Handle both release event and repository_dispatch if [ "$EVENT_NAME" = "repository_dispatch" ]; then echo " - Trigger: repository_dispatch" echo " - Tag: $DISPATCH_TAG" echo " - Prerelease: $DISPATCH_PRERELEASE" else echo " - Trigger: release" echo " - Tag: $RELEASE_TAG" echo " - Prerelease: $RELEASE_PRERELEASE" fi echo "" echo "📁 Package files to publish:" ls -lh dist/ echo "" echo "✅ All checks passed - proceeding with publication" - name: Publish to Test PyPI if: (github.event_name == 'release' && github.event.release.prerelease == true) || (github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease == true) uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true # Don't fail if version already exists - name: Publish to PyPI if: (github.event_name == 'release' && github.event.release.prerelease != true) || (github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease != true) uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: skip-existing: true # Don't fail if version already exists # Verify the published package is installable (skipped for prereleases which go to Test PyPI) verify-publish: needs: publish if: ${{ !(github.event_name == 'release' && github.event.release.prerelease == true) && !(github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease == true) }} runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Determine published version id: version env: EVENT_NAME: ${{ github.event_name }} PAYLOAD_TAG: ${{ github.event.client_payload.tag }} RELEASE_TAG: ${{ github.event.release.tag_name }} run: | if [ "$EVENT_NAME" = "repository_dispatch" ]; then TAG="$PAYLOAD_TAG" else TAG="$RELEASE_TAG" fi # Strip leading 'v' if present VERSION="${TAG#v}" echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Published version: $VERSION" - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y libsqlcipher-dev - name: Wait for PyPI index to update run: | echo "Waiting 60 seconds for PyPI index propagation..." sleep 60 - name: Verify package installation run: | python3 -m venv /tmp/verify-env . /tmp/verify-env/bin/activate VERSION="${{ steps.version.outputs.version }}" echo "Installing local-deep-research==${VERSION}..." # Retry up to 5 times with increasing delay for PyPI propagation for i in 1 2 3 4 5; do if pip install "local-deep-research==${VERSION}" 2>&1; then echo "Installation successful on attempt $i" break fi if [ "$i" -lt 5 ]; then echo "Attempt $i failed, waiting $((i * 30))s..." sleep $((i * 30)) else echo "::error::Failed to install local-deep-research==${VERSION} after 5 attempts" exit 1 fi done # Verify import works python3 -c "import local_deep_research; print(f'Successfully imported v{local_deep_research.__version__}')"