learningcircuit--local-deep-research/.github/workflows/backwards-compatibility.yml

name: Backwards Compatibility

on:
  workflow_call:  # Allows this workflow to be called by other workflows
  pull_request:
    types: [opened, synchronize, reopened]
    branches: [main]
    paths:
      # Core encryption and database management
      - 'src/local_deep_research/database/encrypted_db.py'
      - 'src/local_deep_research/database/sqlcipher_*.py'
      - 'src/local_deep_research/database/auth_db.py'
      - 'src/local_deep_research/database/session_*.py'
      # Database initialization and migrations
      - 'src/local_deep_research/database/initialize.py'
      - 'src/local_deep_research/database/library_init.py'
      - 'src/local_deep_research/database/thread_local_session.py'
      # Database models (schema changes)
      - 'src/local_deep_research/database/models/*.py'
      # Encryption settings configuration
      - 'src/local_deep_research/settings/env_definitions/db_config.py'
      # Dependencies
      - 'pyproject.toml'
      - 'pdm.lock'
      # Alembic migration infrastructure
      - 'src/local_deep_research/database/alembic_runner.py'
      - 'src/local_deep_research/database/migrations/**'
      # Test files (ensure tests themselves are valid)
      - 'tests/performance/database/test_backwards_compatibility.py'
      - 'tests/database/test_encryption_constants.py'
      - 'tests/database/test_alembic_migrations.py'
      - 'tests/database/test_migration_0003_indexes.py'
      - 'tests/database/test_migration_0004_app_settings.py'
      - 'tests/performance/database/scripts/create_compat_db.py'
  push:
    branches: [main]
  release:
    types: [created]
  schedule:
    # Weekly on Sundays at 2am UTC - catch dependency drift
    - cron: '0 2 * * 0'
  workflow_dispatch:

# No concurrency group — intentionally omitted.
# This workflow triggers on both pull_request and workflow_call (from
# ci-gate.yml / release-gate.yml). A shared concurrency key would cause
# direct PR runs and workflow_call runs to cancel each other mid-flight.
# See #3554 (reverted in #3599) for context.

# Top-level permissions set to minimum (OSSF Scorecard Token-Permissions)
permissions: {}

jobs:
  # Fast test - runs on every trigger including PRs.
  # Provides quick feedback on encryption constant stability.
  encryption-constants:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    name: Verify Encryption Constants
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up PDM
        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
        with:
          python-version: '3.12'

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force

      - name: Install dependencies
        run: pdm install --dev

      - name: Run encryption constants tests
        run: |
          pdm run pytest tests/database/test_encryption_constants.py -v --tb=short

  # Slow test - full PyPI version compatibility.
  # Skipped on PRs because it installs previous PyPI versions which is slow
  # and the encryption-constants job already provides fast PR feedback.
  pypi-compatibility:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    name: PyPI Version Compatibility
    permissions:
      contents: read
    # Run on main push, releases, schedule, manual dispatch, and workflow_call (release gate)
    if: |
      github.event_name == 'push' ||
      github.event_name == 'release' ||
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'workflow_call'

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up PDM
        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
        with:
          python-version: '3.12'

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force

      - name: Install test dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest

      - name: Get previous PyPI version
        id: prev-version
        run: |
          # Get list of available versions from PyPI
          VERSIONS_OUTPUT=$(pip index versions local-deep-research 2>&1) || true

          if echo "$VERSIONS_OUTPUT" | grep -q "Available versions:"; then
            VERSIONS=$(echo "$VERSIONS_OUTPUT" | grep "Available versions:" | cut -d: -f2 | tr ',' '\n' | tr -d ' ')
          else
            echo "::warning::Could not fetch versions from PyPI: ${VERSIONS_OUTPUT}"
            echo "skip=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          if [ -z "$VERSIONS" ]; then
            echo "::warning::No versions found on PyPI"
            echo "skip=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Get current and previous versions
          CURRENT=$(echo "$VERSIONS" | head -1)
          PREVIOUS=$(echo "$VERSIONS" | head -2 | tail -1)

          if [ "$CURRENT" = "$PREVIOUS" ] || [ -z "$PREVIOUS" ]; then
            echo "No previous version available"
            echo "skip=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          {
            echo "current=$CURRENT"
            echo "previous=$PREVIOUS"
            echo "skip=false"
          } >> "$GITHUB_OUTPUT"
          echo "Previous version: $PREVIOUS, Current: $CURRENT"

      - name: Create database with previous version
        id: create-db
        if: steps.prev-version.outputs.skip != 'true'
        run: |
          # Create isolated venv for previous version
          python -m venv prev_venv
          source prev_venv/bin/activate

          # Install previous version — may fail if the published package
          # has broken dependency metadata (e.g. requests version conflict).
          # Upgrade pip for reliable dependency resolution in fresh venvs.
          python -m pip install --upgrade pip
          if ! pip install "local-deep-research==${{ steps.prev-version.outputs.previous }}" 2>&1; then
            echo "::warning::Previous version ${{ steps.prev-version.outputs.previous }} has unresolvable dependencies — skipping compat test"
            echo "install_failed=true" >> "$GITHUB_OUTPUT"
            deactivate
            exit 0
          fi
          echo "install_failed=false" >> "$GITHUB_OUTPUT"

          # Create test database
          mkdir -p test_db
          python tests/performance/database/scripts/create_compat_db.py test_db compat_user "TestPass123!"

          deactivate

          echo "Database created with version ${{ steps.prev-version.outputs.previous }}"
          ls -la test_db/

      - name: Install current version
        if: steps.prev-version.outputs.skip != 'true' && steps.create-db.outputs.install_failed != 'true'
        run: pdm install --dev

      - name: Test opening database with current version
        if: steps.prev-version.outputs.skip != 'true' && steps.create-db.outputs.install_failed != 'true'
        run: |
          # Run the backwards compatibility test with RUN_SLOW_TESTS enabled
          RUN_SLOW_TESTS=true pdm run pytest tests/performance/database/test_backwards_compatibility.py -v --tb=short

      - name: Report skipped
        if: steps.prev-version.outputs.skip == 'true' || steps.create-db.outputs.install_failed == 'true'
        run: |
          if [ "${{ steps.create-db.outputs.install_failed }}" = "true" ]; then
            echo "::notice::Backwards compatibility test skipped - previous version has unresolvable dependency conflicts"
          else
            echo "::notice::Backwards compatibility test skipped - no previous PyPI version available"
          fi

  # Database migration tests - validates Alembic migration infrastructure.
  # Runs on any trigger — including PRs that touch migration-relevant paths
  # (gated by the top-level `pull_request: paths:` filter).
  migration-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    name: Database Migration Tests
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up PDM
        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
        with:
          python-version: '3.12'

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force

      - name: Install dependencies
        run: pdm install --dev

      - name: Run Alembic migration tests
        run: |
          pdm run pytest tests/database/test_alembic_migrations.py -v --tb=short

      - name: Run migration version tests
        run: |
          pdm run pytest tests/database/test_migration_0003_indexes.py tests/database/test_migration_0004_app_settings.py -v --tb=short

      - name: Run database initialization tests
        run: |
          pdm run pytest tests/database/test_initialize_functions.py tests/test_database_initialization.py -v --tb=short

      - name: Run encrypted DB integration tests
        run: |
          pdm run pytest tests/auth_tests/test_encrypted_db.py -v --tb=short