topoteretes--cognee/.github/workflows/performance_report.yml

name: performance report

# Reusable workflow: runs the percentile performance report once for a given
# dataset + mode, uploads the JSON + HTML artifacts to S3, and exposes the
# headline metrics + HTML object key as outputs for the caller (the Slack bot in
# nightly_tests.yml).
#
# Both backends run on every call, as two separate jobs:
#   - file_based : cognee defaults (SQLite + Kuzu + LanceDB), no services.
#   - postgres   : full Postgres (relational + PGVector + Postgres graph) via a
#                  `services: postgres` container (mirrors e2e_tests.yml).
# Each job exposes its own metrics + html_key outputs (file_based_* / postgres_*).
on:
  workflow_call:
    inputs:
      mode:
        description: "Run mode: 'mock_llm' (no API calls) or 'llm' (real LLM)."
        required: true
        type: string
      label:
        description: "Dataset label, used in the S3 output path and display name."
        required: true
        type: string
      runs:
        description: "Number of sequential benchmark runs."
        required: false
        type: string
        default: '10'
      num_memories:
        description: "If set, forwarded as --num-memories (limit input documents)."
        required: false
        type: string
        default: ''
      memories_key:
        description: "S3 object key (under the bucket) downloaded and used as --memories."
        required: true
        type: string
      mock_memories_key:
        description: "If set, S3 object key downloaded and used as --mock-memories."
        required: false
        type: string
        default: ''
    outputs:
      file_based_metrics:
        description: "File-based run: success + add/cognify/search/total p50/p90/p99."
        value: ${{ jobs.file_based.outputs.metrics }}
      file_based_html_key:
        description: "File-based run: S3 object key of the HTML report."
        value: ${{ jobs.file_based.outputs.html_key }}
      postgres_metrics:
        description: "Postgres run: success + add/cognify/search/total p50/p90/p99."
        value: ${{ jobs.postgres.outputs.metrics }}
      postgres_html_key:
        description: "Postgres run: S3 object key of the HTML report."
        value: ${{ jobs.postgres.outputs.html_key }}

env:
  ENV: 'dev'
  COGNEE_SKIP_CONNECTION_TEST: 'true'
  RUNTIME__LOG_LEVEL: ERROR
  BUCKET: github-runner-cognee-tests

jobs:
  # ── File-based backend: SQLite + Kuzu + LanceDB (cognee defaults) ────────────
  file_based:
    name: file_based — ${{ inputs.label }} (${{ inputs.mode }})
    runs-on: ubuntu-22.04
    outputs:
      metrics: ${{ steps.parse.outputs.metrics }}
      html_key: ${{ steps.upload.outputs.html_key }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Cognee Setup
        uses: ./.github/actions/cognee_setup
        with:
          python-version: '3.11.x'

      - name: Download dataset from S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_DEV_USER_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_DEV_USER_SECRET_KEY }}
          AWS_DEFAULT_REGION: eu-west-1
          MEMORIES_KEY: ${{ inputs.memories_key }}
          MOCK_MEMORIES_KEY: ${{ inputs.mock_memories_key }}
        run: |
          set -euo pipefail
          mkdir -p performance_datasets
          aws s3 cp "s3://$BUCKET/$MEMORIES_KEY" performance_datasets/memories.json
          echo "MEMORIES_FILE=$PWD/performance_datasets/memories.json" >> "$GITHUB_ENV"
          if [ -n "$MOCK_MEMORIES_KEY" ]; then
            aws s3 cp "s3://$BUCKET/$MOCK_MEMORIES_KEY" performance_datasets/mock_memories.json
            echo "MOCK_MEMORIES_FILE=$PWD/performance_datasets/mock_memories.json" >> "$GITHUB_ENV"
          fi

      - name: Run performance report
        id: run
        env:
          PYTHONFAULTHANDLER: 1
          # LLM + embedding config via the standard CI secrets (see e2e_tests.yml).
          # Only exercised for mode == 'llm'; the mock run hits no external APIs.
          LLM_MODEL: ${{ secrets.LLM_MODEL }}
          LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          LLM_ARGS: ${{ secrets.LLM_ARGS }}
          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
        run: |
          set -euo pipefail
          TS="$(date -u '+%Y-%m-%d_%H-%M-%SZ')"
          JSON_PATH="performance_results/file_based/${{ inputs.label }}/${{ inputs.mode }}_${TS}.json"
          HTML_PATH="performance_results/file_based/${{ inputs.label }}/${{ inputs.mode }}_${TS}.html"
          mkdir -p "$(dirname "$JSON_PATH")"
          echo "JSON_PATH=$JSON_PATH" >> "$GITHUB_ENV"
          echo "HTML_PATH=$HTML_PATH" >> "$GITHUB_ENV"

          ARGS=(--runs "${{ inputs.runs }}" --memories "$MEMORIES_FILE")
          if [ -n "${MOCK_MEMORIES_FILE:-}" ]; then
            ARGS+=(--mock-memories "$MOCK_MEMORIES_FILE")
          fi
          if [ "${{ inputs.mode }}" = "mock_llm" ]; then
            ARGS+=(--mock-llm)
          fi
          if [ -n "${{ inputs.num_memories }}" ]; then
            ARGS+=(--num-memories "${{ inputs.num_memories }}")
          fi

          uv run python cognee/tests/performance/statistics_percentile_report.py \
            "${ARGS[@]}" \
            --output "$JSON_PATH" \
            --html "$HTML_PATH"

      - name: Upload reports to S3
        id: upload
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_DEV_USER_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_DEV_USER_SECRET_KEY }}
          AWS_DEFAULT_REGION: eu-west-1
        run: |
          set -euo pipefail
          aws s3 cp "$JSON_PATH" "s3://$BUCKET/$JSON_PATH" --content-type application/json
          aws s3 cp "$HTML_PATH" "s3://$BUCKET/$HTML_PATH" --content-type text/html
          # Presigning is done by the caller (the Slack job), NOT here: a presigned URL
          # embeds the AWS access key id, and GitHub scrubs registered secrets from
          # reusable-workflow outputs — which would blank the link. Pass only the
          # (non-secret) object key across the boundary.
          echo "html_key=$HTML_PATH" >> "$GITHUB_OUTPUT"

      - name: Parse headline metrics
        id: parse
        run: |
          set -euo pipefail
          METRICS="$(jq -c '{
            success: "\(.succeeded)/\(.num_runs)",
            add:     {p50: .stats.add_time_s.p50,          p90: .stats.add_time_s.p90,          p99: .stats.add_time_s.p99},
            cognify: {p50: .stats.cognify_time_s.p50,      p90: .stats.cognify_time_s.p90,      p99: .stats.cognify_time_s.p99},
            search:  {p50: .stats.search_time.p50,         p90: .stats.search_time.p90,         p99: .stats.search_time.p99},
            total:   {p50: .stats.total_ingest_time_s.p50, p90: .stats.total_ingest_time_s.p90, p99: .stats.total_ingest_time_s.p99}
          }' "$JSON_PATH")"
          echo "metrics=$METRICS" >> "$GITHUB_OUTPUT"

  # ── Postgres backend: Postgres relational + PGVector + Postgres graph ────────
  postgres:
    name: postgres — ${{ inputs.label }} (${{ inputs.mode }})
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
    services:
      postgres:
        image: ghcr.io/topoteretes/pgvector:pg17
        credentials:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
        env:
          POSTGRES_USER: cognee
          POSTGRES_PASSWORD: cognee
          POSTGRES_DB: cognee_db
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    outputs:
      metrics: ${{ steps.parse.outputs.metrics }}
      html_key: ${{ steps.upload.outputs.html_key }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Cognee Setup
        uses: ./.github/actions/cognee_setup
        with:
          python-version: '3.11.x'
          extra-dependencies: "postgres"

      - name: Download dataset from S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_DEV_USER_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_DEV_USER_SECRET_KEY }}
          AWS_DEFAULT_REGION: eu-west-1
          MEMORIES_KEY: ${{ inputs.memories_key }}
          MOCK_MEMORIES_KEY: ${{ inputs.mock_memories_key }}
        run: |
          set -euo pipefail
          mkdir -p performance_datasets
          aws s3 cp "s3://$BUCKET/$MEMORIES_KEY" performance_datasets/memories.json
          echo "MEMORIES_FILE=$PWD/performance_datasets/memories.json" >> "$GITHUB_ENV"
          if [ -n "$MOCK_MEMORIES_KEY" ]; then
            aws s3 cp "s3://$BUCKET/$MOCK_MEMORIES_KEY" performance_datasets/mock_memories.json
            echo "MOCK_MEMORIES_FILE=$PWD/performance_datasets/mock_memories.json" >> "$GITHUB_ENV"
          fi

      - name: Run performance report
        id: run
        env:
          PYTHONFAULTHANDLER: 1
          # LLM + embedding config via the standard CI secrets (see e2e_tests.yml).
          # Only exercised for mode == 'llm'; the mock run hits no external APIs.
          LLM_MODEL: ${{ secrets.LLM_MODEL }}
          LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          LLM_ARGS: ${{ secrets.LLM_ARGS }}
          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
          # Full Postgres stack (creds match the postgres service above; mirrors e2e_tests.yml).
          DB_PROVIDER: 'postgres'
          DB_HOST: '127.0.0.1'
          DB_PORT: '5432'
          DB_USERNAME: cognee
          DB_PASSWORD: cognee
          DB_NAME: cognee_db
          VECTOR_DB_PROVIDER: 'pgvector'
          VECTOR_DB_HOST: '127.0.0.1'
          VECTOR_DB_PORT: '5432'
          VECTOR_DB_USERNAME: cognee
          VECTOR_DB_PASSWORD: cognee
          VECTOR_DATASET_DATABASE_HANDLER: 'pgvector'
          GRAPH_DATABASE_PROVIDER: 'postgres'
          GRAPH_DATABASE_HOST: '127.0.0.1'
          GRAPH_DATABASE_PORT: '5432'
          GRAPH_DATABASE_USERNAME: cognee
          GRAPH_DATABASE_PASSWORD: cognee
          GRAPH_DATABASE_NAME: cognee_db
        run: |
          set -euo pipefail
          TS="$(date -u '+%Y-%m-%d_%H-%M-%SZ')"
          JSON_PATH="performance_results/postgres/${{ inputs.label }}/${{ inputs.mode }}_${TS}.json"
          HTML_PATH="performance_results/postgres/${{ inputs.label }}/${{ inputs.mode }}_${TS}.html"
          mkdir -p "$(dirname "$JSON_PATH")"
          echo "JSON_PATH=$JSON_PATH" >> "$GITHUB_ENV"
          echo "HTML_PATH=$HTML_PATH" >> "$GITHUB_ENV"

          ARGS=(--runs "${{ inputs.runs }}" --memories "$MEMORIES_FILE")
          if [ -n "${MOCK_MEMORIES_FILE:-}" ]; then
            ARGS+=(--mock-memories "$MOCK_MEMORIES_FILE")
          fi
          if [ "${{ inputs.mode }}" = "mock_llm" ]; then
            ARGS+=(--mock-llm)
          fi
          if [ -n "${{ inputs.num_memories }}" ]; then
            ARGS+=(--num-memories "${{ inputs.num_memories }}")
          fi

          uv run python cognee/tests/performance/statistics_percentile_report.py \
            "${ARGS[@]}" \
            --output "$JSON_PATH" \
            --html "$HTML_PATH"

      - name: Upload reports to S3
        id: upload
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_DEV_USER_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_DEV_USER_SECRET_KEY }}
          AWS_DEFAULT_REGION: eu-west-1
        run: |
          set -euo pipefail
          aws s3 cp "$JSON_PATH" "s3://$BUCKET/$JSON_PATH" --content-type application/json
          aws s3 cp "$HTML_PATH" "s3://$BUCKET/$HTML_PATH" --content-type text/html
          # Presigning is done by the caller (the Slack job), NOT here: a presigned URL
          # embeds the AWS access key id, and GitHub scrubs registered secrets from
          # reusable-workflow outputs — which would blank the link. Pass only the
          # (non-secret) object key across the boundary.
          echo "html_key=$HTML_PATH" >> "$GITHUB_OUTPUT"

      - name: Parse headline metrics
        id: parse
        run: |
          set -euo pipefail
          METRICS="$(jq -c '{
            success: "\(.succeeded)/\(.num_runs)",
            add:     {p50: .stats.add_time_s.p50,          p90: .stats.add_time_s.p90,          p99: .stats.add_time_s.p99},
            cognify: {p50: .stats.cognify_time_s.p50,      p90: .stats.cognify_time_s.p90,      p99: .stats.cognify_time_s.p99},
            search:  {p50: .stats.search_time.p50,         p90: .stats.search_time.p90,         p99: .stats.search_time.p99},
            total:   {p50: .stats.total_ingest_time_s.p50, p90: .stats.total_ingest_time_s.p90, p99: .stats.total_ingest_time_s.p99}
          }' "$JSON_PATH")"
          echo "metrics=$METRICS" >> "$GITHUB_OUTPUT"