chore: import upstream snapshot with attribution
Ruff / Ruff (push) Has been cancelled
Test / Core Tests (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.10) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.11) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.12) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.13) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.9) (push) Has been cancelled
Test / Full Coverage (Python 3.11) (push) Has been cancelled
Test / Core Provider Tests (OpenAI) (push) Has been cancelled
Test / Core Provider Tests (Anthropic) (push) Has been cancelled
Test / Core Provider Tests (Google) (push) Has been cancelled
Test / Core Provider Tests (Other) (push) Has been cancelled
Test / Anthropic Tests (push) Has been cancelled
Test / Gemini Tests (push) Has been cancelled
Test / Google GenAI Tests (push) Has been cancelled
Test / Vertex AI Tests (push) Has been cancelled
Test / OpenAI Tests (push) Has been cancelled
Test / Writer Tests (push) Has been cancelled
Test / Auto Client Tests (push) Has been cancelled
ty / type-check (push) Has been cancelled
Ruff / Ruff (push) Has been cancelled
Test / Core Tests (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.10) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.11) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.12) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.13) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.9) (push) Has been cancelled
Test / Full Coverage (Python 3.11) (push) Has been cancelled
Test / Core Provider Tests (OpenAI) (push) Has been cancelled
Test / Core Provider Tests (Anthropic) (push) Has been cancelled
Test / Core Provider Tests (Google) (push) Has been cancelled
Test / Core Provider Tests (Other) (push) Has been cancelled
Test / Anthropic Tests (push) Has been cancelled
Test / Gemini Tests (push) Has been cancelled
Test / Google GenAI Tests (push) Has been cancelled
Test / Vertex AI Tests (push) Has been cancelled
Test / OpenAI Tests (push) Has been cancelled
Test / Writer Tests (push) Has been cancelled
Test / Auto Client Tests (push) Has been cancelled
ty / type-check (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,432 @@
|
||||
name: Test
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Core tests without LLM providers
|
||||
core-tests:
|
||||
name: Core Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Run core tests
|
||||
run: >-
|
||||
uv run --frozen pytest tests/ --asyncio-mode=auto -n auto
|
||||
--ignore=tests/coverage
|
||||
--ignore=tests/test_batch_processor_coverage.py
|
||||
-k 'not test_core_providers and not test_openai and not test_anthropic
|
||||
and not test_gemini and not test_genai and not test_writer and not
|
||||
test_vertexai and not docs'
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
||||
|
||||
# Offline coverage tests must not be filtered by provider names.
|
||||
offline-coverage-tests:
|
||||
name: Offline Coverage Tests (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# xai-sdk requires Python 3.10+, so Python 3.9 is a smoke job.
|
||||
# Only SDK-dependent xAI tests and source are omitted there.
|
||||
- python-version: "3.9"
|
||||
pytest-args: >-
|
||||
--ignore=tests/coverage/test_xai_client_coverage.py
|
||||
--ignore=tests/coverage/test_xai_handlers_coverage.py
|
||||
-k 'not test_xai_builder_forwards_real_client_and_mode and
|
||||
not test_xai_builder_reports_unavailable_factory and
|
||||
not (test_tail_builder_factory_failure_is_logged_and_propagated
|
||||
and _build_xai)'
|
||||
coverage-omit: instructor/v2/providers/xai/*
|
||||
- python-version: "3.10"
|
||||
pytest-args: ""
|
||||
coverage-omit: ""
|
||||
- python-version: "3.11"
|
||||
pytest-args: ""
|
||||
coverage-omit: ""
|
||||
- python-version: "3.12"
|
||||
pytest-args: ""
|
||||
coverage-omit: ""
|
||||
- python-version: "3.13"
|
||||
pytest-args: ""
|
||||
coverage-omit: ""
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
PYTHONWARNINGS: error::ResourceWarning
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache-suffix: offline-py${{ matrix.python-version }}
|
||||
- name: Set up Python
|
||||
run: uv python install ${{ matrix.python-version }}
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Run offline coverage tests
|
||||
run: |
|
||||
uv run --frozen coverage erase
|
||||
uv run --frozen coverage run -m pytest \
|
||||
tests/coverage tests/test_batch_processor_coverage.py \
|
||||
--asyncio-mode=auto --strict-config --strict-markers \
|
||||
-W error::pytest.PytestUnraisableExceptionWarning \
|
||||
-W error::pytest.PytestUnhandledThreadExceptionWarning \
|
||||
${{ matrix.pytest-args }}
|
||||
if [ "${{ matrix.python-version }}" = "3.9" ]; then
|
||||
uv run --frozen coverage report \
|
||||
--omit="${{ matrix.coverage-omit }}" --show-missing
|
||||
else
|
||||
uv run --frozen coverage report \
|
||||
--omit="${{ matrix.coverage-omit }}" --show-missing --fail-under=90
|
||||
fi
|
||||
|
||||
# Enforce repository-wide statement and branch coverage without provider secrets.
|
||||
full-coverage:
|
||||
name: Full Coverage (Python 3.11)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Run full offline coverage
|
||||
run: |
|
||||
uv run --frozen coverage erase
|
||||
uv run --frozen coverage run --branch -m pytest tests/ \
|
||||
--asyncio-mode=auto --strict-config --strict-markers \
|
||||
-k 'not docs'
|
||||
uv run --frozen coverage report --show-missing --fail-under=99
|
||||
uv run --frozen coverage json -o "${RUNNER_TEMP}/coverage.json"
|
||||
uv run --frozen python -c 'import json, os; totals = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "coverage.json")))["totals"]; missing = totals["missing_lines"]; print(f"Missing statements: {missing}"); raise SystemExit(1 if missing else 0)'
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
|
||||
# Core provider tests for OpenAI
|
||||
core-openai:
|
||||
name: Core Provider Tests (OpenAI)
|
||||
runs-on: ubuntu-latest
|
||||
needs: core-tests
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip core provider tests (OpenAI)
|
||||
if: ${{ env.OPENAI_API_KEY == '' }}
|
||||
run: echo "Skipping OpenAI core provider tests (missing OPENAI_API_KEY)."
|
||||
- name: Run core provider tests (OpenAI)
|
||||
if: ${{ env.OPENAI_API_KEY != '' }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "openai"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
# Core provider tests for Anthropic
|
||||
core-anthropic:
|
||||
name: Core Provider Tests (Anthropic)
|
||||
runs-on: ubuntu-latest
|
||||
needs: core-tests
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip core provider tests (Anthropic)
|
||||
if: ${{ env.ANTHROPIC_API_KEY == '' }}
|
||||
run: echo "Skipping Anthropic core provider tests (missing ANTHROPIC_API_KEY)."
|
||||
- name: Run core provider tests (Anthropic)
|
||||
if: ${{ env.ANTHROPIC_API_KEY != '' }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "anthropic"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
# Core provider tests for Google
|
||||
core-google:
|
||||
name: Core Provider Tests (Google)
|
||||
runs-on: ubuntu-latest
|
||||
needs: core-tests
|
||||
env:
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip core provider tests (Google)
|
||||
if: ${{ env.GOOGLE_API_KEY == '' || env.GOOGLE_GENAI_MODEL == '' }}
|
||||
run: echo "Skipping Google core provider tests (missing GOOGLE_API_KEY or GOOGLE_GENAI_MODEL)."
|
||||
- name: Run core provider tests (Google)
|
||||
if: ${{ env.GOOGLE_API_KEY != '' && env.GOOGLE_GENAI_MODEL != '' }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "google"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
# Core provider tests for other providers
|
||||
core-other:
|
||||
name: Core Provider Tests (Other)
|
||||
runs-on: ubuntu-latest
|
||||
needs: core-tests
|
||||
env:
|
||||
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
||||
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
|
||||
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
||||
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip core provider tests (Other)
|
||||
if: >-
|
||||
${{ env.COHERE_API_KEY == '' && env.XAI_API_KEY == ''
|
||||
&& env.MISTRAL_API_KEY == '' && env.CEREBRAS_API_KEY == ''
|
||||
&& env.FIREWORKS_API_KEY == '' && env.WRITER_API_KEY == ''
|
||||
&& env.PERPLEXITY_API_KEY == '' }}
|
||||
run: echo "Skipping core provider tests (Other) (missing provider secrets)."
|
||||
- name: Run core provider tests (Cohere, xAI, Mistral, etc)
|
||||
if: >-
|
||||
${{ env.COHERE_API_KEY != '' || env.XAI_API_KEY != ''
|
||||
|| env.MISTRAL_API_KEY != '' || env.CEREBRAS_API_KEY != ''
|
||||
|| env.FIREWORKS_API_KEY != '' || env.WRITER_API_KEY != ''
|
||||
|| env.PERPLEXITY_API_KEY != '' }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "cohere or xai or mistral or cerebras or fireworks or writer or perplexity"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
||||
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
|
||||
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
||||
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
|
||||
# Provider tests run in parallel
|
||||
provider-tests:
|
||||
name: ${{ matrix.provider.name }} Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [core-openai, core-anthropic, core-google, core-other]
|
||||
env:
|
||||
PROVIDER_API_KEY: ${{ secrets[matrix.provider.env_key] }}
|
||||
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
provider:
|
||||
- name: OpenAI
|
||||
env_key: OPENAI_API_KEY
|
||||
test_path: tests/llm/test_openai
|
||||
- name: Anthropic
|
||||
env_key: ANTHROPIC_API_KEY
|
||||
test_path: tests/llm/test_anthropic
|
||||
- name: Gemini
|
||||
env_key: GOOGLE_API_KEY
|
||||
test_path: tests/llm/test_gemini
|
||||
- name: Google GenAI
|
||||
env_key: GOOGLE_API_KEY
|
||||
test_path: tests/llm/test_genai
|
||||
- name: Vertex AI
|
||||
env_key: GOOGLE_API_KEY
|
||||
test_path: tests/llm/test_vertexai
|
||||
- name: Writer
|
||||
env_key: WRITER_API_KEY
|
||||
test_path: tests/llm/test_writer
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
cache-suffix: provider-${{ matrix.provider.name }}-py311
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip ${{ matrix.provider.name }} tests
|
||||
if: >-
|
||||
${{ env.PROVIDER_API_KEY == '' ||
|
||||
((matrix.provider.name == 'Gemini' || matrix.provider.name == 'Google GenAI'
|
||||
|| matrix.provider.name == 'Vertex AI') && env.GOOGLE_GENAI_MODEL == '') }}
|
||||
run: >-
|
||||
echo "Skipping ${{ matrix.provider.name }} tests
|
||||
(missing ${{ matrix.provider.env_key }} or GOOGLE_GENAI_MODEL)."
|
||||
- name: Run ${{ matrix.provider.name }} tests
|
||||
if: >-
|
||||
${{ env.PROVIDER_API_KEY != '' &&
|
||||
((matrix.provider.name != 'Gemini' && matrix.provider.name != 'Google GenAI'
|
||||
&& matrix.provider.name != 'Vertex AI') || env.GOOGLE_GENAI_MODEL != '') }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest ${{ matrix.provider.test_path }} --asyncio-mode=auto -n auto
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
${{ matrix.provider.env_key }}: ${{ secrets[matrix.provider.env_key] }}
|
||||
|
||||
# Auto client needs multiple providers
|
||||
auto-client-test:
|
||||
name: Auto Client Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [core-openai, core-anthropic, core-google, core-other]
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.11"
|
||||
- name: Set up Python
|
||||
run: uv python install 3.11
|
||||
- name: Install the project
|
||||
run: uv sync --frozen --all-extras
|
||||
- name: Skip Auto Client tests
|
||||
if: >-
|
||||
${{ env.OPENAI_API_KEY == '' || env.GOOGLE_API_KEY == ''
|
||||
|| env.COHERE_API_KEY == '' || env.ANTHROPIC_API_KEY == ''
|
||||
|| env.XAI_API_KEY == '' }}
|
||||
run: echo "Skipping Auto Client tests (missing one or more provider secrets)."
|
||||
- name: Run Auto Client tests
|
||||
if: >-
|
||||
${{ env.OPENAI_API_KEY != '' && env.GOOGLE_API_KEY != ''
|
||||
&& env.COHERE_API_KEY != '' && env.ANTHROPIC_API_KEY != ''
|
||||
&& env.XAI_API_KEY != '' }}
|
||||
run: |
|
||||
set +e
|
||||
uv run --frozen pytest tests/providers/test_auto_client.py --asyncio-mode=auto -n auto
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -eq 5 ]; then
|
||||
echo "No tests collected; treating as success."
|
||||
exit 0
|
||||
fi
|
||||
exit $status
|
||||
env:
|
||||
INSTRUCTOR_ENV: CI
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
Reference in New Issue
Block a user