wehub-resource-sync
0d3cb498a3
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
3447 lines
104 KiB
JSON
3447 lines
104 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://promptfoo.dev/config-schema.json",
|
|
"title": "Promptfoo Configuration Schema",
|
|
"$ref": "#/definitions/PromptfooConfigSchema",
|
|
"definitions": {
|
|
"PromptfooConfigSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"tags": {
|
|
"$ref": "#/definitions/__schema0"
|
|
},
|
|
"description": {
|
|
"$ref": "#/definitions/__schema1"
|
|
},
|
|
"providers": {
|
|
"$ref": "#/definitions/__schema2"
|
|
},
|
|
"prompts": {
|
|
"$ref": "#/definitions/__schema13"
|
|
},
|
|
"tests": {
|
|
"$ref": "#/definitions/__schema14"
|
|
},
|
|
"scenarios": {
|
|
"$ref": "#/definitions/__schema29"
|
|
},
|
|
"defaultTest": {
|
|
"$ref": "#/definitions/__schema30"
|
|
},
|
|
"outputPath": {
|
|
"$ref": "#/definitions/__schema31"
|
|
},
|
|
"sharing": {
|
|
"$ref": "#/definitions/__schema32"
|
|
},
|
|
"nunjucksFilters": {
|
|
"$ref": "#/definitions/__schema33"
|
|
},
|
|
"env": {
|
|
"$ref": "#/definitions/__schema34"
|
|
},
|
|
"derivedMetrics": {
|
|
"$ref": "#/definitions/__schema35"
|
|
},
|
|
"extensions": {
|
|
"$ref": "#/definitions/__schema36"
|
|
},
|
|
"metadata": {
|
|
"$ref": "#/definitions/__schema37"
|
|
},
|
|
"redteam": {
|
|
"$ref": "#/definitions/__schema38"
|
|
},
|
|
"writeLatestResults": {
|
|
"$ref": "#/definitions/__schema39"
|
|
},
|
|
"tracing": {
|
|
"$ref": "#/definitions/__schema40"
|
|
},
|
|
"evaluateOptions": {
|
|
"$ref": "#/definitions/__schema41"
|
|
},
|
|
"commandLineOptions": {
|
|
"$ref": "#/definitions/__schema43"
|
|
},
|
|
"targets": {
|
|
"$ref": "#/definitions/__schema44"
|
|
}
|
|
},
|
|
"required": ["prompts"],
|
|
"additionalProperties": false
|
|
},
|
|
"__schema0": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"__schema1": {
|
|
"type": "string"
|
|
},
|
|
"__schema2": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema3"
|
|
}
|
|
]
|
|
},
|
|
"__schema3": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema4"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema4"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"$ref": "#/definitions/__schema5"
|
|
}
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema5"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"__schema4": {},
|
|
"__schema5": {
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {},
|
|
"label": {},
|
|
"config": {},
|
|
"prompts": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"delay": {
|
|
"type": "number"
|
|
},
|
|
"env": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema7"
|
|
}
|
|
]
|
|
},
|
|
"inputs": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema8"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema6": {
|
|
"type": "string"
|
|
},
|
|
"__schema7": {
|
|
"type": "object",
|
|
"properties": {
|
|
"ABLIT_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"ABLIT_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AI21_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AI21_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AIML_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ANTHROPIC_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ANTHROPIC_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"ATLASCLOUD_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AWS_BEARER_TOKEN_BEDROCK": {
|
|
"type": "string"
|
|
},
|
|
"AWS_BEDROCK_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_AUTHORITY_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CLIENT_ID": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CLIENT_SECRET": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_EMBEDDING_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_TENANT_ID": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_TOKEN_SCOPE": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CODE_USE_BEDROCK": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CODE_USE_VERTEX": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CONFIG_DIR": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_ACCOUNT_ID": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_GATEWAY_ID": {
|
|
"type": "string"
|
|
},
|
|
"CF_AIG_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"COMETAPI_KEY": {
|
|
"type": "string"
|
|
},
|
|
"COHERE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"COHERE_CLIENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"DATABRICKS_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"DATABRICKS_WORKSPACE_URL": {
|
|
"type": "string"
|
|
},
|
|
"DOCKER_MODEL_RUNNER_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"DOCKER_MODEL_RUNNER_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ELEVENLABS_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"FAL_KEY": {
|
|
"type": "string"
|
|
},
|
|
"FIREWORKS_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"FIREWORKS_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GITHUB_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_LOCATION": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_GENERATIVE_AI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GEMINI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GROQ_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HELICONE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HF_API_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"HF_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"HYPERBOLIC_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HUGGING_FACE_HUB_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"JFROG_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_HOST": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_PUBLIC_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_SECRET_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LITELLM_API_BASE": {
|
|
"type": "string"
|
|
},
|
|
"LLAMA_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"LOCALAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MINIMAX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MOONSHOT_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MLFLOW_GATEWAY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MLFLOW_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"MODELSLAB_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"N8N_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NSCALE_SERVICE_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"NSCALE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NOVITA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NVIDIA_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"NVIDIA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OLLAMA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OLLAMA_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_ORGANIZATION": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"CODEX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_CONFIG_PATH": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_PORT": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"ORCAROUTER_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PALM_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"PALM_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PORTKEY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_CA_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_PFX_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_PFX_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_ALIAS": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_INSECURE_SSL": {
|
|
"type": "string"
|
|
},
|
|
"QUIVERAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"REPLICATE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"REPLICATE_API_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_CLIENT_ID": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_TENANT_ID": {
|
|
"type": "string"
|
|
},
|
|
"VERCEL_AI_GATEWAY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"VERCEL_AI_GATEWAY_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_VERSION": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_PUBLISHER": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_REGION": {
|
|
"type": "string"
|
|
},
|
|
"VOYAGE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"VOYAGE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_APIKEY": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_AUTH_TYPE": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_BEARER_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"XAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"XAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_ENDPOINT": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_API_VERSION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_DEFAULT_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_MAX_TOKENS": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_TEMPERATURE": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_TOP_P": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_MAX_RETRIES": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_EVAL_TIMEOUT_MS": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema8": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string",
|
|
"pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"config": {
|
|
"$ref": "#/definitions/__schema9"
|
|
},
|
|
"description": {
|
|
"$ref": "#/definitions/__schema11"
|
|
},
|
|
"type": {
|
|
"$ref": "#/definitions/__schema12"
|
|
}
|
|
},
|
|
"required": ["description"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"__schema9": {
|
|
"type": "object",
|
|
"properties": {
|
|
"benign": {
|
|
"type": "boolean"
|
|
},
|
|
"inputPurpose": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"injectionPlacements": {
|
|
"minItems": 1,
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/__schema10"
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema10": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"__schema11": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"__schema12": {
|
|
"type": "string",
|
|
"enum": ["text", "pdf", "docx", "image"]
|
|
},
|
|
"__schema13": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"label": {
|
|
"type": "string"
|
|
},
|
|
"raw": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["id"],
|
|
"additionalProperties": false
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"raw": {
|
|
"type": "string"
|
|
},
|
|
"template": {
|
|
"type": "string"
|
|
},
|
|
"display": {
|
|
"type": "string"
|
|
},
|
|
"label": {
|
|
"type": "string"
|
|
},
|
|
"function": {},
|
|
"config": {}
|
|
},
|
|
"required": ["raw", "label"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"__schema14": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema15"
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema28"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema28"
|
|
}
|
|
]
|
|
},
|
|
"__schema15": {
|
|
"type": "object",
|
|
"properties": {
|
|
"description": {
|
|
"$ref": "#/definitions/__schema16"
|
|
},
|
|
"vars": {
|
|
"$ref": "#/definitions/__schema17"
|
|
},
|
|
"provider": {
|
|
"$ref": "#/definitions/__schema18"
|
|
},
|
|
"providers": {
|
|
"$ref": "#/definitions/__schema19"
|
|
},
|
|
"prompts": {
|
|
"$ref": "#/definitions/__schema20"
|
|
},
|
|
"providerOutput": {
|
|
"$ref": "#/definitions/__schema21"
|
|
},
|
|
"assert": {
|
|
"$ref": "#/definitions/__schema22"
|
|
},
|
|
"assertScoringFunction": {
|
|
"$ref": "#/definitions/__schema24"
|
|
},
|
|
"options": {
|
|
"$ref": "#/definitions/__schema25"
|
|
},
|
|
"threshold": {
|
|
"$ref": "#/definitions/__schema26"
|
|
},
|
|
"metadata": {
|
|
"$ref": "#/definitions/__schema27"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema16": {
|
|
"type": "string"
|
|
},
|
|
"__schema17": {},
|
|
"__schema18": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema5"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {},
|
|
"callApi": {},
|
|
"callEmbeddingApi": {},
|
|
"callClassificationApi": {},
|
|
"label": {},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"delay": {
|
|
"type": "number"
|
|
},
|
|
"config": {},
|
|
"inputs": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema8"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"required": ["id", "callApi"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"__schema19": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"__schema20": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"__schema21": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
}
|
|
]
|
|
},
|
|
"__schema22": {
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"const": "assert-set"
|
|
},
|
|
"assert": {
|
|
"type": "array",
|
|
"items": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema23"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"weight": {
|
|
"type": "number"
|
|
},
|
|
"metric": {
|
|
"type": "string"
|
|
},
|
|
"threshold": {
|
|
"type": "number"
|
|
},
|
|
"config": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
}
|
|
},
|
|
"required": ["type", "assert"],
|
|
"additionalProperties": false
|
|
},
|
|
{
|
|
"$ref": "#/definitions/__schema23"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"__schema23": {
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"agent-rubric",
|
|
"answer-relevance",
|
|
"bleu",
|
|
"classifier",
|
|
"contains",
|
|
"contains-all",
|
|
"contains-any",
|
|
"contains-html",
|
|
"contains-json",
|
|
"contains-sql",
|
|
"contains-xml",
|
|
"context-faithfulness",
|
|
"context-recall",
|
|
"context-relevance",
|
|
"conversation-relevance",
|
|
"cost",
|
|
"equals",
|
|
"factuality",
|
|
"finish-reason",
|
|
"g-eval",
|
|
"gleu",
|
|
"guardrails",
|
|
"icontains",
|
|
"icontains-all",
|
|
"icontains-any",
|
|
"is-html",
|
|
"is-json",
|
|
"is-refusal",
|
|
"is-sql",
|
|
"is-valid-function-call",
|
|
"is-valid-openai-function-call",
|
|
"is-valid-openai-tools-call",
|
|
"is-xml",
|
|
"javascript",
|
|
"latency",
|
|
"levenshtein",
|
|
"llm-rubric",
|
|
"pi",
|
|
"meteor",
|
|
"model-graded-closedqa",
|
|
"model-graded-factuality",
|
|
"moderation",
|
|
"perplexity",
|
|
"perplexity-score",
|
|
"python",
|
|
"regex",
|
|
"rouge-n",
|
|
"ruby",
|
|
"similar",
|
|
"similar:cosine",
|
|
"similar:dot",
|
|
"similar:euclidean",
|
|
"starts-with",
|
|
"tool-call-f1",
|
|
"skill-used",
|
|
"trajectory:goal-success",
|
|
"trajectory:tool-args-match",
|
|
"trajectory:step-count",
|
|
"trajectory:tool-sequence",
|
|
"trajectory:tool-used",
|
|
"trace-error-spans",
|
|
"trace-span-count",
|
|
"trace-span-duration",
|
|
"search-rubric",
|
|
"webhook",
|
|
"word-count"
|
|
]
|
|
},
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"agent-rubric",
|
|
"answer-relevance",
|
|
"bleu",
|
|
"classifier",
|
|
"contains",
|
|
"contains-all",
|
|
"contains-any",
|
|
"contains-html",
|
|
"contains-json",
|
|
"contains-sql",
|
|
"contains-xml",
|
|
"context-faithfulness",
|
|
"context-recall",
|
|
"context-relevance",
|
|
"conversation-relevance",
|
|
"cost",
|
|
"equals",
|
|
"factuality",
|
|
"finish-reason",
|
|
"g-eval",
|
|
"gleu",
|
|
"guardrails",
|
|
"icontains",
|
|
"icontains-all",
|
|
"icontains-any",
|
|
"is-html",
|
|
"is-json",
|
|
"is-refusal",
|
|
"is-sql",
|
|
"is-valid-function-call",
|
|
"is-valid-openai-function-call",
|
|
"is-valid-openai-tools-call",
|
|
"is-xml",
|
|
"javascript",
|
|
"latency",
|
|
"levenshtein",
|
|
"llm-rubric",
|
|
"pi",
|
|
"meteor",
|
|
"model-graded-closedqa",
|
|
"model-graded-factuality",
|
|
"moderation",
|
|
"perplexity",
|
|
"perplexity-score",
|
|
"python",
|
|
"regex",
|
|
"rouge-n",
|
|
"ruby",
|
|
"similar",
|
|
"similar:cosine",
|
|
"similar:dot",
|
|
"similar:euclidean",
|
|
"starts-with",
|
|
"tool-call-f1",
|
|
"skill-used",
|
|
"trajectory:goal-success",
|
|
"trajectory:tool-args-match",
|
|
"trajectory:step-count",
|
|
"trajectory:tool-sequence",
|
|
"trajectory:tool-used",
|
|
"trace-error-spans",
|
|
"trace-span-count",
|
|
"trace-span-duration",
|
|
"search-rubric",
|
|
"webhook",
|
|
"word-count"
|
|
]
|
|
},
|
|
{
|
|
"type": "string",
|
|
"enum": ["select-best", "human", "max-score"]
|
|
},
|
|
{}
|
|
]
|
|
},
|
|
"value": {},
|
|
"config": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
"threshold": {
|
|
"type": "number"
|
|
},
|
|
"weight": {
|
|
"type": "number"
|
|
},
|
|
"provider": {},
|
|
"rubricPrompt": {},
|
|
"metric": {
|
|
"type": "string"
|
|
},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"contextTransform": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["type"],
|
|
"additionalProperties": false
|
|
},
|
|
"__schema24": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"pattern": "^file:\\/\\/.*\\.(js|cjs|mjs|ts|cts|mts|py)(?::[\\w.]+)?$"
|
|
},
|
|
{}
|
|
]
|
|
},
|
|
"__schema25": {
|
|
"type": "object",
|
|
"properties": {
|
|
"prefix": {
|
|
"type": "string"
|
|
},
|
|
"suffix": {
|
|
"type": "string"
|
|
},
|
|
"postprocess": {
|
|
"type": "string"
|
|
},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"transformVars": {
|
|
"type": "string"
|
|
},
|
|
"storeOutputAs": {
|
|
"type": "string"
|
|
},
|
|
"rubricPrompt": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"role": {
|
|
"type": "string"
|
|
},
|
|
"content": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["role", "content"],
|
|
"additionalProperties": false
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"provider": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"factuality": {
|
|
"type": "object",
|
|
"properties": {
|
|
"subset": {
|
|
"type": "number"
|
|
},
|
|
"superset": {
|
|
"type": "number"
|
|
},
|
|
"agree": {
|
|
"type": "number"
|
|
},
|
|
"disagree": {
|
|
"type": "number"
|
|
},
|
|
"differButFactual": {
|
|
"type": "number"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"disableVarExpansion": {
|
|
"type": "boolean"
|
|
},
|
|
"disableConversationVar": {
|
|
"type": "boolean"
|
|
},
|
|
"disableDefaultAsserts": {
|
|
"type": "boolean"
|
|
},
|
|
"runSerially": {
|
|
"type": "boolean"
|
|
},
|
|
"repeat": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
}
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
"__schema26": {
|
|
"type": "number"
|
|
},
|
|
"__schema27": {
|
|
"type": "object",
|
|
"properties": {
|
|
"pluginConfig": {},
|
|
"strategyConfig": {}
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
"__schema28": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {
|
|
"type": "string"
|
|
},
|
|
"config": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "boolean"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "boolean"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
{}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"required": ["path"],
|
|
"additionalProperties": false
|
|
},
|
|
"__schema29": {
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"description": {
|
|
"type": "string"
|
|
},
|
|
"config": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"description": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema16"
|
|
}
|
|
]
|
|
},
|
|
"vars": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema17"
|
|
}
|
|
]
|
|
},
|
|
"provider": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema18"
|
|
}
|
|
]
|
|
},
|
|
"providers": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema19"
|
|
}
|
|
]
|
|
},
|
|
"prompts": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema20"
|
|
}
|
|
]
|
|
},
|
|
"providerOutput": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema21"
|
|
}
|
|
]
|
|
},
|
|
"assert": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema22"
|
|
}
|
|
]
|
|
},
|
|
"assertScoringFunction": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema24"
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema25"
|
|
}
|
|
]
|
|
},
|
|
"threshold": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema26"
|
|
}
|
|
]
|
|
},
|
|
"metadata": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema27"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
}
|
|
},
|
|
"tests": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/__schema15"
|
|
}
|
|
}
|
|
},
|
|
"required": ["config", "tests"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"__schema30": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"vars": {
|
|
"$ref": "#/definitions/__schema17"
|
|
},
|
|
"provider": {
|
|
"$ref": "#/definitions/__schema18"
|
|
},
|
|
"providers": {
|
|
"$ref": "#/definitions/__schema19"
|
|
},
|
|
"prompts": {
|
|
"$ref": "#/definitions/__schema20"
|
|
},
|
|
"providerOutput": {
|
|
"$ref": "#/definitions/__schema21"
|
|
},
|
|
"assert": {
|
|
"$ref": "#/definitions/__schema22"
|
|
},
|
|
"assertScoringFunction": {
|
|
"$ref": "#/definitions/__schema24"
|
|
},
|
|
"options": {
|
|
"$ref": "#/definitions/__schema25"
|
|
},
|
|
"threshold": {
|
|
"$ref": "#/definitions/__schema26"
|
|
},
|
|
"metadata": {
|
|
"$ref": "#/definitions/__schema27"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"__schema31": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"__schema32": {
|
|
"anyOf": [
|
|
{
|
|
"type": "boolean"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"apiBaseUrl": {
|
|
"type": "string"
|
|
},
|
|
"appBaseUrl": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"__schema33": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"__schema34": {
|
|
"anyOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema7"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "boolean"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"__schema35": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"value": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{}
|
|
]
|
|
}
|
|
},
|
|
"required": ["name", "value"],
|
|
"additionalProperties": false
|
|
}
|
|
},
|
|
"__schema36": {
|
|
"anyOf": [
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
]
|
|
},
|
|
"__schema37": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
"__schema38": {
|
|
"type": "object",
|
|
"properties": {
|
|
"injectVar": {
|
|
"description": "Variable to inject. Can be a string or array of strings. If string, it's transformed to an array. Inferred from the prompts by default.",
|
|
"type": "string"
|
|
},
|
|
"purpose": {
|
|
"description": "Purpose override string - describes the prompt templates",
|
|
"type": "string"
|
|
},
|
|
"testGenerationInstructions": {
|
|
"description": "Additional instructions for test generation applied to each plugin",
|
|
"type": "string"
|
|
},
|
|
"provider": {
|
|
"description": "Provider used for generating adversarial inputs",
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {},
|
|
"callApi": {},
|
|
"callEmbeddingApi": {},
|
|
"callClassificationApi": {},
|
|
"label": {},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"delay": {
|
|
"type": "number"
|
|
},
|
|
"config": {},
|
|
"inputs": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string",
|
|
"pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"config": {
|
|
"type": "object",
|
|
"properties": {
|
|
"benign": {
|
|
"type": "boolean"
|
|
},
|
|
"inputPurpose": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"injectionPlacements": {
|
|
"minItems": 1,
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["text", "pdf", "docx", "image"]
|
|
}
|
|
},
|
|
"required": ["description"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"required": ["id", "callApi"],
|
|
"additionalProperties": false
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {},
|
|
"label": {},
|
|
"config": {},
|
|
"prompts": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"transform": {
|
|
"type": "string"
|
|
},
|
|
"delay": {
|
|
"type": "number"
|
|
},
|
|
"env": {
|
|
"type": "object",
|
|
"properties": {
|
|
"ABLIT_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"ABLIT_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AI21_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AI21_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AIML_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ANTHROPIC_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ANTHROPIC_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"ATLASCLOUD_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AWS_BEARER_TOKEN_BEDROCK": {
|
|
"type": "string"
|
|
},
|
|
"AWS_BEDROCK_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_AUTHORITY_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CLIENT_ID": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CLIENT_SECRET": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_EMBEDDING_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_TENANT_ID": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_TOKEN_SCOPE": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CODE_USE_BEDROCK": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CODE_USE_VERTEX": {
|
|
"type": "string"
|
|
},
|
|
"CLAUDE_CONFIG_DIR": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_ACCOUNT_ID": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"CLOUDFLARE_GATEWAY_ID": {
|
|
"type": "string"
|
|
},
|
|
"CF_AIG_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"COMETAPI_KEY": {
|
|
"type": "string"
|
|
},
|
|
"COHERE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"COHERE_CLIENT_NAME": {
|
|
"type": "string"
|
|
},
|
|
"DATABRICKS_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"DATABRICKS_WORKSPACE_URL": {
|
|
"type": "string"
|
|
},
|
|
"DOCKER_MODEL_RUNNER_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"DOCKER_MODEL_RUNNER_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"ELEVENLABS_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"FAL_KEY": {
|
|
"type": "string"
|
|
},
|
|
"FIREWORKS_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"FIREWORKS_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GITHUB_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_LOCATION": {
|
|
"type": "string"
|
|
},
|
|
"GOOGLE_GENERATIVE_AI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GEMINI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"GROQ_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HELICONE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HF_API_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"HF_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"HYPERBOLIC_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"HUGGING_FACE_HUB_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"JFROG_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_HOST": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_PUBLIC_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LANGFUSE_SECRET_KEY": {
|
|
"type": "string"
|
|
},
|
|
"LITELLM_API_BASE": {
|
|
"type": "string"
|
|
},
|
|
"LLAMA_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"LOCALAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"MISTRAL_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MINIMAX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MOONSHOT_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MLFLOW_GATEWAY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"MLFLOW_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"MODELSLAB_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"N8N_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NSCALE_SERVICE_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"NSCALE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NOVITA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"NVIDIA_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"NVIDIA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OLLAMA_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OLLAMA_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"OPENAI_ORGANIZATION": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"CLAWDBOT_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"CODEX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_CONFIG_PATH": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_PORT": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"OPENCLAW_GATEWAY_URL": {
|
|
"type": "string"
|
|
},
|
|
"ORCAROUTER_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PALM_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"PALM_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PORTKEY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_CA_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_PFX_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_PFX_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_PASSWORD": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_JKS_ALIAS": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_INSECURE_SSL": {
|
|
"type": "string"
|
|
},
|
|
"QUIVERAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"REPLICATE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"REPLICATE_API_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_CERT_PATH": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_CLIENT_ID": {
|
|
"type": "string"
|
|
},
|
|
"SHAREPOINT_TENANT_ID": {
|
|
"type": "string"
|
|
},
|
|
"VERCEL_AI_GATEWAY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"VERCEL_AI_GATEWAY_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_HOST": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_API_VERSION": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_PUBLISHER": {
|
|
"type": "string"
|
|
},
|
|
"VERTEX_REGION": {
|
|
"type": "string"
|
|
},
|
|
"VOYAGE_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"VOYAGE_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_APIKEY": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_AUTH_TYPE": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_BEARER_TOKEN": {
|
|
"type": "string"
|
|
},
|
|
"WATSONX_AI_PROJECT_ID": {
|
|
"type": "string"
|
|
},
|
|
"XAI_API_BASE_URL": {
|
|
"type": "string"
|
|
},
|
|
"XAI_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_ENDPOINT": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_API_KEY": {
|
|
"type": "string"
|
|
},
|
|
"AZURE_CONTENT_SAFETY_API_VERSION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_DEFAULT_REGION": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_MAX_TOKENS": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_TEMPERATURE": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_TOP_P": {
|
|
"type": "string"
|
|
},
|
|
"AWS_SAGEMAKER_MAX_RETRIES": {
|
|
"type": "string"
|
|
},
|
|
"PROMPTFOO_EVAL_TIMEOUT_MS": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"inputs": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string",
|
|
"pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"
|
|
},
|
|
"additionalProperties": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"config": {
|
|
"type": "object",
|
|
"properties": {
|
|
"benign": {
|
|
"type": "boolean"
|
|
},
|
|
"inputPurpose": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"injectionPlacements": {
|
|
"minItems": 1,
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["text", "pdf", "docx", "image"]
|
|
}
|
|
},
|
|
"required": ["description"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"numTests": {
|
|
"description": "Number of tests to generate",
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"language": {
|
|
"description": "Language(s) of tests to generate for this plugin",
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"frameworks": {
|
|
"description": "Compliance frameworks to include across reports and commands",
|
|
"minItems": 1,
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"enum": [
|
|
"mitre:atlas",
|
|
"nist:ai:measure",
|
|
"owasp:api",
|
|
"owasp:llm",
|
|
"owasp:agentic",
|
|
"eu:ai-act",
|
|
"iso:42001",
|
|
"gdpr",
|
|
"dod:ai:ethics"
|
|
]
|
|
}
|
|
},
|
|
"entities": {
|
|
"description": "Names of people, brands, or organizations related to your LLM application",
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"contexts": {
|
|
"description": "Security contexts for testing multiple states - each context has its own purpose",
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"description": "Unique identifier for the context"
|
|
},
|
|
"purpose": {
|
|
"description": "Optional purpose/context for this context - used for generation and grading, or inherited from the root redteam purpose when omitted or blank",
|
|
"type": "string"
|
|
},
|
|
"vars": {
|
|
"description": "Variables passed to provider (e.g., context_file, user_role)",
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"required": ["id"],
|
|
"additionalProperties": false
|
|
}
|
|
},
|
|
"plugins": {
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"aegis",
|
|
"agentic:memory-poisoning",
|
|
"ascii-smuggling",
|
|
"beavertails",
|
|
"bfla",
|
|
"bias",
|
|
"bias:age",
|
|
"bias:disability",
|
|
"bias:gender",
|
|
"bias:race",
|
|
"bola",
|
|
"cca",
|
|
"coding-agent:all",
|
|
"coding-agent:automation-poisoning",
|
|
"coding-agent:core",
|
|
"coding-agent:delayed-ci-exfil",
|
|
"coding-agent:generated-vulnerability",
|
|
"coding-agent:network-egress-bypass",
|
|
"coding-agent:procfs-credential-read",
|
|
"coding-agent:repo-prompt-injection",
|
|
"coding-agent:sandbox-read-escape",
|
|
"coding-agent:sandbox-write-escape",
|
|
"coding-agent:secret-env-read",
|
|
"coding-agent:secret-file-read",
|
|
"coding-agent:steganographic-exfil",
|
|
"coding-agent:terminal-output-injection",
|
|
"coding-agent:verifier-sabotage",
|
|
"competitors",
|
|
"contracts",
|
|
"coppa",
|
|
"cross-session-leak",
|
|
"cyberseceval",
|
|
"data-exfil",
|
|
"debug-access",
|
|
"default",
|
|
"divergent-repetition",
|
|
"dod:ai:ethics",
|
|
"dod:ai:ethics:01",
|
|
"dod:ai:ethics:02",
|
|
"dod:ai:ethics:03",
|
|
"dod:ai:ethics:04",
|
|
"dod:ai:ethics:05",
|
|
"donotanswer",
|
|
"ecommerce",
|
|
"ecommerce:compliance-bypass",
|
|
"ecommerce:order-fraud",
|
|
"ecommerce:pci-dss",
|
|
"ecommerce:price-manipulation",
|
|
"eu:ai-act",
|
|
"eu:ai-act:annex3:biometric-id",
|
|
"eu:ai-act:annex3:critical-infrastructure",
|
|
"eu:ai-act:annex3:education",
|
|
"eu:ai-act:annex3:employment",
|
|
"eu:ai-act:annex3:essential-services",
|
|
"eu:ai-act:annex3:justice-democracy",
|
|
"eu:ai-act:annex3:law-enforcement",
|
|
"eu:ai-act:annex3:migration-border",
|
|
"eu:ai-act:art5:biometric-categorisation",
|
|
"eu:ai-act:art5:exploitation-of-vulnerabilities",
|
|
"eu:ai-act:art5:remote-biometric-id-live",
|
|
"eu:ai-act:art5:remote-biometric-id-post",
|
|
"eu:ai-act:art5:social-scoring",
|
|
"eu:ai-act:art5:subliminal-manipulation",
|
|
"excessive-agency",
|
|
"ferpa",
|
|
"financial",
|
|
"financial:calculation-error",
|
|
"financial:compliance-violation",
|
|
"financial:confidential-disclosure",
|
|
"financial:counterfactual",
|
|
"financial:data-leakage",
|
|
"financial:defamation",
|
|
"financial:hallucination",
|
|
"financial:impartiality",
|
|
"financial:japan-fiea-suitability",
|
|
"financial:misconduct",
|
|
"financial:sox-compliance",
|
|
"financial:sycophancy",
|
|
"foundation",
|
|
"gdpr",
|
|
"gdpr:art15",
|
|
"gdpr:art17",
|
|
"gdpr:art22",
|
|
"gdpr:art25",
|
|
"gdpr:art32",
|
|
"gdpr:art5",
|
|
"gdpr:art9",
|
|
"goal-misalignment",
|
|
"guardrails-eval",
|
|
"hallucination",
|
|
"harmbench",
|
|
"harmful",
|
|
"harmful:chemical-biological-weapons",
|
|
"harmful:child-exploitation",
|
|
"harmful:copyright-violations",
|
|
"harmful:cybercrime",
|
|
"harmful:cybercrime:malicious-code",
|
|
"harmful:graphic-content",
|
|
"harmful:harassment-bullying",
|
|
"harmful:hate",
|
|
"harmful:illegal-activities",
|
|
"harmful:illegal-drugs",
|
|
"harmful:illegal-drugs:meth",
|
|
"harmful:indiscriminate-weapons",
|
|
"harmful:insults",
|
|
"harmful:intellectual-property",
|
|
"harmful:misinformation-disinformation",
|
|
"harmful:non-violent-crime",
|
|
"harmful:privacy",
|
|
"harmful:profanity",
|
|
"harmful:radicalization",
|
|
"harmful:self-harm",
|
|
"harmful:sex-crime",
|
|
"harmful:sexual-content",
|
|
"harmful:specialized-advice",
|
|
"harmful:unsafe-practices",
|
|
"harmful:violent-crime",
|
|
"harmful:weapons:ied",
|
|
"hijacking",
|
|
"illegal-activity",
|
|
"imitation",
|
|
"indirect-prompt-injection",
|
|
"insurance",
|
|
"insurance:coverage-discrimination",
|
|
"insurance:data-disclosure",
|
|
"insurance:network-misinformation",
|
|
"insurance:phi-disclosure",
|
|
"intent",
|
|
"iso:42001",
|
|
"iso:42001:accountability",
|
|
"iso:42001:fairness",
|
|
"iso:42001:privacy",
|
|
"iso:42001:robustness",
|
|
"iso:42001:safety",
|
|
"iso:42001:security",
|
|
"iso:42001:transparency",
|
|
"mcp",
|
|
"medical",
|
|
"medical:anchoring-bias",
|
|
"medical:fda:ai-disclosure",
|
|
"medical:fda:cyber-access-control",
|
|
"medical:fda:cyber-audit-tampering",
|
|
"medical:hallucination",
|
|
"medical:incorrect-knowledge",
|
|
"medical:off-label-use",
|
|
"medical:prioritization-error",
|
|
"medical:sycophancy",
|
|
"misinformation",
|
|
"mitre:atlas",
|
|
"mitre:atlas:ai-attack-staging",
|
|
"mitre:atlas:ai-model-access",
|
|
"mitre:atlas:collection",
|
|
"mitre:atlas:command-and-control",
|
|
"mitre:atlas:credential-access",
|
|
"mitre:atlas:defense-evasion",
|
|
"mitre:atlas:discovery",
|
|
"mitre:atlas:execution",
|
|
"mitre:atlas:exfiltration",
|
|
"mitre:atlas:impact",
|
|
"mitre:atlas:initial-access",
|
|
"mitre:atlas:lateral-movement",
|
|
"mitre:atlas:ml-attack-staging",
|
|
"mitre:atlas:persistence",
|
|
"mitre:atlas:privilege-escalation",
|
|
"mitre:atlas:reconnaissance",
|
|
"mitre:atlas:resource-development",
|
|
"model-identification",
|
|
"nist:ai",
|
|
"nist:ai:measure",
|
|
"nist:ai:measure:1.1",
|
|
"nist:ai:measure:1.2",
|
|
"nist:ai:measure:2.1",
|
|
"nist:ai:measure:2.10",
|
|
"nist:ai:measure:2.11",
|
|
"nist:ai:measure:2.12",
|
|
"nist:ai:measure:2.13",
|
|
"nist:ai:measure:2.2",
|
|
"nist:ai:measure:2.3",
|
|
"nist:ai:measure:2.4",
|
|
"nist:ai:measure:2.5",
|
|
"nist:ai:measure:2.6",
|
|
"nist:ai:measure:2.7",
|
|
"nist:ai:measure:2.8",
|
|
"nist:ai:measure:2.9",
|
|
"nist:ai:measure:3.1",
|
|
"nist:ai:measure:3.2",
|
|
"nist:ai:measure:3.3",
|
|
"nist:ai:measure:4.1",
|
|
"nist:ai:measure:4.2",
|
|
"nist:ai:measure:4.3",
|
|
"off-topic",
|
|
"overreliance",
|
|
"owasp:agentic",
|
|
"owasp:agentic:asi01",
|
|
"owasp:agentic:asi02",
|
|
"owasp:agentic:asi03",
|
|
"owasp:agentic:asi04",
|
|
"owasp:agentic:asi05",
|
|
"owasp:agentic:asi06",
|
|
"owasp:agentic:asi07",
|
|
"owasp:agentic:asi08",
|
|
"owasp:agentic:asi09",
|
|
"owasp:agentic:asi10",
|
|
"owasp:api",
|
|
"owasp:api:01",
|
|
"owasp:api:02",
|
|
"owasp:api:03",
|
|
"owasp:api:04",
|
|
"owasp:api:05",
|
|
"owasp:api:06",
|
|
"owasp:api:07",
|
|
"owasp:api:08",
|
|
"owasp:api:09",
|
|
"owasp:api:10",
|
|
"owasp:llm",
|
|
"owasp:llm:01",
|
|
"owasp:llm:02",
|
|
"owasp:llm:03",
|
|
"owasp:llm:04",
|
|
"owasp:llm:05",
|
|
"owasp:llm:06",
|
|
"owasp:llm:07",
|
|
"owasp:llm:08",
|
|
"owasp:llm:09",
|
|
"owasp:llm:10",
|
|
"owasp:llm:redteam:implementation",
|
|
"owasp:llm:redteam:model",
|
|
"owasp:llm:redteam:runtime",
|
|
"owasp:llm:redteam:system",
|
|
"personal-safety",
|
|
"pharmacy",
|
|
"pharmacy:controlled-substance-compliance",
|
|
"pharmacy:dosage-calculation",
|
|
"pharmacy:drug-interaction",
|
|
"pii",
|
|
"pii:api-db",
|
|
"pii:direct",
|
|
"pii:session",
|
|
"pii:social",
|
|
"pliny",
|
|
"policy",
|
|
"politics",
|
|
"prompt-extraction",
|
|
"rag-document-exfiltration",
|
|
"rag-poisoning",
|
|
"rag-source-attribution",
|
|
"rbac",
|
|
"realestate",
|
|
"realestate:accessibility-discrimination",
|
|
"realestate:advertising-discrimination",
|
|
"realestate:discriminatory-listings",
|
|
"realestate:fair-housing-discrimination",
|
|
"realestate:lending-discrimination",
|
|
"realestate:source-of-income",
|
|
"realestate:steering",
|
|
"realestate:valuation-bias",
|
|
"reasoning-dos",
|
|
"religion",
|
|
"shell-injection",
|
|
"special-token-injection",
|
|
"sql-injection",
|
|
"ssrf",
|
|
"system-prompt-override",
|
|
"teen-safety",
|
|
"teen-safety:age-restricted-goods-and-services",
|
|
"teen-safety:dangerous-content",
|
|
"teen-safety:dangerous-roleplay",
|
|
"teen-safety:harmful-body-ideals",
|
|
"telecom",
|
|
"telecom:accessibility-violation",
|
|
"telecom:account-takeover",
|
|
"telecom:billing-misinformation",
|
|
"telecom:coverage-misinformation",
|
|
"telecom:cpni-disclosure",
|
|
"telecom:e911-misinformation",
|
|
"telecom:fraud-enablement",
|
|
"telecom:law-enforcement-request-handling",
|
|
"telecom:location-disclosure",
|
|
"telecom:porting-misinformation",
|
|
"telecom:tcpa-violation",
|
|
"telecom:unauthorized-changes",
|
|
"tool-discovery",
|
|
"tool-discovery:multi-turn",
|
|
"toxic-chat",
|
|
"toxicity",
|
|
"unsafebench",
|
|
"unverifiable-claims",
|
|
"vlguard",
|
|
"vlsu",
|
|
"wordplay",
|
|
"xstest"
|
|
]
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Name of the plugin or path to custom plugin"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"aegis",
|
|
"agentic:memory-poisoning",
|
|
"ascii-smuggling",
|
|
"beavertails",
|
|
"bfla",
|
|
"bias",
|
|
"bias:age",
|
|
"bias:disability",
|
|
"bias:gender",
|
|
"bias:race",
|
|
"bola",
|
|
"cca",
|
|
"coding-agent:all",
|
|
"coding-agent:automation-poisoning",
|
|
"coding-agent:core",
|
|
"coding-agent:delayed-ci-exfil",
|
|
"coding-agent:generated-vulnerability",
|
|
"coding-agent:network-egress-bypass",
|
|
"coding-agent:procfs-credential-read",
|
|
"coding-agent:repo-prompt-injection",
|
|
"coding-agent:sandbox-read-escape",
|
|
"coding-agent:sandbox-write-escape",
|
|
"coding-agent:secret-env-read",
|
|
"coding-agent:secret-file-read",
|
|
"coding-agent:steganographic-exfil",
|
|
"coding-agent:terminal-output-injection",
|
|
"coding-agent:verifier-sabotage",
|
|
"competitors",
|
|
"contracts",
|
|
"coppa",
|
|
"cross-session-leak",
|
|
"cyberseceval",
|
|
"data-exfil",
|
|
"debug-access",
|
|
"default",
|
|
"divergent-repetition",
|
|
"dod:ai:ethics",
|
|
"dod:ai:ethics:01",
|
|
"dod:ai:ethics:02",
|
|
"dod:ai:ethics:03",
|
|
"dod:ai:ethics:04",
|
|
"dod:ai:ethics:05",
|
|
"donotanswer",
|
|
"ecommerce",
|
|
"ecommerce:compliance-bypass",
|
|
"ecommerce:order-fraud",
|
|
"ecommerce:pci-dss",
|
|
"ecommerce:price-manipulation",
|
|
"eu:ai-act",
|
|
"eu:ai-act:annex3:biometric-id",
|
|
"eu:ai-act:annex3:critical-infrastructure",
|
|
"eu:ai-act:annex3:education",
|
|
"eu:ai-act:annex3:employment",
|
|
"eu:ai-act:annex3:essential-services",
|
|
"eu:ai-act:annex3:justice-democracy",
|
|
"eu:ai-act:annex3:law-enforcement",
|
|
"eu:ai-act:annex3:migration-border",
|
|
"eu:ai-act:art5:biometric-categorisation",
|
|
"eu:ai-act:art5:exploitation-of-vulnerabilities",
|
|
"eu:ai-act:art5:remote-biometric-id-live",
|
|
"eu:ai-act:art5:remote-biometric-id-post",
|
|
"eu:ai-act:art5:social-scoring",
|
|
"eu:ai-act:art5:subliminal-manipulation",
|
|
"excessive-agency",
|
|
"ferpa",
|
|
"financial",
|
|
"financial:calculation-error",
|
|
"financial:compliance-violation",
|
|
"financial:confidential-disclosure",
|
|
"financial:counterfactual",
|
|
"financial:data-leakage",
|
|
"financial:defamation",
|
|
"financial:hallucination",
|
|
"financial:impartiality",
|
|
"financial:japan-fiea-suitability",
|
|
"financial:misconduct",
|
|
"financial:sox-compliance",
|
|
"financial:sycophancy",
|
|
"foundation",
|
|
"gdpr",
|
|
"gdpr:art15",
|
|
"gdpr:art17",
|
|
"gdpr:art22",
|
|
"gdpr:art25",
|
|
"gdpr:art32",
|
|
"gdpr:art5",
|
|
"gdpr:art9",
|
|
"goal-misalignment",
|
|
"guardrails-eval",
|
|
"hallucination",
|
|
"harmbench",
|
|
"harmful",
|
|
"harmful:chemical-biological-weapons",
|
|
"harmful:child-exploitation",
|
|
"harmful:copyright-violations",
|
|
"harmful:cybercrime",
|
|
"harmful:cybercrime:malicious-code",
|
|
"harmful:graphic-content",
|
|
"harmful:harassment-bullying",
|
|
"harmful:hate",
|
|
"harmful:illegal-activities",
|
|
"harmful:illegal-drugs",
|
|
"harmful:illegal-drugs:meth",
|
|
"harmful:indiscriminate-weapons",
|
|
"harmful:insults",
|
|
"harmful:intellectual-property",
|
|
"harmful:misinformation-disinformation",
|
|
"harmful:non-violent-crime",
|
|
"harmful:privacy",
|
|
"harmful:profanity",
|
|
"harmful:radicalization",
|
|
"harmful:self-harm",
|
|
"harmful:sex-crime",
|
|
"harmful:sexual-content",
|
|
"harmful:specialized-advice",
|
|
"harmful:unsafe-practices",
|
|
"harmful:violent-crime",
|
|
"harmful:weapons:ied",
|
|
"hijacking",
|
|
"illegal-activity",
|
|
"imitation",
|
|
"indirect-prompt-injection",
|
|
"insurance",
|
|
"insurance:coverage-discrimination",
|
|
"insurance:data-disclosure",
|
|
"insurance:network-misinformation",
|
|
"insurance:phi-disclosure",
|
|
"intent",
|
|
"iso:42001",
|
|
"iso:42001:accountability",
|
|
"iso:42001:fairness",
|
|
"iso:42001:privacy",
|
|
"iso:42001:robustness",
|
|
"iso:42001:safety",
|
|
"iso:42001:security",
|
|
"iso:42001:transparency",
|
|
"mcp",
|
|
"medical",
|
|
"medical:anchoring-bias",
|
|
"medical:fda:ai-disclosure",
|
|
"medical:fda:cyber-access-control",
|
|
"medical:fda:cyber-audit-tampering",
|
|
"medical:hallucination",
|
|
"medical:incorrect-knowledge",
|
|
"medical:off-label-use",
|
|
"medical:prioritization-error",
|
|
"medical:sycophancy",
|
|
"misinformation",
|
|
"mitre:atlas",
|
|
"mitre:atlas:ai-attack-staging",
|
|
"mitre:atlas:ai-model-access",
|
|
"mitre:atlas:collection",
|
|
"mitre:atlas:command-and-control",
|
|
"mitre:atlas:credential-access",
|
|
"mitre:atlas:defense-evasion",
|
|
"mitre:atlas:discovery",
|
|
"mitre:atlas:execution",
|
|
"mitre:atlas:exfiltration",
|
|
"mitre:atlas:impact",
|
|
"mitre:atlas:initial-access",
|
|
"mitre:atlas:lateral-movement",
|
|
"mitre:atlas:ml-attack-staging",
|
|
"mitre:atlas:persistence",
|
|
"mitre:atlas:privilege-escalation",
|
|
"mitre:atlas:reconnaissance",
|
|
"mitre:atlas:resource-development",
|
|
"model-identification",
|
|
"nist:ai",
|
|
"nist:ai:measure",
|
|
"nist:ai:measure:1.1",
|
|
"nist:ai:measure:1.2",
|
|
"nist:ai:measure:2.1",
|
|
"nist:ai:measure:2.10",
|
|
"nist:ai:measure:2.11",
|
|
"nist:ai:measure:2.12",
|
|
"nist:ai:measure:2.13",
|
|
"nist:ai:measure:2.2",
|
|
"nist:ai:measure:2.3",
|
|
"nist:ai:measure:2.4",
|
|
"nist:ai:measure:2.5",
|
|
"nist:ai:measure:2.6",
|
|
"nist:ai:measure:2.7",
|
|
"nist:ai:measure:2.8",
|
|
"nist:ai:measure:2.9",
|
|
"nist:ai:measure:3.1",
|
|
"nist:ai:measure:3.2",
|
|
"nist:ai:measure:3.3",
|
|
"nist:ai:measure:4.1",
|
|
"nist:ai:measure:4.2",
|
|
"nist:ai:measure:4.3",
|
|
"off-topic",
|
|
"overreliance",
|
|
"owasp:agentic",
|
|
"owasp:agentic:asi01",
|
|
"owasp:agentic:asi02",
|
|
"owasp:agentic:asi03",
|
|
"owasp:agentic:asi04",
|
|
"owasp:agentic:asi05",
|
|
"owasp:agentic:asi06",
|
|
"owasp:agentic:asi07",
|
|
"owasp:agentic:asi08",
|
|
"owasp:agentic:asi09",
|
|
"owasp:agentic:asi10",
|
|
"owasp:api",
|
|
"owasp:api:01",
|
|
"owasp:api:02",
|
|
"owasp:api:03",
|
|
"owasp:api:04",
|
|
"owasp:api:05",
|
|
"owasp:api:06",
|
|
"owasp:api:07",
|
|
"owasp:api:08",
|
|
"owasp:api:09",
|
|
"owasp:api:10",
|
|
"owasp:llm",
|
|
"owasp:llm:01",
|
|
"owasp:llm:02",
|
|
"owasp:llm:03",
|
|
"owasp:llm:04",
|
|
"owasp:llm:05",
|
|
"owasp:llm:06",
|
|
"owasp:llm:07",
|
|
"owasp:llm:08",
|
|
"owasp:llm:09",
|
|
"owasp:llm:10",
|
|
"owasp:llm:redteam:implementation",
|
|
"owasp:llm:redteam:model",
|
|
"owasp:llm:redteam:runtime",
|
|
"owasp:llm:redteam:system",
|
|
"personal-safety",
|
|
"pharmacy",
|
|
"pharmacy:controlled-substance-compliance",
|
|
"pharmacy:dosage-calculation",
|
|
"pharmacy:drug-interaction",
|
|
"pii",
|
|
"pii:api-db",
|
|
"pii:direct",
|
|
"pii:session",
|
|
"pii:social",
|
|
"pliny",
|
|
"policy",
|
|
"politics",
|
|
"prompt-extraction",
|
|
"rag-document-exfiltration",
|
|
"rag-poisoning",
|
|
"rag-source-attribution",
|
|
"rbac",
|
|
"realestate",
|
|
"realestate:accessibility-discrimination",
|
|
"realestate:advertising-discrimination",
|
|
"realestate:discriminatory-listings",
|
|
"realestate:fair-housing-discrimination",
|
|
"realestate:lending-discrimination",
|
|
"realestate:source-of-income",
|
|
"realestate:steering",
|
|
"realestate:valuation-bias",
|
|
"reasoning-dos",
|
|
"religion",
|
|
"shell-injection",
|
|
"special-token-injection",
|
|
"sql-injection",
|
|
"ssrf",
|
|
"system-prompt-override",
|
|
"teen-safety",
|
|
"teen-safety:age-restricted-goods-and-services",
|
|
"teen-safety:dangerous-content",
|
|
"teen-safety:dangerous-roleplay",
|
|
"teen-safety:harmful-body-ideals",
|
|
"telecom",
|
|
"telecom:accessibility-violation",
|
|
"telecom:account-takeover",
|
|
"telecom:billing-misinformation",
|
|
"telecom:coverage-misinformation",
|
|
"telecom:cpni-disclosure",
|
|
"telecom:e911-misinformation",
|
|
"telecom:fraud-enablement",
|
|
"telecom:law-enforcement-request-handling",
|
|
"telecom:location-disclosure",
|
|
"telecom:porting-misinformation",
|
|
"telecom:tcpa-violation",
|
|
"telecom:unauthorized-changes",
|
|
"tool-discovery",
|
|
"tool-discovery:multi-turn",
|
|
"toxic-chat",
|
|
"toxicity",
|
|
"unsafebench",
|
|
"unverifiable-claims",
|
|
"vlguard",
|
|
"vlsu",
|
|
"wordplay",
|
|
"xstest"
|
|
]
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Name of the plugin"
|
|
},
|
|
"numTests": {
|
|
"description": "Number of tests to generate for this plugin",
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"config": {
|
|
"description": "Plugin-specific configuration",
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
},
|
|
"severity": {
|
|
"description": "Severity level for this plugin",
|
|
"type": "string",
|
|
"enum": ["critical", "high", "medium", "low", "informational"]
|
|
}
|
|
},
|
|
"required": ["id"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"description": "Plugins to use for redteam generation"
|
|
},
|
|
"strategies": {
|
|
"default": ["default"],
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"audio",
|
|
"authoritative-markup-injection",
|
|
"base64",
|
|
"basic",
|
|
"best-of-n",
|
|
"camelcase",
|
|
"citation",
|
|
"crescendo",
|
|
"custom",
|
|
"default",
|
|
"emoji",
|
|
"gcg",
|
|
"goat",
|
|
"hex",
|
|
"homoglyph",
|
|
"image",
|
|
"indirect-web-pwn",
|
|
"jailbreak",
|
|
"jailbreak-templates",
|
|
"jailbreak:composite",
|
|
"jailbreak:hydra",
|
|
"jailbreak:likert",
|
|
"jailbreak:meta",
|
|
"jailbreak:tree",
|
|
"layer",
|
|
"leetspeak",
|
|
"math-prompt",
|
|
"mischievous-user",
|
|
"morse",
|
|
"multilingual",
|
|
"other-encodings",
|
|
"piglatin",
|
|
"prompt-injection",
|
|
"retry",
|
|
"rot13",
|
|
"video"
|
|
]
|
|
},
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"audio",
|
|
"authoritative-markup-injection",
|
|
"base64",
|
|
"basic",
|
|
"best-of-n",
|
|
"camelcase",
|
|
"citation",
|
|
"crescendo",
|
|
"custom",
|
|
"default",
|
|
"emoji",
|
|
"gcg",
|
|
"goat",
|
|
"hex",
|
|
"homoglyph",
|
|
"image",
|
|
"indirect-web-pwn",
|
|
"jailbreak",
|
|
"jailbreak-templates",
|
|
"jailbreak:composite",
|
|
"jailbreak:hydra",
|
|
"jailbreak:likert",
|
|
"jailbreak:meta",
|
|
"jailbreak:tree",
|
|
"layer",
|
|
"leetspeak",
|
|
"math-prompt",
|
|
"mischievous-user",
|
|
"morse",
|
|
"multilingual",
|
|
"other-encodings",
|
|
"piglatin",
|
|
"prompt-injection",
|
|
"retry",
|
|
"rot13",
|
|
"video"
|
|
]
|
|
},
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
]
|
|
},
|
|
"config": {
|
|
"description": "Strategy-specific configuration",
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {}
|
|
}
|
|
},
|
|
"required": ["id"],
|
|
"additionalProperties": false
|
|
}
|
|
]
|
|
},
|
|
"description": "Strategies to use for redteam generation.\n\nDefaults to basic, jailbreak:meta, jailbreak:composite\nSupports audio, authoritative-markup-injection, base64, basic, best-of-n, camelcase, citation, crescendo, custom, default, emoji, gcg, goat, hex, homoglyph, image, indirect-web-pwn, jailbreak, jailbreak-templates, jailbreak:composite, jailbreak:hydra, jailbreak:likert, jailbreak:meta, jailbreak:tree, layer, leetspeak, math-prompt, mischievous-user, morse, multilingual, other-encodings, piglatin, prompt-injection, retry, rot13, video"
|
|
},
|
|
"maxConcurrency": {
|
|
"description": "Maximum number of concurrent API calls",
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"maxCharsPerMessage": {
|
|
"description": "Maximum number of characters allowed per generated user message",
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"delay": {
|
|
"description": "Delay in milliseconds between plugin API calls",
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"excludeTargetOutputFromAgenticAttackGeneration": {
|
|
"description": "Whether to exclude target output from the agentific attack generation process",
|
|
"type": "boolean"
|
|
},
|
|
"tracing": {
|
|
"description": "Tracing defaults applied to all strategies unless overridden",
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema0"
|
|
}
|
|
]
|
|
},
|
|
"graderExamples": {
|
|
"description": "Global grading examples that apply to all plugins",
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"output": {
|
|
"type": "string"
|
|
},
|
|
"pass": {
|
|
"type": "boolean"
|
|
},
|
|
"score": {
|
|
"type": "number"
|
|
},
|
|
"reason": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["output", "pass", "score", "reason"],
|
|
"additionalProperties": false
|
|
}
|
|
}
|
|
},
|
|
"required": ["plugins", "strategies"],
|
|
"additionalProperties": false,
|
|
"definitions": {
|
|
"__schema0": {
|
|
"type": "object",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"includeInAttack": {
|
|
"type": "boolean"
|
|
},
|
|
"includeInGrading": {
|
|
"type": "boolean"
|
|
},
|
|
"includeInternalSpans": {
|
|
"type": "boolean"
|
|
},
|
|
"maxSpans": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"maxDepth": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"maxRetries": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"retryDelayMs": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"spanFilter": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"sanitizeAttributes": {
|
|
"type": "boolean"
|
|
},
|
|
"strategies": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema0"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
}
|
|
}
|
|
},
|
|
"__schema39": {
|
|
"type": "boolean"
|
|
},
|
|
"__schema40": {
|
|
"type": "object",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"failOnReceiverStartFailure": {
|
|
"type": "boolean"
|
|
},
|
|
"commandToolNames": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"otlp": {
|
|
"type": "object",
|
|
"properties": {
|
|
"http": {
|
|
"type": "object",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"port": {
|
|
"type": "number"
|
|
},
|
|
"host": {
|
|
"type": "string"
|
|
},
|
|
"acceptFormats": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"enum": ["protobuf", "json"]
|
|
}
|
|
},
|
|
"redactAttributes": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"required": ["enabled", "port", "host", "acceptFormats"],
|
|
"additionalProperties": false
|
|
},
|
|
"grpc": {
|
|
"type": "object",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"port": {
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": ["enabled", "port"],
|
|
"additionalProperties": false
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"storage": {
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["sqlite"]
|
|
},
|
|
"retentionDays": {
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": ["type", "retentionDays"],
|
|
"additionalProperties": false
|
|
},
|
|
"forwarding": {
|
|
"type": "object",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"endpoint": {
|
|
"type": "string"
|
|
},
|
|
"headers": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"required": ["enabled", "endpoint"],
|
|
"additionalProperties": false
|
|
}
|
|
},
|
|
"required": ["enabled"],
|
|
"additionalProperties": false
|
|
},
|
|
"__schema41": {
|
|
"type": "object",
|
|
"properties": {
|
|
"cache": {
|
|
"type": "boolean"
|
|
},
|
|
"delay": {
|
|
"type": "number"
|
|
},
|
|
"generateSuggestions": {
|
|
"type": "boolean"
|
|
},
|
|
"suggestionsCount": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 50
|
|
},
|
|
"interactiveProviders": {
|
|
"type": "boolean"
|
|
},
|
|
"maxConcurrency": {
|
|
"type": "number"
|
|
},
|
|
"progressCallback": {},
|
|
"repeat": {
|
|
"type": "number"
|
|
},
|
|
"showProgressBar": {
|
|
"type": "boolean"
|
|
},
|
|
"timeoutMs": {
|
|
"type": "number"
|
|
},
|
|
"maxEvalTimeMs": {
|
|
"type": "number"
|
|
},
|
|
"isRedteam": {
|
|
"type": "boolean"
|
|
},
|
|
"silent": {
|
|
"type": "boolean"
|
|
},
|
|
"filterRange": {
|
|
"$ref": "#/definitions/__schema42"
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema42": {
|
|
"type": "string"
|
|
},
|
|
"__schema43": {
|
|
"type": "object",
|
|
"properties": {
|
|
"description": {
|
|
"type": "string"
|
|
},
|
|
"prompts": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"providers": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"output": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"maxConcurrency": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"repeat": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"delay": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"vars": {
|
|
"type": "string"
|
|
},
|
|
"tests": {
|
|
"type": "string"
|
|
},
|
|
"config": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"assertions": {
|
|
"type": "string"
|
|
},
|
|
"modelOutputs": {
|
|
"type": "string"
|
|
},
|
|
"verbose": {
|
|
"type": "boolean"
|
|
},
|
|
"grader": {
|
|
"type": "string"
|
|
},
|
|
"tableCellMaxLength": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"write": {
|
|
"type": "boolean"
|
|
},
|
|
"cache": {
|
|
"type": "boolean"
|
|
},
|
|
"table": {
|
|
"type": "boolean"
|
|
},
|
|
"share": {
|
|
"type": "boolean"
|
|
},
|
|
"noShare": {
|
|
"type": "boolean"
|
|
},
|
|
"progressBar": {
|
|
"type": "boolean"
|
|
},
|
|
"watch": {
|
|
"type": "boolean"
|
|
},
|
|
"filterErrorsOnly": {
|
|
"type": "string"
|
|
},
|
|
"filterFailing": {
|
|
"type": "string"
|
|
},
|
|
"filterFailingOnly": {
|
|
"type": "string"
|
|
},
|
|
"filterFirstN": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"filterMetadata": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"filterPattern": {
|
|
"type": "string"
|
|
},
|
|
"filterPrompts": {
|
|
"type": "string"
|
|
},
|
|
"filterProviders": {
|
|
"type": "string"
|
|
},
|
|
"filterRange": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema42"
|
|
}
|
|
]
|
|
},
|
|
"filterSample": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"filterSampleSeed": {
|
|
"type": "integer",
|
|
"minimum": -9007199254740991,
|
|
"maximum": 9007199254740991
|
|
},
|
|
"filterTargets": {
|
|
"type": "string"
|
|
},
|
|
"var": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"tags": {
|
|
"type": "object",
|
|
"propertyNames": {
|
|
"type": "string"
|
|
},
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"generateSuggestions": {
|
|
"type": "boolean"
|
|
},
|
|
"suggestionsCount": {
|
|
"type": "integer",
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 50
|
|
},
|
|
"promptPrefix": {
|
|
"type": "string"
|
|
},
|
|
"promptSuffix": {
|
|
"type": "string"
|
|
},
|
|
"retryErrors": {
|
|
"type": "boolean"
|
|
},
|
|
"envPath": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"extension": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"__schema44": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/definitions/__schema3"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|