chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
+942
@@ -0,0 +1,942 @@
|
||||
import fs from 'node:fs';
|
||||
|
||||
import * as yaml from 'js-yaml';
|
||||
|
||||
const HTTP_METHODS = new Set(['get', 'post', 'put', 'patch', 'delete']);
|
||||
const PROMPT_FIELDS = new Set(['message', 'prompt', 'q', 'query', 'question', 'input', 'text']);
|
||||
|
||||
function usage(message) {
|
||||
if (message) {
|
||||
console.error(message);
|
||||
}
|
||||
console.error(
|
||||
'Usage: node openapi-operation-to-config.mjs --spec openapi.yaml --operation-id op --base-url-env API_BASE_URL [--token-env API_TOKEN] [--auth-header Authorization] [--auth-prefix Bearer|none|custom] [--label label] [--output promptfooconfig.yaml]',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index];
|
||||
const value = argv[index + 1];
|
||||
if (!key?.startsWith('--') || value === undefined) {
|
||||
usage(`Invalid argument near ${key ?? '<end>'}`);
|
||||
}
|
||||
args[key.slice(2)] = value;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function asRecord(value, context) {
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
usage(`${context} must be an object`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveRef(document, schema, seen = new Set()) {
|
||||
if (!schema?.$ref) {
|
||||
return schema;
|
||||
}
|
||||
if (!schema.$ref.startsWith('#/')) {
|
||||
usage(`Only local OpenAPI refs are supported: ${schema.$ref}`);
|
||||
}
|
||||
if (seen.has(schema.$ref)) {
|
||||
usage(`Circular OpenAPI ref detected: ${schema.$ref}`);
|
||||
}
|
||||
const activeRefs = new Set(seen);
|
||||
activeRefs.add(schema.$ref);
|
||||
const parts = schema.$ref
|
||||
.replace(/^#\//, '')
|
||||
.split('/')
|
||||
.map((part) => part.replaceAll('~1', '/').replaceAll('~0', '~'));
|
||||
let current = document;
|
||||
for (const part of parts) {
|
||||
const currentRecord = asRecord(current, schema.$ref);
|
||||
if (!(part in currentRecord)) {
|
||||
usage(`OpenAPI ref not found: ${schema.$ref}`);
|
||||
}
|
||||
current = currentRecord[part];
|
||||
}
|
||||
return resolveRef(document, current, activeRefs);
|
||||
}
|
||||
|
||||
function getJsonMediaEntry(document, content) {
|
||||
const contentRecord = asRecord(content || {}, 'content');
|
||||
const exactJson = contentRecord['application/json'];
|
||||
const entry = exactJson
|
||||
? ['application/json', exactJson]
|
||||
: Object.entries(contentRecord).find(([mediaType]) => isJsonMediaType(mediaType));
|
||||
return entry
|
||||
? { mediaType: entry[0], media: asRecord(resolveRef(document, entry[1]), entry[0]) }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function getRequestMediaEntry(document, content) {
|
||||
const contentRecord = asRecord(content || {}, 'content');
|
||||
const jsonEntry = getJsonMediaEntry(document, contentRecord);
|
||||
if (jsonEntry) {
|
||||
return jsonEntry;
|
||||
}
|
||||
const formEntry = Object.entries(contentRecord).find(([mediaType]) =>
|
||||
isFormUrlEncodedMediaType(mediaType),
|
||||
);
|
||||
if (formEntry) {
|
||||
return {
|
||||
mediaType: formEntry[0],
|
||||
media: asRecord(resolveRef(document, formEntry[1]), formEntry[0]),
|
||||
};
|
||||
}
|
||||
const multipartEntry = Object.entries(contentRecord).find(([mediaType]) =>
|
||||
isMultipartMediaType(mediaType),
|
||||
);
|
||||
if (multipartEntry) {
|
||||
return {
|
||||
mediaType: multipartEntry[0],
|
||||
media: asRecord(resolveRef(document, multipartEntry[1]), multipartEntry[0]),
|
||||
};
|
||||
}
|
||||
const textEntry = Object.entries(contentRecord).find(([mediaType]) => isTextMediaType(mediaType));
|
||||
return textEntry
|
||||
? { mediaType: textEntry[0], media: asRecord(resolveRef(document, textEntry[1]), textEntry[0]) }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function isJsonMediaType(mediaType) {
|
||||
const normalized = mediaType.split(';', 1)[0].trim().toLowerCase();
|
||||
return (
|
||||
normalized === 'application/json' ||
|
||||
normalized.endsWith('/json') ||
|
||||
normalized.endsWith('+json')
|
||||
);
|
||||
}
|
||||
|
||||
function isFormUrlEncodedMediaType(mediaType) {
|
||||
return mediaType.split(';', 1)[0].trim().toLowerCase() === 'application/x-www-form-urlencoded';
|
||||
}
|
||||
|
||||
function isMultipartMediaType(mediaType) {
|
||||
return mediaType.split(';', 1)[0].trim().toLowerCase() === 'multipart/form-data';
|
||||
}
|
||||
|
||||
function isTextMediaType(mediaType) {
|
||||
return mediaType.split(';', 1)[0].trim().toLowerCase().startsWith('text/');
|
||||
}
|
||||
|
||||
function schemaFromMedia(document, media) {
|
||||
return media?.schema ? resolveRef(document, media.schema) : undefined;
|
||||
}
|
||||
|
||||
function findOperation(document, operationId) {
|
||||
const paths = asRecord(document.paths, 'paths');
|
||||
for (const [pathTemplate, pathItem] of Object.entries(paths)) {
|
||||
const pathItemRecord = asRecord(pathItem, pathTemplate);
|
||||
for (const [method, operation] of Object.entries(pathItemRecord)) {
|
||||
if (
|
||||
HTTP_METHODS.has(method) &&
|
||||
asRecord(operation, `${method} ${pathTemplate}`).operationId === operationId
|
||||
) {
|
||||
return {
|
||||
pathTemplate,
|
||||
pathItem: pathItemRecord,
|
||||
method: method.toUpperCase(),
|
||||
operation: asRecord(operation, operationId),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
usage(`Operation not found: ${operationId}`);
|
||||
}
|
||||
|
||||
function effectiveParameters(document, pathItem, operation) {
|
||||
const parameters = [
|
||||
...(Array.isArray(pathItem.parameters) ? pathItem.parameters : []),
|
||||
...(Array.isArray(operation.parameters) ? operation.parameters : []),
|
||||
].map((parameter) => asRecord(resolveRef(document, parameter), 'parameter'));
|
||||
const byLocationAndName = new Map();
|
||||
for (const parameter of parameters) {
|
||||
byLocationAndName.set(`${parameter.in}:${parameter.name}`, parameter);
|
||||
}
|
||||
return [...byLocationAndName.values()];
|
||||
}
|
||||
|
||||
function schemaProperties(document, schema) {
|
||||
const resolved = asRecord(resolveRef(document, schema || { type: 'object' }), 'schema');
|
||||
const variantProperties = schemaVariant(resolved)
|
||||
? schemaProperties(document, schemaVariant(resolved))
|
||||
: {};
|
||||
const allOfProperties = Array.isArray(resolved.allOf)
|
||||
? Object.assign({}, ...resolved.allOf.map((part) => schemaProperties(document, part)))
|
||||
: {};
|
||||
return {
|
||||
...variantProperties,
|
||||
...allOfProperties,
|
||||
...asRecord(resolved.properties || {}, 'schema.properties'),
|
||||
};
|
||||
}
|
||||
|
||||
function schemaRequired(document, schema) {
|
||||
const resolved = asRecord(resolveRef(document, schema || { type: 'object' }), 'schema');
|
||||
const variantRequired = schemaVariant(resolved)
|
||||
? schemaRequired(document, schemaVariant(resolved))
|
||||
: [];
|
||||
const allOfRequired = Array.isArray(resolved.allOf)
|
||||
? resolved.allOf.flatMap((part) => schemaRequired(document, part))
|
||||
: [];
|
||||
const required = Array.isArray(resolved.required) ? resolved.required : [];
|
||||
return [...new Set([...variantRequired, ...allOfRequired, ...required])];
|
||||
}
|
||||
|
||||
function schemaVariant(schema) {
|
||||
if (Array.isArray(schema.oneOf) && schema.oneOf.length > 0) {
|
||||
return schema.oneOf[0];
|
||||
}
|
||||
if (Array.isArray(schema.anyOf) && schema.anyOf.length > 0) {
|
||||
return schema.anyOf[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function schemaType(schema) {
|
||||
if (Array.isArray(schema.type)) {
|
||||
return schema.type.find((type) => type !== 'null');
|
||||
}
|
||||
if (schema.type) {
|
||||
return schema.type;
|
||||
}
|
||||
if (schema.properties) {
|
||||
return 'object';
|
||||
}
|
||||
if (schema.items) {
|
||||
return 'array';
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isScalarSchema(document, schema) {
|
||||
if (!schema || typeof schema !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, schema), 'schema');
|
||||
const variant = schemaVariant(resolved);
|
||||
if (variant) {
|
||||
return isScalarSchema(document, variant);
|
||||
}
|
||||
const type = schemaType(resolved);
|
||||
return ['boolean', 'integer', 'number', 'string'].includes(type);
|
||||
}
|
||||
|
||||
function schemaArrayItems(document, schema) {
|
||||
if (!schema || typeof schema !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, schema), 'schema');
|
||||
const variant = schemaVariant(resolved);
|
||||
if (variant) {
|
||||
return schemaArrayItems(document, variant);
|
||||
}
|
||||
if (Array.isArray(resolved.allOf)) {
|
||||
for (const part of resolved.allOf) {
|
||||
const items = schemaArrayItems(document, part);
|
||||
if (items) {
|
||||
return items;
|
||||
}
|
||||
}
|
||||
}
|
||||
return schemaType(resolved) === 'array' ? resolveRef(document, resolved.items || {}) : undefined;
|
||||
}
|
||||
|
||||
function stringSample(schema, name) {
|
||||
const format = typeof schema.format === 'string' ? schema.format.toLowerCase() : '';
|
||||
if (format === 'email' || name.endsWith('_email') || name === 'email') {
|
||||
return 'user@example.com';
|
||||
}
|
||||
if (format === 'uuid') {
|
||||
return '00000000-0000-4000-8000-000000000000';
|
||||
}
|
||||
if (format === 'uri' || format === 'url' || format === 'uri-reference') {
|
||||
return 'https://example.test/resource';
|
||||
}
|
||||
if (format === 'date-time') {
|
||||
return '2026-04-17T00:00:00Z';
|
||||
}
|
||||
if (format === 'date') {
|
||||
return '2026-04-17';
|
||||
}
|
||||
if (format === 'time') {
|
||||
return '12:00:00';
|
||||
}
|
||||
return sampleValue(name);
|
||||
}
|
||||
|
||||
function orderedProperties(document, schema) {
|
||||
const resolved = asRecord(schema || { type: 'object' }, 'schema');
|
||||
const properties = schemaProperties(document, resolved);
|
||||
const required = schemaRequired(document, resolved);
|
||||
return [...required, ...Object.keys(properties).filter((name) => !required.includes(name))];
|
||||
}
|
||||
|
||||
function schemaHasFlag(document, schema, flag) {
|
||||
if (!schema || typeof schema !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, schema), 'schema');
|
||||
if (resolved[flag] === true) {
|
||||
return true;
|
||||
}
|
||||
const variant = schemaVariant(resolved);
|
||||
if (variant && schemaHasFlag(document, variant, flag)) {
|
||||
return true;
|
||||
}
|
||||
return Array.isArray(resolved.allOf)
|
||||
? resolved.allOf.some((part) => schemaHasFlag(document, part, flag))
|
||||
: false;
|
||||
}
|
||||
|
||||
function isReadOnlySchema(document, schema) {
|
||||
return schemaHasFlag(document, schema, 'readOnly');
|
||||
}
|
||||
|
||||
function isWriteOnlySchema(document, schema) {
|
||||
return schemaHasFlag(document, schema, 'writeOnly');
|
||||
}
|
||||
|
||||
function isMultipartFileSchema(document, schema) {
|
||||
if (!schema || typeof schema !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, schema), 'schema');
|
||||
const variant = schemaVariant(resolved);
|
||||
if (variant && isMultipartFileSchema(document, variant)) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
Array.isArray(resolved.allOf) &&
|
||||
resolved.allOf.some((part) => isMultipartFileSchema(document, part))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const type = schemaType(resolved);
|
||||
const format = typeof resolved.format === 'string' ? resolved.format.toLowerCase() : '';
|
||||
if (type === 'array') {
|
||||
return isMultipartFileSchema(document, resolved.items);
|
||||
}
|
||||
return type === 'string' && ['binary', 'base64'].includes(format);
|
||||
}
|
||||
|
||||
function responseOutputField(document, properties) {
|
||||
const responseProperties = Object.fromEntries(
|
||||
Object.entries(properties).filter(([_name, schema]) => !isWriteOnlySchema(document, schema)),
|
||||
);
|
||||
for (const candidate of [
|
||||
'output',
|
||||
'answer',
|
||||
'response',
|
||||
'result',
|
||||
'text',
|
||||
'content',
|
||||
'message',
|
||||
]) {
|
||||
if (candidate in responseProperties) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return Object.keys(responseProperties)[0];
|
||||
}
|
||||
|
||||
function responseAccessor(base, field) {
|
||||
return /^[A-Za-z_$][0-9A-Za-z_$]*$/.test(field)
|
||||
? `${base}.${field}`
|
||||
: `${base}[${JSON.stringify(field)}]`;
|
||||
}
|
||||
|
||||
function successResponse(document, operation) {
|
||||
const responses = asRecord(operation.responses || {}, 'responses');
|
||||
const status = responses['200']
|
||||
? '200'
|
||||
: Object.keys(responses)
|
||||
.filter((candidate) => /^2\d\d$/.test(candidate))
|
||||
.sort()[0];
|
||||
return {
|
||||
status,
|
||||
response: status
|
||||
? asRecord(resolveRef(document, responses[status]), `responses.${status}`)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
function sampleValue(name) {
|
||||
if (PROMPT_FIELDS.has(name)) {
|
||||
return 'Say exactly PONG.';
|
||||
}
|
||||
if (name.endsWith('_id')) {
|
||||
return `sample-${name.replaceAll('_', '-')}`;
|
||||
}
|
||||
return `sample-${name}`;
|
||||
}
|
||||
|
||||
function isPlainRecord(value) {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function firstExample(document, examples) {
|
||||
if (Array.isArray(examples) && examples.length > 0) {
|
||||
return examples[0];
|
||||
}
|
||||
if (!isPlainRecord(examples)) {
|
||||
return undefined;
|
||||
}
|
||||
const first = Object.values(examples)[0];
|
||||
if (!isPlainRecord(first)) {
|
||||
return undefined;
|
||||
}
|
||||
const resolved = resolveRef(document, first);
|
||||
return isPlainRecord(resolved) && 'value' in resolved ? resolved.value : undefined;
|
||||
}
|
||||
|
||||
function objectSample(document, object, context) {
|
||||
if (!isPlainRecord(object)) {
|
||||
return undefined;
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, object), context);
|
||||
if ('example' in resolved && resolved.example !== null) {
|
||||
return resolved.example;
|
||||
}
|
||||
return firstExample(document, resolved.examples);
|
||||
}
|
||||
|
||||
function schemaSample(document, schema, name, depth = 0) {
|
||||
if (PROMPT_FIELDS.has(name)) {
|
||||
return 'Say exactly PONG.';
|
||||
}
|
||||
if (!schema || typeof schema !== 'object') {
|
||||
return sampleValue(name);
|
||||
}
|
||||
const resolved = asRecord(resolveRef(document, schema), `${name} schema`);
|
||||
if ('const' in resolved && resolved.const !== null) {
|
||||
return resolved.const;
|
||||
}
|
||||
if ('example' in resolved && resolved.example !== null) {
|
||||
return resolved.example;
|
||||
}
|
||||
if ('default' in resolved && resolved.default !== null) {
|
||||
return resolved.default;
|
||||
}
|
||||
if (
|
||||
Array.isArray(resolved.examples) &&
|
||||
resolved.examples.length > 0 &&
|
||||
resolved.examples[0] !== null
|
||||
) {
|
||||
return resolved.examples[0];
|
||||
}
|
||||
if (Array.isArray(resolved.enum) && resolved.enum.length > 0) {
|
||||
return resolved.enum[0];
|
||||
}
|
||||
const variant = schemaVariant(resolved);
|
||||
if (variant) {
|
||||
return schemaSample(document, variant, name, depth + 1);
|
||||
}
|
||||
if (depth > 4) {
|
||||
return sampleValue(name);
|
||||
}
|
||||
const type = schemaType(resolved);
|
||||
if (type === 'integer' || type === 'number') {
|
||||
return 1;
|
||||
}
|
||||
if (type === 'boolean') {
|
||||
return true;
|
||||
}
|
||||
if (type === 'array') {
|
||||
return [schemaSample(document, resolved.items, name, depth + 1)];
|
||||
}
|
||||
if (type === 'object') {
|
||||
const properties = schemaProperties(document, resolved);
|
||||
const names = orderedProperties(document, resolved).slice(0, 6);
|
||||
return Object.fromEntries(
|
||||
names.map((propertyName) => [
|
||||
propertyName,
|
||||
schemaSample(document, properties[propertyName], propertyName, depth + 1),
|
||||
]),
|
||||
);
|
||||
}
|
||||
if (type === 'string' || typeof resolved.format === 'string') {
|
||||
return stringSample(resolved, name);
|
||||
}
|
||||
return sampleValue(name);
|
||||
}
|
||||
|
||||
function parameterSample(document, parameter) {
|
||||
if (PROMPT_FIELDS.has(parameter.name)) {
|
||||
return 'Say exactly PONG.';
|
||||
}
|
||||
const example = objectSample(document, parameter, `${parameter.name} parameter`);
|
||||
return example === undefined
|
||||
? schemaSample(
|
||||
document,
|
||||
parameter.schema,
|
||||
parameter.in === 'header' ? varName(parameter.name) : parameter.name,
|
||||
)
|
||||
: example;
|
||||
}
|
||||
|
||||
function inputTemplate(name) {
|
||||
return PROMPT_FIELDS.has(name) ? '{{prompt}}' : `{{${varName(name)}}}`;
|
||||
}
|
||||
|
||||
function encodedInputTemplate(name) {
|
||||
return PROMPT_FIELDS.has(name) ? '{{prompt | urlencode}}' : `{{${varName(name)} | urlencode}}`;
|
||||
}
|
||||
|
||||
function formEncodedBody(fields) {
|
||||
return fields
|
||||
.map((name) => `${encodeURIComponent(name)}=${encodedInputTemplate(name)}`)
|
||||
.join('&');
|
||||
}
|
||||
|
||||
function multipartPart(document, name, schema) {
|
||||
if (isMultipartFileSchema(document, schema)) {
|
||||
return {
|
||||
kind: 'file',
|
||||
name,
|
||||
filename: `promptfoo-${varName(name)}.pdf`,
|
||||
source: {
|
||||
type: 'generated',
|
||||
generator: 'basic-document',
|
||||
format: 'pdf',
|
||||
text: `Promptfoo generated document for ${inputTemplate(name)}.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
return { kind: 'field', name, value: inputTemplate(name) };
|
||||
}
|
||||
|
||||
function varName(name) {
|
||||
return name
|
||||
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2')
|
||||
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
|
||||
.replaceAll(/[^A-Za-z0-9_]/g, '_')
|
||||
.replace(/^(\d)/, '_$1')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// Parameters with these names (or these suffixes) are treated as credentials
|
||||
// even when they are modeled as plain header/query/cookie parameters rather
|
||||
// than as OpenAPI securitySchemes. Copying their literal `example` values into
|
||||
// the generated config could leak real tokens, so the helper forces
|
||||
// `{{env.<UPPER_NAME>}}` placeholders for them instead.
|
||||
const CREDENTIAL_PARAM_NAMES = new Set([
|
||||
'authorization',
|
||||
'bearer',
|
||||
'token',
|
||||
'access_token',
|
||||
'auth_token',
|
||||
'api_key',
|
||||
'apikey',
|
||||
'x_api_key',
|
||||
'x_auth_token',
|
||||
'x_access_token',
|
||||
'secret',
|
||||
'password',
|
||||
'csrf_token',
|
||||
'xsrf_token',
|
||||
'session',
|
||||
'sessionid',
|
||||
'session_id',
|
||||
'sid',
|
||||
]);
|
||||
const CREDENTIAL_SUFFIX_REGEX =
|
||||
/(^|_)(api_key|apikey|auth_token|access_token|bearer|password|secret|token|authorization)$/;
|
||||
function isCredentialParamName(name) {
|
||||
const normalized = varName(name);
|
||||
return CREDENTIAL_PARAM_NAMES.has(normalized) || CREDENTIAL_SUFFIX_REGEX.test(normalized);
|
||||
}
|
||||
function credentialEnvName(paramName) {
|
||||
return varName(paramName).toUpperCase();
|
||||
}
|
||||
function credentialPlaceholder(paramName) {
|
||||
return `{{env.${credentialEnvName(paramName)}}}`;
|
||||
}
|
||||
|
||||
function authFromScheme(scheme) {
|
||||
if (scheme.type === 'apiKey' && typeof scheme.name === 'string') {
|
||||
if (scheme.in === 'header') {
|
||||
return { location: 'header', name: scheme.name, prefix: 'none' };
|
||||
}
|
||||
if (scheme.in === 'query') {
|
||||
return { location: 'query', name: scheme.name, prefix: 'none' };
|
||||
}
|
||||
if (scheme.in === 'cookie') {
|
||||
return { location: 'cookie', name: scheme.name, prefix: 'none' };
|
||||
}
|
||||
}
|
||||
if (scheme.type === 'http' && typeof scheme.scheme === 'string') {
|
||||
const httpScheme = scheme.scheme.toLowerCase();
|
||||
if (httpScheme === 'bearer') {
|
||||
return { location: 'header', name: 'Authorization', prefix: 'Bearer' };
|
||||
}
|
||||
if (httpScheme === 'basic') {
|
||||
return { location: 'header', name: 'Authorization', prefix: 'Basic' };
|
||||
}
|
||||
}
|
||||
if (scheme.type === 'oauth2' || scheme.type === 'openIdConnect') {
|
||||
return { location: 'header', name: 'Authorization', prefix: 'Bearer' };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function inferAuths(document, operation) {
|
||||
const components = asRecord(document.components || {}, 'components');
|
||||
const securitySchemes = asRecord(components.securitySchemes || {}, 'components.securitySchemes');
|
||||
const hasOperationSecurity = Array.isArray(operation.security);
|
||||
const hasDocumentSecurity = Array.isArray(document.security);
|
||||
const securityRequirements = hasOperationSecurity
|
||||
? operation.security
|
||||
: hasDocumentSecurity
|
||||
? document.security
|
||||
: undefined;
|
||||
|
||||
if (!securityRequirements) {
|
||||
return undefined;
|
||||
}
|
||||
if (securityRequirements.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let partiallySupportedAuths;
|
||||
for (const requirement of securityRequirements) {
|
||||
const requirementRecord = asRecord(requirement, 'security requirement');
|
||||
const requirementNames = Object.keys(requirementRecord);
|
||||
if (requirementNames.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const auths = [];
|
||||
for (const name of requirementNames) {
|
||||
if (!(name in securitySchemes)) {
|
||||
continue;
|
||||
}
|
||||
const scheme = asRecord(
|
||||
resolveRef(document, securitySchemes[name]),
|
||||
`securitySchemes.${name}`,
|
||||
);
|
||||
const auth = authFromScheme(scheme);
|
||||
if (!auth) {
|
||||
continue;
|
||||
}
|
||||
auths.push(auth);
|
||||
}
|
||||
if (auths.length === requirementNames.length) {
|
||||
return auths;
|
||||
}
|
||||
if (auths.length > 0 && !partiallySupportedAuths) {
|
||||
partiallySupportedAuths = auths;
|
||||
}
|
||||
}
|
||||
|
||||
return partiallySupportedAuths ?? [];
|
||||
}
|
||||
|
||||
function authValue(tokenEnv, prefix) {
|
||||
return prefix === 'none' ? `{{env.${tokenEnv}}}` : `${prefix} {{env.${tokenEnv}}}`;
|
||||
}
|
||||
|
||||
function appendCookieHeader(headers, name, value) {
|
||||
const cookie = `${name}=${value}`;
|
||||
headers.Cookie = headers.Cookie ? `${headers.Cookie}; ${cookie}` : cookie;
|
||||
}
|
||||
|
||||
function authConfigs(args, document, operation) {
|
||||
if (args['auth-header']) {
|
||||
const prefix = args['auth-prefix'] ?? 'Bearer';
|
||||
return {
|
||||
auths: [{ location: 'header', name: args['auth-header'], prefix }],
|
||||
};
|
||||
}
|
||||
const inferred = inferAuths(document, operation);
|
||||
const auths =
|
||||
inferred === undefined
|
||||
? [{ location: 'header', name: 'Authorization', prefix: 'Bearer' }]
|
||||
: inferred;
|
||||
return {
|
||||
auths: auths.map((auth) => ({
|
||||
...auth,
|
||||
// --auth-prefix only overrides schemes that already carry a prefix (Bearer/Basic/etc.);
|
||||
// API-key-style auths with prefix 'none' stay untouched so the emitted header value is raw.
|
||||
prefix: args['auth-prefix'] && auth.prefix !== 'none' ? args['auth-prefix'] : auth.prefix,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (!args.spec || !args['operation-id'] || !args['base-url-env']) {
|
||||
usage('Missing --spec, --operation-id, or --base-url-env');
|
||||
}
|
||||
|
||||
const document = yaml.load(fs.readFileSync(args.spec, 'utf8'), {
|
||||
// Preserve js-yaml v4's support for YAML merge keys in user-supplied specs.
|
||||
schema: yaml.CORE_SCHEMA.withTags(yaml.mergeTag),
|
||||
});
|
||||
const { pathTemplate, pathItem, method, operation } = findOperation(
|
||||
asRecord(document, 'OpenAPI document'),
|
||||
args['operation-id'],
|
||||
);
|
||||
const pathVars = [...pathTemplate.matchAll(/\{([^}]+)\}/g)].map((match) => match[1]);
|
||||
const promptPath = pathTemplate.replace(/\{([^}]+)\}/g, (_match, name) =>
|
||||
encodedInputTemplate(name),
|
||||
);
|
||||
const parameters = effectiveParameters(document, pathItem, operation);
|
||||
const parameterSamples = Object.fromEntries(
|
||||
parameters
|
||||
.filter((parameter) => typeof parameter.name === 'string')
|
||||
.map((parameter) => [varName(parameter.name), parameterSample(document, parameter)]),
|
||||
);
|
||||
const queryFields = parameters
|
||||
.filter((parameter) => parameter.in === 'query')
|
||||
.map((parameter) => parameter.name)
|
||||
.filter((name) => typeof name === 'string');
|
||||
const headerFields = parameters
|
||||
.filter((parameter) => parameter.in === 'header')
|
||||
.map((parameter) => parameter.name)
|
||||
.filter((name) => typeof name === 'string');
|
||||
const cookieFields = parameters
|
||||
.filter((parameter) => parameter.in === 'cookie')
|
||||
.map((parameter) => parameter.name)
|
||||
.filter((name) => typeof name === 'string');
|
||||
const requestBody = asRecord(resolveRef(document, operation.requestBody || {}), 'requestBody');
|
||||
const requestMediaEntry = getRequestMediaEntry(
|
||||
document,
|
||||
asRecord(requestBody.content || {}, 'requestBody.content'),
|
||||
);
|
||||
const requestMedia = requestMediaEntry?.media;
|
||||
const requestExample = objectSample(document, requestMedia, 'requestBody application/json');
|
||||
const requestExampleIsArray = Array.isArray(requestExample);
|
||||
const requestExampleArrayItemIsObject = requestExampleIsArray && isPlainRecord(requestExample[0]);
|
||||
const requestExampleIsScalar =
|
||||
requestExample !== undefined && !requestExampleIsArray && !isPlainRecord(requestExample);
|
||||
const requestSchema = schemaFromMedia(document, requestMedia);
|
||||
const requestIsFormUrlEncoded = isFormUrlEncodedMediaType(requestMediaEntry?.mediaType || '');
|
||||
const requestIsMultipart = isMultipartMediaType(requestMediaEntry?.mediaType || '');
|
||||
const requestIsText = isTextMediaType(requestMediaEntry?.mediaType || '');
|
||||
const requestArrayItemSchema =
|
||||
requestSchema && !requestIsFormUrlEncoded && !requestIsMultipart && !requestIsText
|
||||
? schemaArrayItems(document, requestSchema)
|
||||
: undefined;
|
||||
const requestArrayItemProperties = requestArrayItemSchema
|
||||
? schemaProperties(document, requestArrayItemSchema)
|
||||
: {};
|
||||
const requestArrayItemIsObject = Object.keys(requestArrayItemProperties).length > 0;
|
||||
const requestBodyIsArray = Boolean(
|
||||
!requestIsFormUrlEncoded &&
|
||||
!requestIsMultipart &&
|
||||
!requestIsText &&
|
||||
(requestArrayItemSchema || requestExampleIsArray),
|
||||
);
|
||||
const requestBodyArrayItemIsObject = requestArrayItemSchema
|
||||
? requestArrayItemIsObject
|
||||
: requestExampleArrayItemIsObject;
|
||||
const effectiveRequestSchema = requestArrayItemSchema || requestSchema;
|
||||
const requestBodyIsScalar =
|
||||
(!effectiveRequestSchema && requestExampleIsScalar) ||
|
||||
Boolean(
|
||||
effectiveRequestSchema &&
|
||||
!requestBodyIsArray &&
|
||||
!requestIsFormUrlEncoded &&
|
||||
!requestIsMultipart &&
|
||||
isScalarSchema(document, effectiveRequestSchema),
|
||||
);
|
||||
const requestProperties =
|
||||
requestArrayItemSchema && !requestArrayItemIsObject
|
||||
? { message: requestArrayItemSchema }
|
||||
: requestBodyIsScalar && effectiveRequestSchema
|
||||
? { message: effectiveRequestSchema }
|
||||
: effectiveRequestSchema
|
||||
? schemaProperties(document, effectiveRequestSchema)
|
||||
: {};
|
||||
const requestExampleFields = isPlainRecord(requestExample)
|
||||
? requestExample
|
||||
: Array.isArray(requestExample) && isPlainRecord(requestExample[0])
|
||||
? requestExample[0]
|
||||
: Array.isArray(requestExample) && requestExample.length > 0
|
||||
? { message: requestExample[0] }
|
||||
: requestExampleIsScalar
|
||||
? { message: requestExample }
|
||||
: {};
|
||||
const bodyFields = effectiveRequestSchema
|
||||
? (requestIsText || requestBodyIsScalar || (requestArrayItemSchema && !requestArrayItemIsObject)
|
||||
? ['message']
|
||||
: orderedProperties(document, effectiveRequestSchema)
|
||||
).filter(
|
||||
(name) =>
|
||||
!pathVars.includes(name) &&
|
||||
!queryFields.includes(name) &&
|
||||
!isReadOnlySchema(document, requestProperties[name]),
|
||||
)
|
||||
: Object.keys(requestExampleFields).filter(
|
||||
(name) => !pathVars.includes(name) && !queryFields.includes(name),
|
||||
);
|
||||
const { status: responseStatus, response } = successResponse(document, operation);
|
||||
const responseContent = asRecord(
|
||||
response.content || {},
|
||||
`responses.${responseStatus ?? 'success'}.content`,
|
||||
);
|
||||
const responseMediaEntry = getJsonMediaEntry(document, responseContent);
|
||||
const responseMedia = responseMediaEntry?.media;
|
||||
const responseExample = objectSample(document, responseMedia, 'response application/json');
|
||||
const responseExampleIsArray = Array.isArray(responseExample);
|
||||
const responseExampleFields = isPlainRecord(responseExample)
|
||||
? responseExample
|
||||
: responseExampleIsArray && isPlainRecord(responseExample[0])
|
||||
? responseExample[0]
|
||||
: {};
|
||||
const responseSchema = schemaFromMedia(document, responseMedia);
|
||||
const responseArrayItemSchema = responseSchema
|
||||
? schemaArrayItems(document, responseSchema)
|
||||
: undefined;
|
||||
const responseIsArray = Boolean(responseArrayItemSchema || responseExampleIsArray);
|
||||
const responseProperties = responseArrayItemSchema
|
||||
? schemaProperties(document, responseArrayItemSchema)
|
||||
: responseSchema
|
||||
? schemaProperties(document, responseSchema)
|
||||
: responseExampleFields;
|
||||
const responseField = responseOutputField(document, responseProperties);
|
||||
|
||||
const credentialHeaderFields = headerFields.filter(isCredentialParamName);
|
||||
const credentialQueryFields = queryFields.filter(isCredentialParamName);
|
||||
const credentialCookieFields = cookieFields.filter(isCredentialParamName);
|
||||
const credentialParamNames = new Set([
|
||||
...credentialHeaderFields,
|
||||
...credentialQueryFields,
|
||||
...credentialCookieFields,
|
||||
]);
|
||||
const headers =
|
||||
requestMediaEntry && !requestIsMultipart
|
||||
? { 'Content-Type': requestMediaEntry?.mediaType || 'application/json' }
|
||||
: {};
|
||||
for (const name of headerFields) {
|
||||
headers[name] = credentialParamNames.has(name)
|
||||
? credentialPlaceholder(name)
|
||||
: `{{${varName(name)}}}`;
|
||||
}
|
||||
for (const name of cookieFields) {
|
||||
appendCookieHeader(
|
||||
headers,
|
||||
name,
|
||||
credentialParamNames.has(name) ? credentialPlaceholder(name) : `{{${varName(name)}}}`,
|
||||
);
|
||||
}
|
||||
const queryParams = Object.fromEntries(
|
||||
queryFields.map((name) => [
|
||||
name,
|
||||
credentialParamNames.has(name) ? credentialPlaceholder(name) : inputTemplate(name),
|
||||
]),
|
||||
);
|
||||
if (args['token-env']) {
|
||||
for (const auth of authConfigs(args, document, operation).auths) {
|
||||
const value = authValue(args['token-env'], auth.prefix);
|
||||
if (auth.location === 'query') {
|
||||
queryParams[auth.name] = value;
|
||||
} else if (auth.location === 'cookie') {
|
||||
appendCookieHeader(headers, auth.name, value);
|
||||
} else {
|
||||
headers[auth.name] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const providerConfig = {
|
||||
url: `{{env.${args['base-url-env']}}}${promptPath}`,
|
||||
method,
|
||||
stateful: false,
|
||||
};
|
||||
if (Object.keys(headers).length > 0) {
|
||||
providerConfig.headers = headers;
|
||||
}
|
||||
if (bodyFields.length > 0) {
|
||||
if (requestIsMultipart) {
|
||||
providerConfig.multipart = {
|
||||
parts: bodyFields.map((name) => multipartPart(document, name, requestProperties[name])),
|
||||
};
|
||||
} else if (requestIsText || requestBodyIsScalar) {
|
||||
providerConfig.body = inputTemplate(bodyFields[0]);
|
||||
} else if (requestBodyIsArray) {
|
||||
providerConfig.body = [
|
||||
requestBodyArrayItemIsObject
|
||||
? Object.fromEntries(bodyFields.map((name) => [name, inputTemplate(name)]))
|
||||
: inputTemplate(bodyFields[0]),
|
||||
];
|
||||
} else {
|
||||
providerConfig.body = requestIsFormUrlEncoded
|
||||
? formEncodedBody(bodyFields)
|
||||
: Object.fromEntries(bodyFields.map((name) => [name, inputTemplate(name)]));
|
||||
}
|
||||
}
|
||||
if (Object.keys(queryParams).length > 0) {
|
||||
providerConfig.queryParams = queryParams;
|
||||
}
|
||||
if (responseField) {
|
||||
providerConfig.transformResponse = responseAccessor(
|
||||
responseIsArray ? 'json[0]' : 'json',
|
||||
responseField,
|
||||
);
|
||||
}
|
||||
|
||||
const nonCredentialQueryFields = queryFields.filter((name) => !credentialParamNames.has(name));
|
||||
const nonCredentialHeaderFields = headerFields.filter((name) => !credentialParamNames.has(name));
|
||||
const nonCredentialCookieFields = cookieFields.filter((name) => !credentialParamNames.has(name));
|
||||
const varNames = [...pathVars, ...nonCredentialQueryFields, ...bodyFields]
|
||||
.filter((name) => inputTemplate(name) !== '{{prompt}}')
|
||||
.map(varName);
|
||||
varNames.push(...nonCredentialHeaderFields.map(varName), ...nonCredentialCookieFields.map(varName));
|
||||
if (!varNames.includes('message')) {
|
||||
varNames.push('message');
|
||||
}
|
||||
const vars = Object.fromEntries([...new Set(varNames)].map((name) => [name, sampleValue(name)]));
|
||||
for (const [name, value] of Object.entries(parameterSamples)) {
|
||||
if (name in vars) {
|
||||
vars[name] = value;
|
||||
}
|
||||
}
|
||||
for (const name of bodyFields) {
|
||||
const key = varName(name);
|
||||
if (key in vars) {
|
||||
vars[key] = PROMPT_FIELDS.has(name)
|
||||
? schemaSample(document, requestProperties[name], name)
|
||||
: (requestExampleFields[name] ?? schemaSample(document, requestProperties[name], name));
|
||||
}
|
||||
}
|
||||
// The canned "Say exactly PONG." prompt is only meaningful when the request
|
||||
// carries a prompt field (path/query/body) — otherwise the message never reaches
|
||||
// the target and `contains: PONG` fails by construction. Fall back to a safer
|
||||
// response-shape assertion.
|
||||
const promptReachesTarget =
|
||||
pathVars.some((name) => PROMPT_FIELDS.has(name)) ||
|
||||
queryFields.some((name) => PROMPT_FIELDS.has(name)) ||
|
||||
bodyFields.some((name) => PROMPT_FIELDS.has(name)) ||
|
||||
(bodyFields.length > 0 && (requestIsText || requestBodyIsScalar));
|
||||
const responseIsJson = Boolean(responseMediaEntry);
|
||||
const smokeAssert = promptReachesTarget
|
||||
? [{ type: 'contains', value: 'PONG' }]
|
||||
: responseIsJson
|
||||
? [{ type: 'is-json' }]
|
||||
: [{ type: 'javascript', value: "typeof output === 'string' && output.length > 0" }];
|
||||
const config = {
|
||||
description: args.description || `Provider setup generated from ${args['operation-id']}`,
|
||||
prompts: ['{{message}}'],
|
||||
providers: [{ id: 'https', label: args.label || args['operation-id'], config: providerConfig }],
|
||||
tests: [
|
||||
{
|
||||
description: promptReachesTarget
|
||||
? `${args['operation-id']} returns text`
|
||||
: `${args['operation-id']} responds successfully`,
|
||||
vars,
|
||||
assert: smokeAssert,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const output = `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n${yaml.dump(config, { lineWidth: 100, noRefs: true })}`;
|
||||
if (args.output) {
|
||||
fs.writeFileSync(args.output, output);
|
||||
} else {
|
||||
process.stdout.write(output);
|
||||
}
|
||||
Reference in New Issue
Block a user