chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
import React, { useState } from 'react';
|
||||
|
||||
import PluginTable from './PluginTable';
|
||||
|
||||
const ApplicationVulnerabilityDropdown = () => {
|
||||
const [openSections, setOpenSections] = useState<Set<string>>(
|
||||
new Set(['security', 'privacy', 'criminal', 'harmful', 'misinformation']),
|
||||
);
|
||||
|
||||
const toggleSection = (section: string) => {
|
||||
setOpenSections((prev) => {
|
||||
const newSections = new Set(prev);
|
||||
if (newSections.has(section)) {
|
||||
newSections.delete(section);
|
||||
} else {
|
||||
newSections.add(section);
|
||||
}
|
||||
return newSections;
|
||||
});
|
||||
};
|
||||
|
||||
const renderSection = (
|
||||
title: string,
|
||||
id: string,
|
||||
description: string,
|
||||
vulnerabilityType: string,
|
||||
) => (
|
||||
<div key={id}>
|
||||
<h3
|
||||
id={`vulnerability-${id}`}
|
||||
onClick={() => toggleSection(id)}
|
||||
style={{ cursor: 'pointer' }}
|
||||
>
|
||||
<a href={`#vulnerability-${id}`} style={{ textDecoration: 'none', color: 'inherit' }}>
|
||||
{title} {openSections.has(id) ? '▼' : '▶'}
|
||||
</a>
|
||||
</h3>
|
||||
<p>{description}</p>
|
||||
{openSections.has(id) && (
|
||||
<PluginTable
|
||||
vulnerabilityType={vulnerabilityType}
|
||||
shouldRenderCategory={false}
|
||||
showApplicationTypes={true}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
const sections = [
|
||||
{
|
||||
title: 'Security Vulnerabilities',
|
||||
id: 'security',
|
||||
description: 'Technical security risks in application context.',
|
||||
vulnerabilityType: 'security',
|
||||
},
|
||||
{
|
||||
title: 'Privacy Violations',
|
||||
id: 'privacy',
|
||||
description: 'Privacy risks in application context.',
|
||||
vulnerabilityType: 'privacy',
|
||||
},
|
||||
{
|
||||
title: 'Criminal Activities',
|
||||
id: 'criminal',
|
||||
description: 'Criminal risks in application context.',
|
||||
vulnerabilityType: 'criminal',
|
||||
},
|
||||
{
|
||||
title: 'Harmful Content',
|
||||
id: 'harmful',
|
||||
description: 'Harmful content risks in application context.',
|
||||
vulnerabilityType: 'harmful',
|
||||
},
|
||||
{
|
||||
title: 'Misinformation and Misuse',
|
||||
id: 'misinformation',
|
||||
description: 'Misinformation risks in application context.',
|
||||
vulnerabilityType: 'misinformation and misuse',
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<div>
|
||||
{sections.map((section) =>
|
||||
renderSection(section.title, section.id, section.description, section.vulnerabilityType),
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ApplicationVulnerabilityDropdown;
|
||||
@@ -0,0 +1,194 @@
|
||||
import React from 'react';
|
||||
|
||||
import { PLUGIN_CATEGORIES, PLUGINS } from './data/plugins';
|
||||
import { getCategoryAnchor } from './utils/categoryUtils';
|
||||
|
||||
import type { Plugin, PluginCategory } from './data/plugins';
|
||||
|
||||
type GroupedPlugins = Record<PluginCategory, Plugin[]>;
|
||||
|
||||
interface PluginTableProps {
|
||||
vulnerabilityType?: string;
|
||||
shouldRenderCategory?: boolean;
|
||||
shouldRenderDescription?: boolean;
|
||||
shouldRenderPluginId?: boolean;
|
||||
shouldGroupByCategory?: boolean;
|
||||
showApplicationTypes?: boolean;
|
||||
showRemoteStatus?: boolean;
|
||||
}
|
||||
|
||||
const styles = {
|
||||
table: {
|
||||
width: '100%',
|
||||
borderCollapse: 'collapse' as const,
|
||||
tableLayout: 'fixed' as const,
|
||||
},
|
||||
th: {
|
||||
padding: '12px 8px',
|
||||
textAlign: 'left' as const,
|
||||
whiteSpace: 'nowrap' as const,
|
||||
borderBottom: '2px solid var(--ifm-table-border-color)',
|
||||
overflow: 'hidden',
|
||||
textOverflow: 'ellipsis',
|
||||
},
|
||||
td: {
|
||||
padding: '8px',
|
||||
borderBottom: '1px solid var(--ifm-table-border-color)',
|
||||
verticalAlign: 'top' as const,
|
||||
overflow: 'hidden',
|
||||
textOverflow: 'ellipsis',
|
||||
},
|
||||
// Column-specific widths
|
||||
columns: {
|
||||
category: { width: '15%', fontWeight: 'bold' },
|
||||
name: { width: '20%', fontWeight: 'bold' },
|
||||
description: { width: '40%', whiteSpace: 'normal' as const },
|
||||
pluginId: { width: '15%' },
|
||||
indicator: { width: '5%', textAlign: 'center' as const },
|
||||
},
|
||||
code: {
|
||||
whiteSpace: 'nowrap' as const,
|
||||
fontSize: '0.9em',
|
||||
},
|
||||
link: {
|
||||
display: 'block',
|
||||
overflow: 'hidden',
|
||||
textOverflow: 'ellipsis',
|
||||
whiteSpace: 'nowrap' as const,
|
||||
},
|
||||
};
|
||||
|
||||
const PluginTable = ({
|
||||
vulnerabilityType,
|
||||
shouldRenderCategory = true,
|
||||
shouldRenderDescription = true,
|
||||
shouldRenderPluginId = true,
|
||||
shouldGroupByCategory = false,
|
||||
showApplicationTypes = false,
|
||||
showRemoteStatus = false,
|
||||
}: PluginTableProps) => {
|
||||
let filteredPlugins = PLUGINS;
|
||||
|
||||
// Apply filters if specified
|
||||
if (vulnerabilityType) {
|
||||
filteredPlugins = filteredPlugins.filter(
|
||||
(plugin) => plugin.vulnerabilityType === vulnerabilityType,
|
||||
);
|
||||
}
|
||||
|
||||
// Group plugins by category if needed
|
||||
const groupedPlugins: GroupedPlugins = {} as GroupedPlugins;
|
||||
if (shouldGroupByCategory) {
|
||||
for (const category of PLUGIN_CATEGORIES) {
|
||||
groupedPlugins[category] = filteredPlugins
|
||||
.filter((plugin) => plugin.category === category)
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<table style={styles.table}>
|
||||
<thead>
|
||||
<tr>
|
||||
{shouldRenderCategory && shouldGroupByCategory && (
|
||||
<th style={{ ...styles.th, ...styles.columns.category }}>Category</th>
|
||||
)}
|
||||
<th style={{ ...styles.th, ...styles.columns.name }}>Plugin Name</th>
|
||||
{shouldRenderDescription && (
|
||||
<th style={{ ...styles.th, ...styles.columns.description }}>Description</th>
|
||||
)}
|
||||
{shouldRenderPluginId && (
|
||||
<th style={{ ...styles.th, ...styles.columns.pluginId }}>Plugin ID</th>
|
||||
)}
|
||||
{showApplicationTypes && (
|
||||
<>
|
||||
<th style={{ ...styles.th, ...styles.columns.indicator }}>RAG</th>
|
||||
<th style={{ ...styles.th, ...styles.columns.indicator }}>Agent</th>
|
||||
<th style={{ ...styles.th, ...styles.columns.indicator }}>Chatbot</th>
|
||||
</>
|
||||
)}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{shouldGroupByCategory
|
||||
? Object.entries(groupedPlugins).map(([category, categoryPlugins]) => (
|
||||
<React.Fragment key={category}>
|
||||
{categoryPlugins.map((plugin, index) => (
|
||||
<tr key={plugin.pluginId}>
|
||||
{index === 0 && shouldRenderCategory && (
|
||||
<td
|
||||
rowSpan={categoryPlugins.length}
|
||||
style={{ ...styles.td, ...styles.columns.category }}
|
||||
id={getCategoryAnchor(plugin.category).slice(1)}
|
||||
>
|
||||
{plugin.category}
|
||||
</td>
|
||||
)}
|
||||
<td style={{ ...styles.td, ...styles.columns.name }}>
|
||||
<a href={plugin.link} style={styles.link} title={plugin.name}>
|
||||
{plugin.name}
|
||||
</a>
|
||||
</td>
|
||||
{shouldRenderDescription && (
|
||||
<td style={{ ...styles.td, ...styles.columns.description }}>
|
||||
{plugin.description}
|
||||
{showRemoteStatus && plugin.isRemote && (
|
||||
<span title="Uses remote inference"> 🌐</span>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{shouldRenderPluginId && (
|
||||
<td style={{ ...styles.td, ...styles.columns.pluginId }}>
|
||||
<code style={styles.code}>{plugin.pluginId}</code>
|
||||
</td>
|
||||
)}
|
||||
{showApplicationTypes && (
|
||||
<>
|
||||
<td style={{ ...styles.td, ...styles.columns.indicator }}>
|
||||
{plugin.applicationTypes?.rag ? '🚨' : '✅'}
|
||||
</td>
|
||||
<td style={{ ...styles.td, ...styles.columns.indicator }}>
|
||||
{plugin.applicationTypes?.agent ? '🚨' : '✅'}
|
||||
</td>
|
||||
<td style={{ ...styles.td, ...styles.columns.indicator }}>
|
||||
{plugin.applicationTypes?.chat ? '🚨' : '✅'}
|
||||
</td>
|
||||
</>
|
||||
)}
|
||||
</tr>
|
||||
))}
|
||||
</React.Fragment>
|
||||
))
|
||||
: filteredPlugins.map((plugin) => (
|
||||
<tr key={plugin.pluginId}>
|
||||
<td style={styles.td}>
|
||||
<a href={plugin.link}>{plugin.name}</a>
|
||||
</td>
|
||||
{shouldRenderDescription && (
|
||||
<td style={styles.td}>
|
||||
{plugin.description}
|
||||
{showRemoteStatus && plugin.isRemote && (
|
||||
<span title="Uses remote inference"> 🌐</span>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{shouldRenderPluginId && (
|
||||
<td style={styles.td}>
|
||||
<code style={styles.code}>{plugin.pluginId}</code>
|
||||
</td>
|
||||
)}
|
||||
{showApplicationTypes && (
|
||||
<>
|
||||
<td style={styles.td}>{plugin.applicationTypes?.rag ? '🚨' : '✅'}</td>
|
||||
<td style={styles.td}>{plugin.applicationTypes?.agent ? '🚨' : '✅'}</td>
|
||||
<td style={styles.td}>{plugin.applicationTypes?.chat ? '🚨' : '✅'}</td>
|
||||
</>
|
||||
)}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
);
|
||||
};
|
||||
|
||||
export default PluginTable;
|
||||
@@ -0,0 +1,150 @@
|
||||
import React from 'react';
|
||||
|
||||
import { strategies } from './data/strategies';
|
||||
|
||||
type GroupedStrategies = Record<string, typeof strategies>;
|
||||
|
||||
const categoryOrder: (typeof strategies)[number]['category'][] = [
|
||||
'Static (Single-Turn)',
|
||||
'Dynamic (Single-Turn)',
|
||||
'Multi-turn',
|
||||
'Regression',
|
||||
'Custom',
|
||||
];
|
||||
|
||||
const groupedStrategies = strategies.reduce<GroupedStrategies>((acc, strategy) => {
|
||||
if (!acc[strategy.category]) {
|
||||
acc[strategy.category] = [];
|
||||
}
|
||||
acc[strategy.category].push(strategy);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
Object.values(groupedStrategies).forEach((categoryStrategies) => {
|
||||
categoryStrategies.sort((a, b) => a.displayName.localeCompare(b.displayName));
|
||||
});
|
||||
|
||||
const RecommendedBadge = () => (
|
||||
<span
|
||||
style={{
|
||||
backgroundColor: 'var(--ifm-color-primary)',
|
||||
color: 'white',
|
||||
padding: '2px 6px',
|
||||
borderRadius: '4px',
|
||||
fontSize: '0.75em',
|
||||
marginLeft: '8px',
|
||||
verticalAlign: 'middle',
|
||||
}}
|
||||
>
|
||||
Recommended
|
||||
</span>
|
||||
);
|
||||
|
||||
interface StrategyTableProps {
|
||||
shouldRenderCategory?: boolean;
|
||||
shouldRenderStrategy?: boolean;
|
||||
shouldRenderDescription?: boolean;
|
||||
shouldRenderLongDescription?: boolean;
|
||||
shouldRenderCost?: boolean;
|
||||
shouldRenderAsrIncrease?: boolean;
|
||||
showRemoteStatus?: boolean;
|
||||
}
|
||||
|
||||
const StrategyTable = ({
|
||||
shouldRenderCategory = true,
|
||||
shouldRenderStrategy = true,
|
||||
shouldRenderDescription = true,
|
||||
shouldRenderLongDescription = true,
|
||||
shouldRenderCost = true,
|
||||
shouldRenderAsrIncrease = true,
|
||||
showRemoteStatus = false,
|
||||
}: StrategyTableProps) => {
|
||||
return (
|
||||
<div className="strategy-table-wrapper">
|
||||
<table className="strategy-table">
|
||||
<thead>
|
||||
<tr>
|
||||
{shouldRenderCategory && (
|
||||
<th style={{ verticalAlign: 'top', textAlign: 'left' }}>Category</th>
|
||||
)}
|
||||
{shouldRenderStrategy && <th>Strategy</th>}
|
||||
{shouldRenderDescription && <th>Description</th>}
|
||||
{shouldRenderLongDescription && <th>Details</th>}
|
||||
{shouldRenderCost && <th>Cost</th>}
|
||||
{shouldRenderAsrIncrease && (
|
||||
<th>
|
||||
ASR Increase
|
||||
<sup
|
||||
style={{ cursor: 'help' }}
|
||||
title="Relative increase in Attack Success Rate compared to running the same test without any strategy"
|
||||
>
|
||||
*
|
||||
</sup>
|
||||
</th>
|
||||
)}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{categoryOrder.map((category) => {
|
||||
const categoryStrategies = groupedStrategies[category] || [];
|
||||
return (
|
||||
<React.Fragment key={category}>
|
||||
{categoryStrategies.map((strategy, index) => (
|
||||
<tr key={strategy.strategy} className={index % 2 === 0 ? 'even' : 'odd'}>
|
||||
{index === 0 && shouldRenderCategory && (
|
||||
<td
|
||||
rowSpan={categoryStrategies.length}
|
||||
style={{ verticalAlign: 'top' }}
|
||||
className="category-cell"
|
||||
>
|
||||
{category}
|
||||
</td>
|
||||
)}
|
||||
{shouldRenderStrategy && (
|
||||
<td className="strategy-cell">
|
||||
{strategy.link ? (
|
||||
<a href={strategy.link} className="strategy-link">
|
||||
{strategy.displayName}
|
||||
{strategy.recommended && <RecommendedBadge />}
|
||||
</a>
|
||||
) : (
|
||||
<>
|
||||
{strategy.displayName}
|
||||
{strategy.recommended && <RecommendedBadge />}
|
||||
</>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{shouldRenderDescription && (
|
||||
<td>
|
||||
{strategy.description}
|
||||
{showRemoteStatus && strategy.isRemote && (
|
||||
<span title="Uses remote inference"> 🌐</span>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{shouldRenderLongDescription && (
|
||||
<td className="details-cell">{strategy.longDescription}</td>
|
||||
)}
|
||||
{shouldRenderCost && <td className="metric-cell">{strategy.cost}</td>}
|
||||
{shouldRenderAsrIncrease && (
|
||||
<td className="metric-cell">{strategy.asrIncrease}</td>
|
||||
)}
|
||||
</tr>
|
||||
))}
|
||||
</React.Fragment>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{shouldRenderAsrIncrease && (
|
||||
<div style={{ fontSize: '0.8em', marginTop: '8px', color: 'var(--ifm-color-gray-600)' }}>
|
||||
* ASR Increase: Relative increase in Attack Success Rate compared to running the same test
|
||||
without any strategy
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default StrategyTable;
|
||||
@@ -0,0 +1,50 @@
|
||||
import React from 'react';
|
||||
|
||||
import { PLUGINS } from './data/plugins';
|
||||
|
||||
interface VulnerabilityCategoriesTablesProps {
|
||||
label: string;
|
||||
vulnerabilityType?: string;
|
||||
}
|
||||
|
||||
const VulnerabilityCategoriesTables = ({
|
||||
label,
|
||||
vulnerabilityType,
|
||||
}: VulnerabilityCategoriesTablesProps) => {
|
||||
// If vulnerabilityType is provided, only show plugins of that type
|
||||
const relevantPlugins = vulnerabilityType
|
||||
? PLUGINS.filter((plugin) => plugin.vulnerabilityType === vulnerabilityType)
|
||||
: PLUGINS.filter((plugin) => plugin.label === label);
|
||||
|
||||
// Sort the plugins alphabetically by name
|
||||
const sortedPlugins = relevantPlugins.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
return (
|
||||
<div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Description</th>
|
||||
<th>Plugin ID</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{sortedPlugins.map((plugin) => (
|
||||
<tr key={plugin.pluginId}>
|
||||
<td>
|
||||
<a href={plugin.link}>{plugin.name}</a>
|
||||
</td>
|
||||
<td>{plugin.description}</td>
|
||||
<td>
|
||||
<code>{plugin.pluginId}</code>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default VulnerabilityCategoriesTables;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,381 @@
|
||||
export interface Strategy {
|
||||
category: string;
|
||||
categoryLink?: string;
|
||||
strategy: string;
|
||||
displayName: string;
|
||||
description: string;
|
||||
longDescription: string;
|
||||
cost: string;
|
||||
asrIncrease: string;
|
||||
link?: string;
|
||||
recommended?: boolean;
|
||||
isRemote?: boolean;
|
||||
}
|
||||
|
||||
export const strategies: Strategy[] = [
|
||||
{
|
||||
category: 'Custom',
|
||||
strategy: 'custom',
|
||||
displayName: 'Custom Strategies',
|
||||
description: 'User-defined transformations',
|
||||
longDescription:
|
||||
'Allows creation of custom red team testing approaches by programmatically transforming test cases using JavaScript',
|
||||
cost: 'Variable',
|
||||
asrIncrease: 'Variable',
|
||||
link: '/docs/red-team/strategies/custom/',
|
||||
},
|
||||
{
|
||||
category: 'Custom',
|
||||
strategy: 'custom-strategy',
|
||||
displayName: 'Custom Strategy',
|
||||
description: 'Custom prompt-based multi-turn strategy',
|
||||
longDescription:
|
||||
'Write natural language instructions to create powerful multi-turn red team strategies. No coding required.',
|
||||
cost: 'Variable',
|
||||
asrIncrease: 'Variable',
|
||||
link: '/docs/red-team/strategies/custom-strategy/',
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'best-of-n',
|
||||
displayName: 'Best-of-N',
|
||||
description: 'Parallel sampling attack',
|
||||
longDescription:
|
||||
'Tests multiple variations in parallel using the Best-of-N technique from Anthropic research',
|
||||
cost: 'High',
|
||||
asrIncrease: '40-60%',
|
||||
link: '/docs/red-team/strategies/best-of-n/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'citation',
|
||||
displayName: 'Citation',
|
||||
description: 'Academic framing',
|
||||
longDescription:
|
||||
'Tests vulnerability to academic authority bias by framing harmful requests in research contexts',
|
||||
cost: 'Medium',
|
||||
asrIncrease: '40-60%',
|
||||
link: '/docs/red-team/strategies/citation/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'authoritative-markup-injection',
|
||||
displayName: 'Authoritative Markup Injection',
|
||||
description: 'Structured format authority',
|
||||
longDescription:
|
||||
'Tests vulnerability to authoritative formatting by embedding prompts in structured markup that exploits trust in formatted content',
|
||||
cost: 'Medium',
|
||||
asrIncrease: '40-60%',
|
||||
link: '/docs/red-team/strategies/authoritative-markup-injection/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'jailbreak:composite',
|
||||
displayName: 'Composite Jailbreaks',
|
||||
description: 'Combined techniques',
|
||||
longDescription:
|
||||
'Chains multiple jailbreak techniques from research papers to create more sophisticated attacks',
|
||||
cost: 'Medium',
|
||||
asrIncrease: '60-80%',
|
||||
link: '/docs/red-team/strategies/composite-jailbreaks/',
|
||||
recommended: true,
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'gcg',
|
||||
displayName: 'GCG',
|
||||
description: 'Gradient-based optimization',
|
||||
longDescription:
|
||||
'Implements the Greedy Coordinate Gradient attack method for finding adversarial prompts using gradient-based search techniques',
|
||||
cost: 'High',
|
||||
asrIncrease: '0-10%',
|
||||
link: '/docs/red-team/strategies/gcg/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'jailbreak',
|
||||
displayName: 'Jailbreak',
|
||||
description: 'Lightweight iterative refinement',
|
||||
longDescription:
|
||||
'Uses an LLM-as-a-Judge to iteratively refine prompts until they bypass security controls',
|
||||
cost: 'High',
|
||||
asrIncrease: '60-80%',
|
||||
link: '/docs/red-team/strategies/iterative/',
|
||||
recommended: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'jailbreak:meta',
|
||||
displayName: 'Meta-Agent Jailbreaks',
|
||||
description: 'Strategic taxonomy builder',
|
||||
longDescription:
|
||||
'Builds custom attack taxonomies and learns from all attempts using persistent strategic memory to choose which attack types work against your specific target',
|
||||
cost: 'High',
|
||||
asrIncrease: '70-90%',
|
||||
link: '/docs/red-team/strategies/meta/',
|
||||
recommended: true,
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'jailbreak:likert',
|
||||
displayName: 'Likert-based Jailbreaks',
|
||||
description: 'Academic evaluation framework',
|
||||
longDescription:
|
||||
'Leverages academic evaluation frameworks and Likert scales to frame harmful requests within research contexts',
|
||||
cost: 'Medium',
|
||||
asrIncrease: '40-60%',
|
||||
link: '/docs/red-team/strategies/likert/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'math-prompt',
|
||||
displayName: 'Math Prompt',
|
||||
description: 'Mathematical encoding',
|
||||
longDescription:
|
||||
'Tests resilience against mathematical notation-based attacks using set theory and abstract algebra',
|
||||
cost: 'Medium',
|
||||
asrIncrease: '40-60%',
|
||||
link: '/docs/red-team/strategies/math-prompt/',
|
||||
},
|
||||
{
|
||||
category: 'Dynamic (Single-Turn)',
|
||||
strategy: 'jailbreak:tree',
|
||||
displayName: 'Tree-based',
|
||||
description: 'Branching attack paths',
|
||||
longDescription:
|
||||
'Creates a tree of attack variations based on the Tree of Attacks research paper',
|
||||
cost: 'High',
|
||||
asrIncrease: '60-80%',
|
||||
link: '/docs/red-team/strategies/tree/',
|
||||
},
|
||||
{
|
||||
category: 'Multi-turn',
|
||||
strategy: 'crescendo',
|
||||
displayName: 'Crescendo',
|
||||
description: 'Gradual escalation',
|
||||
longDescription:
|
||||
'Gradually escalates prompt harm over multiple turns while using backtracking to optimize attack paths',
|
||||
cost: 'High',
|
||||
asrIncrease: '70-90%',
|
||||
link: '/docs/red-team/strategies/multi-turn/',
|
||||
},
|
||||
{
|
||||
category: 'Multi-turn',
|
||||
strategy: 'goat',
|
||||
displayName: 'GOAT',
|
||||
description: 'Generative Offensive Agent Tester',
|
||||
longDescription:
|
||||
'Uses a Generative Offensive Agent Tester to dynamically generate multi-turn conversations',
|
||||
cost: 'High',
|
||||
asrIncrease: '70-90%',
|
||||
link: '/docs/red-team/strategies/goat/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Multi-turn',
|
||||
strategy: 'jailbreak:hydra',
|
||||
displayName: 'Hydra Multi-turn',
|
||||
description: 'Adaptive multi-turn branching',
|
||||
longDescription:
|
||||
'Adaptive multi-turn jailbreak agent that pivots across branches with persistent scan-wide memory to uncover hidden vulnerabilities',
|
||||
cost: 'High',
|
||||
asrIncrease: '70-90%',
|
||||
link: '/docs/red-team/strategies/hydra/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Multi-turn',
|
||||
strategy: 'mischievous-user',
|
||||
displayName: 'Mischievous User',
|
||||
description: 'Mischievous user conversations',
|
||||
longDescription: 'Simulates a multi-turn conversation between a mischievous user and an agent',
|
||||
cost: 'High',
|
||||
asrIncrease: '10-20%',
|
||||
link: '/docs/red-team/strategies/mischievous-user/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'video',
|
||||
displayName: 'Video Encoding',
|
||||
description: 'Text-to-video encoding bypass',
|
||||
longDescription:
|
||||
'Tests handling of text embedded in videos and encoded as base64 to potentially bypass text-based content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/video/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'image',
|
||||
displayName: 'Image Encoding',
|
||||
description: 'Text-to-image encoding bypass',
|
||||
longDescription:
|
||||
'Tests handling of text embedded in images and encoded as base64 to potentially bypass text-based content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/image/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'audio',
|
||||
displayName: 'Audio Encoding',
|
||||
description: 'Text-to-speech encoding bypass',
|
||||
longDescription:
|
||||
'Tests handling of text converted to speech audio and encoded as base64 to potentially bypass text-based content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/audio/',
|
||||
isRemote: true,
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'base64',
|
||||
displayName: 'Base64',
|
||||
description: 'Base64 encoding bypass',
|
||||
longDescription:
|
||||
'Tests detection and handling of Base64-encoded malicious payloads to bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/base64/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'hex',
|
||||
displayName: 'Hex',
|
||||
description: 'Hex encoding bypass',
|
||||
longDescription:
|
||||
'Tests detection and handling of hex-encoded malicious payloads to bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/hex/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'homoglyph',
|
||||
displayName: 'Homoglyph',
|
||||
description: 'Unicode confusable characters',
|
||||
longDescription:
|
||||
'Tests detection and handling of text with homoglyphs (visually similar Unicode characters) to bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/homoglyph/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'basic',
|
||||
displayName: 'Basic',
|
||||
description: 'Plugin-generated test cases',
|
||||
longDescription:
|
||||
'Controls whether original plugin-generated test cases are included without any strategies applied',
|
||||
cost: 'Low',
|
||||
asrIncrease: 'None',
|
||||
link: '/docs/red-team/strategies/basic/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'leetspeak',
|
||||
displayName: 'Leetspeak',
|
||||
description: 'Character substitution',
|
||||
longDescription:
|
||||
'Tests handling of leetspeak-encoded malicious content by replacing standard letters with numbers or special characters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/leetspeak/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'jailbreak-templates',
|
||||
displayName: 'Jailbreak Templates',
|
||||
description: 'Static jailbreak templates',
|
||||
longDescription:
|
||||
'Tests LLM resistance to known jailbreak techniques (DAN, Skeleton Key, etc.) using static templates. Note: Does not cover modern prompt injection techniques.',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/jailbreak-templates/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'rot13',
|
||||
displayName: 'ROT13',
|
||||
description: 'Letter rotation encoding',
|
||||
longDescription:
|
||||
'Tests handling of ROT13-encoded malicious payloads by rotating each letter 13 positions in the alphabet',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/rot13/',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'morse',
|
||||
displayName: 'Morse Code',
|
||||
description: 'Dots and dashes encoding',
|
||||
longDescription:
|
||||
'Tests handling of text encoded in Morse code (dots and dashes) to potentially bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/other-encodings/#morse-code',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'piglatin',
|
||||
displayName: 'Pig Latin',
|
||||
description: 'Word transformation encoding',
|
||||
longDescription:
|
||||
'Tests handling of text transformed into Pig Latin (rearranging word parts) to potentially bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '20-30%',
|
||||
link: '/docs/red-team/strategies/other-encodings/#pig-latin',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'camelcase',
|
||||
displayName: 'camelCase',
|
||||
description: 'camelCase transformation',
|
||||
longDescription:
|
||||
'Tests handling of text transformed into camelCase (removing spaces and capitalizing words) to potentially bypass content filters',
|
||||
cost: 'Low',
|
||||
asrIncrease: '0-5%',
|
||||
link: '/docs/red-team/strategies/other-encodings/#camelcase',
|
||||
},
|
||||
{
|
||||
category: 'Static (Single-Turn)',
|
||||
strategy: 'emoji',
|
||||
displayName: 'Emoji Smuggling',
|
||||
description: 'Variation selector encoding',
|
||||
longDescription:
|
||||
'Tests hiding UTF-8 payloads inside emoji variation selectors to evaluate filter evasion.',
|
||||
cost: 'Low',
|
||||
asrIncrease: '0-5%',
|
||||
link: '/docs/red-team/strategies/other-encodings/#emoji-encoding',
|
||||
},
|
||||
{
|
||||
category: 'Custom',
|
||||
strategy: 'layer',
|
||||
displayName: 'Layer',
|
||||
description: 'Compose multiple strategies',
|
||||
longDescription:
|
||||
'Compose multiple red team strategies sequentially (e.g., jailbreak → base64) to create sophisticated attack chains',
|
||||
cost: 'Variable',
|
||||
asrIncrease: 'Cumulative',
|
||||
link: '/docs/red-team/strategies/layer/',
|
||||
},
|
||||
{
|
||||
category: 'Regression',
|
||||
strategy: 'retry',
|
||||
displayName: 'Retry',
|
||||
description: 'Historical failure testing',
|
||||
longDescription:
|
||||
'Automatically incorporates previously failed test cases into your test suite, creating a regression testing system that learns from past failures',
|
||||
cost: 'Low',
|
||||
asrIncrease: '50-70%',
|
||||
link: '/docs/red-team/strategies/retry/',
|
||||
recommended: false,
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,3 @@
|
||||
export const getCategoryAnchor = (category: string): string => {
|
||||
return '#' + category.toLowerCase().replace(/ and /g, '-').replace(/ /g, '-');
|
||||
};
|
||||
Reference in New Issue
Block a user