chore: import upstream snapshot with attribution
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Waiting to run
Test and Publish Multi-arch Docker Image / test (push) Waiting to run
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Blocked by required conditions
Validate Renovate Config / Validate Renovate Configuration (push) Waiting to run
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Waiting to run
Test and Publish Multi-arch Docker Image / test (push) Waiting to run
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Blocked by required conditions
Validate Renovate Config / Validate Renovate Configuration (push) Waiting to run
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
clampCommentLines,
|
||||
extractValidLineRanges,
|
||||
isLineInDiff,
|
||||
} from '../../../src/codeScan/util/diffLineRanges';
|
||||
|
||||
describe('extractValidLineRanges', () => {
|
||||
it('should handle empty diff', () => {
|
||||
expect(extractValidLineRanges('')).toEqual(new Map());
|
||||
});
|
||||
|
||||
it('should extract ranges from single file with single hunk', () => {
|
||||
const diff = `diff --git a/src/foo.ts b/src/foo.ts
|
||||
--- a/src/foo.ts
|
||||
+++ b/src/foo.ts
|
||||
@@ -10,7 +10,8 @@
|
||||
context line 1
|
||||
context line 2
|
||||
- removed line
|
||||
+ added line 1
|
||||
+ added line 2
|
||||
context line 3
|
||||
context line 4`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
expect(ranges.get('src/foo.ts')).toEqual([{ start: 10, end: 15 }]);
|
||||
});
|
||||
|
||||
it('should extract ranges from single file with multiple hunks (gap between)', () => {
|
||||
const diff = `diff --git a/src/foo.ts b/src/foo.ts
|
||||
--- a/src/foo.ts
|
||||
+++ b/src/foo.ts
|
||||
@@ -10,5 +10,5 @@
|
||||
context
|
||||
- old
|
||||
+ new
|
||||
context
|
||||
context
|
||||
@@ -50,4 +50,5 @@
|
||||
more context
|
||||
+ added
|
||||
even more
|
||||
end`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
const fileRanges = ranges.get('src/foo.ts');
|
||||
|
||||
expect(fileRanges).toHaveLength(2);
|
||||
expect(fileRanges![0]).toEqual({ start: 10, end: 13 });
|
||||
expect(fileRanges![1]).toEqual({ start: 50, end: 53 });
|
||||
});
|
||||
|
||||
it('should extract ranges from multiple files', () => {
|
||||
const diff = `diff --git a/src/a.ts b/src/a.ts
|
||||
--- a/src/a.ts
|
||||
+++ b/src/a.ts
|
||||
@@ -1,3 +1,4 @@
|
||||
line 1
|
||||
+added
|
||||
line 2
|
||||
line 3
|
||||
diff --git a/src/b.ts b/src/b.ts
|
||||
--- a/src/b.ts
|
||||
+++ b/src/b.ts
|
||||
@@ -5,2 +5,3 @@
|
||||
line 5
|
||||
+added
|
||||
line 6`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
|
||||
expect(ranges.get('src/a.ts')).toEqual([{ start: 1, end: 4 }]);
|
||||
expect(ranges.get('src/b.ts')).toEqual([{ start: 5, end: 7 }]);
|
||||
});
|
||||
|
||||
it('should handle new file (all additions)', () => {
|
||||
const diff = `diff --git a/src/new.ts b/src/new.ts
|
||||
new file mode 100644
|
||||
--- /dev/null
|
||||
+++ b/src/new.ts
|
||||
@@ -0,0 +1,5 @@
|
||||
+line 1
|
||||
+line 2
|
||||
+line 3
|
||||
+line 4
|
||||
+line 5`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
expect(ranges.get('src/new.ts')).toEqual([{ start: 1, end: 5 }]);
|
||||
});
|
||||
|
||||
it('should handle deleted file (no valid ranges)', () => {
|
||||
const diff = `diff --git a/src/deleted.ts b/src/deleted.ts
|
||||
deleted file mode 100644
|
||||
--- a/src/deleted.ts
|
||||
+++ /dev/null
|
||||
@@ -1,3 +0,0 @@
|
||||
-line 1
|
||||
-line 2
|
||||
-line 3`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
expect(ranges.has('src/deleted.ts')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('clampCommentLines', () => {
|
||||
const ranges = new Map([
|
||||
[
|
||||
'src/foo.ts',
|
||||
[
|
||||
{ start: 10, end: 20 },
|
||||
{ start: 50, end: 60 },
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
it('should return line unchanged if valid (single-line)', () => {
|
||||
expect(clampCommentLines('src/foo.ts', null, 15, ranges)).toEqual({
|
||||
startLine: null,
|
||||
line: 15,
|
||||
});
|
||||
});
|
||||
|
||||
it('should clamp single-line comment in gap to end of previous hunk', () => {
|
||||
expect(clampCommentLines('src/foo.ts', null, 30, ranges)).toEqual({
|
||||
startLine: null,
|
||||
line: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return both lines unchanged if valid (multi-line)', () => {
|
||||
expect(clampCommentLines('src/foo.ts', 12, 18, ranges)).toEqual({
|
||||
startLine: 12,
|
||||
line: 18,
|
||||
});
|
||||
});
|
||||
|
||||
it('should clamp end line if it extends into gap', () => {
|
||||
// Start at 15 (valid), end at 25 (in gap) -> clamp end to 20
|
||||
expect(clampCommentLines('src/foo.ts', 15, 25, ranges)).toEqual({
|
||||
startLine: 15,
|
||||
line: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return null for unknown file', () => {
|
||||
expect(clampCommentLines('src/unknown.ts', 10, 20, ranges)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null endLine', () => {
|
||||
expect(clampCommentLines('src/foo.ts', 10, null, ranges)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('integration: ENG-1309 scenario', () => {
|
||||
it('should clamp comment that extends into gap between hunks', () => {
|
||||
// Based on the actual PR that triggered ENG-1309
|
||||
const diff = `diff --git a/example-app/src/tools/index.ts b/example-app/src/tools/index.ts
|
||||
--- a/example-app/src/tools/index.ts
|
||||
+++ b/example-app/src/tools/index.ts
|
||||
@@ -53,7 +53,7 @@ export function executeTool(toolCall: ToolCall, userContext: UserContext): ToolR
|
||||
// Secure level: always use authenticated user's role as user_id
|
||||
userId = userContext.role;
|
||||
} else {
|
||||
- // Insecure level: allow any user_id (no access control)
|
||||
+ // Insecure/Medium level: allow any user_id (no access control)
|
||||
const providedUserId = args.user_id as string | undefined;
|
||||
userId = providedUserId === 'current' || providedUserId === 'me' || !providedUserId
|
||||
? userContext.role
|
||||
@@ -68,7 +68,7 @@ export function executeTool(toolCall: ToolCall, userContext: UserContext): ToolR
|
||||
// Secure level: always use authenticated user's role as user_id
|
||||
userId = userContext.role;
|
||||
} else {
|
||||
- // Insecure level: allow any user_id (no access control)
|
||||
+ // Insecure/Medium level: allow any user_id (no access control)
|
||||
const providedUserId = args.user_id as string | undefined;`;
|
||||
|
||||
const ranges = extractValidLineRanges(diff);
|
||||
|
||||
// Agent tried to comment on lines 56-62, but line 62 is in the gap (59-68)
|
||||
const result = clampCommentLines('example-app/src/tools/index.ts', 56, 62, ranges);
|
||||
|
||||
expect(result).toEqual({
|
||||
startLine: 56,
|
||||
line: 59, // clamped from 62 to end of first hunk
|
||||
});
|
||||
|
||||
// Verify the gap detection
|
||||
expect(isLineInDiff('example-app/src/tools/index.ts', 59, ranges)).toBe(true);
|
||||
expect(isLineInDiff('example-app/src/tools/index.ts', 62, ranges)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
ALL_CLEAR_MESSAGE,
|
||||
hasPrPostableFindings,
|
||||
prepareComments,
|
||||
} from '../../../src/codeScan/util/github';
|
||||
import { CodeScanSeverity, type Comment } from '../../../src/types/codeScan';
|
||||
|
||||
describe('prepareComments', () => {
|
||||
it('should return empty arrays and empty review body when no comments', () => {
|
||||
const result = prepareComments([], undefined, undefined);
|
||||
|
||||
expect(result.lineComments).toEqual([]);
|
||||
expect(result.generalComments).toEqual([]);
|
||||
expect(result.reviewBody).toBe('');
|
||||
});
|
||||
|
||||
it('should prepend "All Clear" when only severity=none comments exist', () => {
|
||||
const comments: Comment[] = [
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
severity: CodeScanSeverity.NONE,
|
||||
finding: 'No vulnerabilities found',
|
||||
},
|
||||
];
|
||||
const review = 'Scanned 10 files. No issues detected.';
|
||||
|
||||
const result = prepareComments(comments, review, 'medium');
|
||||
|
||||
expect(result.reviewBody).toContain(ALL_CLEAR_MESSAGE);
|
||||
expect(result.reviewBody).toContain('Scanned 10 files');
|
||||
});
|
||||
|
||||
it('should separate line-specific from general comments', () => {
|
||||
const comments: Comment[] = [
|
||||
{
|
||||
file: 'src/test.ts',
|
||||
line: 42,
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
finding: 'SQL injection vulnerability',
|
||||
},
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
severity: CodeScanSeverity.MEDIUM,
|
||||
finding: 'General security issue',
|
||||
},
|
||||
{
|
||||
file: 'src/file-only.ts',
|
||||
line: null,
|
||||
severity: CodeScanSeverity.LOW,
|
||||
finding: 'File-level security issue',
|
||||
},
|
||||
];
|
||||
|
||||
const result = prepareComments(comments, 'Review text', 'low');
|
||||
|
||||
expect(result.lineComments).toHaveLength(1);
|
||||
expect(result.lineComments[0].file).toBe('src/test.ts');
|
||||
expect(result.generalComments).toHaveLength(2);
|
||||
expect(result.generalComments.map((comment) => comment.file)).toEqual([
|
||||
null,
|
||||
'src/file-only.ts',
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
null,
|
||||
0,
|
||||
-1,
|
||||
1.5,
|
||||
])('should route findings with a non-inlineable line (%s) to general comments', (line) => {
|
||||
const comment: Comment = {
|
||||
file: 'src/example.ts',
|
||||
line,
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
finding: 'File-scoped issue',
|
||||
};
|
||||
|
||||
const result = prepareComments([comment], 'Review', 'low');
|
||||
|
||||
expect(result.lineComments).toEqual([]);
|
||||
expect(result.generalComments).toEqual([comment]);
|
||||
});
|
||||
|
||||
it('should filter out severity=none from general comments', () => {
|
||||
const comments: Comment[] = [
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
severity: CodeScanSeverity.NONE,
|
||||
finding: 'All clear message',
|
||||
},
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
severity: CodeScanSeverity.LOW,
|
||||
finding: 'Actual issue',
|
||||
},
|
||||
];
|
||||
|
||||
const result = prepareComments(comments, 'Review', 'low');
|
||||
|
||||
expect(result.generalComments).toHaveLength(1);
|
||||
expect(result.generalComments[0].severity).toBe(CodeScanSeverity.LOW);
|
||||
});
|
||||
|
||||
it('should recognize fileless findings as PR-postable but reject severity=none comments', () => {
|
||||
expect(
|
||||
hasPrPostableFindings([
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
finding: 'General security issue',
|
||||
},
|
||||
]),
|
||||
).toBe(true);
|
||||
|
||||
expect(
|
||||
hasPrPostableFindings([
|
||||
{
|
||||
file: 'src/test.ts',
|
||||
line: 42,
|
||||
severity: CodeScanSeverity.NONE,
|
||||
finding: 'No issue found',
|
||||
},
|
||||
]),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('should append severity threshold', () => {
|
||||
const result = prepareComments([], 'Review text', 'medium');
|
||||
|
||||
expect(result.reviewBody).toContain('Minimum severity threshold');
|
||||
expect(result.reviewBody).toContain('Medium');
|
||||
});
|
||||
|
||||
it('should sort comments by severity descending', () => {
|
||||
const comments: Comment[] = [
|
||||
{ severity: CodeScanSeverity.LOW, finding: 'Low issue', file: 'a.ts', line: 1 },
|
||||
{ severity: CodeScanSeverity.CRITICAL, finding: 'Critical issue', file: 'b.ts', line: 2 },
|
||||
{ severity: CodeScanSeverity.MEDIUM, finding: 'Medium issue', file: 'c.ts', line: 3 },
|
||||
];
|
||||
|
||||
const result = prepareComments(comments, 'Review', undefined);
|
||||
|
||||
expect(result.lineComments[0].severity).toBe(CodeScanSeverity.CRITICAL);
|
||||
expect(result.lineComments[1].severity).toBe(CodeScanSeverity.MEDIUM);
|
||||
expect(result.lineComments[2].severity).toBe(CodeScanSeverity.LOW);
|
||||
});
|
||||
|
||||
it('should not prepend "All Clear" when vulnerabilities exist', () => {
|
||||
const comments: Comment[] = [
|
||||
{
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
finding: 'Real vulnerability',
|
||||
file: 'test.ts',
|
||||
line: 10,
|
||||
},
|
||||
];
|
||||
|
||||
const result = prepareComments(comments, 'Found issues', 'low');
|
||||
|
||||
expect(result.reviewBody).not.toContain(ALL_CLEAR_MESSAGE);
|
||||
expect(result.reviewBody).toContain('Found issues');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,359 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { hasSarifReportableFindings, scanResponseToSarif } from '../../../src/codeScan/util/sarif';
|
||||
import { CodeScanSeverity, type Comment, type ScanResponse } from '../../../src/types/codeScan';
|
||||
|
||||
function makeFindingResponse(overrides: Partial<Comment> = {}): ScanResponse {
|
||||
return {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src/handler.ts',
|
||||
startLine: 10,
|
||||
line: 12,
|
||||
finding: 'User input reaches the model prompt without sanitization.',
|
||||
severity: CodeScanSeverity.CRITICAL,
|
||||
...overrides,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function fingerprintOf(response: ScanResponse): string {
|
||||
return scanResponseToSarif(response).runs[0].results[0].partialFingerprints.promptfooFindingHash;
|
||||
}
|
||||
|
||||
describe('scanResponseToSarif', () => {
|
||||
it('maps reportable findings into GitHub-compatible SARIF results', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src/chat/handler.ts',
|
||||
startLine: 40,
|
||||
line: 42,
|
||||
finding: 'User input reaches the model prompt without sanitization.',
|
||||
fix: 'Sanitize user input before composing the prompt.',
|
||||
severity: CodeScanSeverity.CRITICAL,
|
||||
aiAgentPrompt: 'Add prompt input validation.',
|
||||
},
|
||||
{
|
||||
file: 'src/chat/render output.ts',
|
||||
line: 87,
|
||||
finding: 'Model output is rendered as raw HTML.',
|
||||
severity: CodeScanSeverity.MEDIUM,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const sarif = scanResponseToSarif(response);
|
||||
|
||||
expect(sarif).toMatchObject({
|
||||
$schema: 'https://json.schemastore.org/sarif-2.1.0.json',
|
||||
version: '2.1.0',
|
||||
runs: [
|
||||
{
|
||||
tool: {
|
||||
driver: {
|
||||
name: 'Promptfoo Code Scan',
|
||||
rules: [
|
||||
{
|
||||
id: 'promptfoo/code-scan-finding',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
results: [
|
||||
{
|
||||
ruleId: 'promptfoo/code-scan-finding',
|
||||
level: 'error',
|
||||
message: {
|
||||
text: 'User input reaches the model prompt without sanitization.',
|
||||
},
|
||||
locations: [
|
||||
{
|
||||
physicalLocation: {
|
||||
artifactLocation: {
|
||||
uri: 'src/chat/handler.ts',
|
||||
},
|
||||
region: {
|
||||
startLine: 40,
|
||||
endLine: 42,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
properties: {
|
||||
severity: CodeScanSeverity.CRITICAL,
|
||||
fix: 'Sanitize user input before composing the prompt.',
|
||||
aiAgentPrompt: 'Add prompt input validation.',
|
||||
},
|
||||
},
|
||||
{
|
||||
ruleId: 'promptfoo/code-scan-finding',
|
||||
level: 'warning',
|
||||
message: {
|
||||
text: 'Model output is rendered as raw HTML.',
|
||||
},
|
||||
locations: [
|
||||
{
|
||||
physicalLocation: {
|
||||
artifactLocation: {
|
||||
uri: 'src/chat/render%20output.ts',
|
||||
},
|
||||
region: {
|
||||
startLine: 87,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(sarif.runs[0].results[0].partialFingerprints.promptfooFindingHash).toMatch(
|
||||
/^[a-f0-9]{64}$/,
|
||||
);
|
||||
});
|
||||
|
||||
it('omits endLine when the comment range is inverted (startLine > line)', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src/inverted.ts',
|
||||
startLine: 100,
|
||||
line: 42,
|
||||
finding: 'Inverted region from upstream data.',
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const region =
|
||||
scanResponseToSarif(response).runs[0].results[0].locations?.[0].physicalLocation.region;
|
||||
|
||||
expect(region).toEqual({ startLine: 100 });
|
||||
});
|
||||
|
||||
it('keeps fingerprints stable when line numbers shift or wording is rephrased slightly', () => {
|
||||
const baseHash = fingerprintOf(makeFindingResponse());
|
||||
const shifted = fingerprintOf(makeFindingResponse({ startLine: 88, line: 90 }));
|
||||
const reworded = fingerprintOf(
|
||||
makeFindingResponse({
|
||||
finding: ' User input reaches the model prompt without sanitization.\n',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(shifted).toBe(baseHash);
|
||||
expect(reworded).toBe(baseHash);
|
||||
});
|
||||
|
||||
it('produces distinct fingerprints for different files or severities', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src/a.ts',
|
||||
line: 5,
|
||||
finding: 'Same finding text.',
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
},
|
||||
{
|
||||
file: 'src/b.ts',
|
||||
line: 5,
|
||||
finding: 'Same finding text.',
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
},
|
||||
{
|
||||
file: 'src/a.ts',
|
||||
line: 5,
|
||||
finding: 'Same finding text.',
|
||||
severity: CodeScanSeverity.LOW,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const hashes = scanResponseToSarif(response).runs[0].results.map(
|
||||
(result) => result.partialFingerprints.promptfooFindingHash,
|
||||
);
|
||||
|
||||
expect(new Set(hashes).size).toBe(3);
|
||||
});
|
||||
|
||||
it('normalizes Windows-style backslash separators in artifact URIs', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src\\windows path\\file.ts',
|
||||
line: 7,
|
||||
finding: 'Windows path leak.',
|
||||
severity: CodeScanSeverity.LOW,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(
|
||||
scanResponseToSarif(response).runs[0].results[0].locations?.[0].physicalLocation
|
||||
.artifactLocation.uri,
|
||||
).toBe('src/windows%20path/file.ts');
|
||||
});
|
||||
|
||||
it.each([
|
||||
[CodeScanSeverity.CRITICAL, 'error'],
|
||||
[CodeScanSeverity.HIGH, 'error'],
|
||||
[CodeScanSeverity.MEDIUM, 'warning'],
|
||||
[CodeScanSeverity.LOW, 'note'],
|
||||
])('maps severity %s to SARIF level %s', (severity, expected) => {
|
||||
const sarif = scanResponseToSarif(makeFindingResponse({ severity }));
|
||||
expect(sarif.runs[0].results[0].level).toBe(expected);
|
||||
});
|
||||
|
||||
it('drops findings with severity NONE even when they have a file and line', () => {
|
||||
const sarif = scanResponseToSarif(makeFindingResponse({ severity: CodeScanSeverity.NONE }));
|
||||
expect(sarif.runs[0].results).toEqual([]);
|
||||
});
|
||||
|
||||
it('emits findings with missing severity as note level (CommentSchema allows it)', () => {
|
||||
const sarif = scanResponseToSarif(makeFindingResponse({ severity: undefined }));
|
||||
expect(sarif.runs[0].results).toHaveLength(1);
|
||||
expect(sarif.runs[0].results[0].level).toBe('note');
|
||||
expect(sarif.runs[0].results[0].properties.severity).toBeUndefined();
|
||||
});
|
||||
|
||||
it('exposes the docs URL via tool.driver.informationUri', () => {
|
||||
const sarif = scanResponseToSarif(makeFindingResponse());
|
||||
expect(sarif.runs[0].tool.driver.informationUri).toBe(
|
||||
'https://www.promptfoo.dev/docs/code-scanning/cli/',
|
||||
);
|
||||
});
|
||||
|
||||
it('emits a file-only finding (no line) with an artifactLocation but no region', () => {
|
||||
const sarif = scanResponseToSarif(
|
||||
makeFindingResponse({ file: 'src/x.ts', line: null, startLine: undefined }),
|
||||
);
|
||||
expect(sarif.runs[0].results).toHaveLength(1);
|
||||
const location = sarif.runs[0].results[0].locations?.[0].physicalLocation;
|
||||
expect(location?.artifactLocation.uri).toBe('src/x.ts');
|
||||
expect(location?.region).toBeUndefined();
|
||||
});
|
||||
|
||||
it('omits the rule descriptor when there are no SARIF results', () => {
|
||||
const sarif = scanResponseToSarif({ success: true, comments: [] });
|
||||
expect(sarif.runs[0].tool.driver.rules).toEqual([]);
|
||||
expect(sarif.runs[0].results).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves the scan review summary under properties.promptfoo', () => {
|
||||
const sarif = scanResponseToSarif({
|
||||
success: true,
|
||||
comments: [],
|
||||
review: 'No issues found in this PR.',
|
||||
});
|
||||
expect(sarif.runs[0].properties?.promptfoo?.review).toBe('No issues found in this PR.');
|
||||
});
|
||||
|
||||
it('omits run.properties when there is no review summary to carry', () => {
|
||||
const sarif = scanResponseToSarif({ success: true, comments: [] });
|
||||
expect(sarif.runs[0].properties).toBeUndefined();
|
||||
});
|
||||
|
||||
it('truncates fingerprint input at the configured max so very long findings collide stably', () => {
|
||||
// Two findings whose first 160 chars match but tails differ → identical fingerprint
|
||||
// (intended: tail variation shouldn't churn the alert id across re-scans).
|
||||
const longHead =
|
||||
'This is a long LLM finding that describes a potential issue with user input. '.repeat(3);
|
||||
const tailA = ` First tail variation A — ${'a'.repeat(50)}`;
|
||||
const tailB = ` Second tail variation B — ${'b'.repeat(50)}`;
|
||||
const hashA = fingerprintOf(makeFindingResponse({ finding: longHead + tailA }));
|
||||
const hashB = fingerprintOf(makeFindingResponse({ finding: longHead + tailB }));
|
||||
expect(hashA).toBe(hashB);
|
||||
|
||||
// Sanity: a divergence inside the first 160 chars does change the hash.
|
||||
const earlyDiverge = fingerprintOf(
|
||||
makeFindingResponse({ finding: 'Different opening text — ' + longHead + tailA }),
|
||||
);
|
||||
expect(earlyDiverge).not.toBe(hashA);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['src/path with spaces/file.ts', 'src/path%20with%20spaces/file.ts'],
|
||||
['src/anchor#fragment.ts', 'src/anchor%23fragment.ts'],
|
||||
['src/café/résumé.ts', 'src/caf%C3%A9/r%C3%A9sum%C3%A9.ts'],
|
||||
['src\\mixed/style/file.ts', 'src/mixed/style/file.ts'],
|
||||
])('encodes artifactLocation.uri "%s" as "%s"', (input, expected) => {
|
||||
const sarif = scanResponseToSarif(
|
||||
makeFindingResponse({ file: input, line: 1, startLine: undefined }),
|
||||
);
|
||||
expect(sarif.runs[0].results[0].locations?.[0].physicalLocation.artifactLocation.uri).toBe(
|
||||
expected,
|
||||
);
|
||||
});
|
||||
|
||||
it('omits non-finding comments and findings without displayable locations', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
finding: 'No issues found.',
|
||||
severity: CodeScanSeverity.NONE,
|
||||
},
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
finding: 'Potential unsafe agent behavior.',
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const sarif = scanResponseToSarif(response);
|
||||
|
||||
expect(sarif.runs[0].results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasSarifReportableFindings', () => {
|
||||
it('returns true when a comment would serialize to a SARIF result', () => {
|
||||
expect(hasSarifReportableFindings(makeFindingResponse())).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for file-only findings supported by SARIF', () => {
|
||||
expect(
|
||||
hasSarifReportableFindings(
|
||||
makeFindingResponse({ file: 'src/handler.ts', line: null, startLine: undefined }),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for an empty comments array', () => {
|
||||
expect(hasSarifReportableFindings({ success: true, comments: [] })).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when every comment is filtered by scanResponseToSarif', () => {
|
||||
const response: ScanResponse = {
|
||||
success: true,
|
||||
comments: [
|
||||
{
|
||||
file: 'src/handler.ts',
|
||||
line: 12,
|
||||
finding: 'No issue found on this line.',
|
||||
severity: CodeScanSeverity.NONE,
|
||||
},
|
||||
{
|
||||
file: null,
|
||||
line: null,
|
||||
finding: 'Advisory not pinned to a file.',
|
||||
severity: CodeScanSeverity.HIGH,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(hasSarifReportableFindings(response)).toBe(false);
|
||||
// Stays in lockstep with the serializer it guards.
|
||||
expect(scanResponseToSarif(response).runs[0].results).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { requestsStructuredCodeScanOutput } from '../../../src/codeScan/util/structuredOutputDetect';
|
||||
|
||||
const node = 'node';
|
||||
const cli = '/path/to/main.js';
|
||||
|
||||
function argv(...rest: string[]): string[] {
|
||||
return [node, cli, ...rest];
|
||||
}
|
||||
|
||||
describe('requestsStructuredCodeScanOutput', () => {
|
||||
it('returns false when the subcommand is not code-scans run', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv())).toBe(false);
|
||||
expect(requestsStructuredCodeScanOutput(argv('eval', '--json'))).toBe(false);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans'))).toBe(false);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'list', '--json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('detects --json on code-scans run', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '.', '--json'))).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--json', '.'))).toBe(true);
|
||||
});
|
||||
|
||||
it('detects --format sarif and --format json (space-separated)', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format', 'sarif'))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format', 'json'))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('detects -f sarif and -f json (short alias, space-separated)', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-f', 'sarif'))).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-f', 'json'))).toBe(true);
|
||||
});
|
||||
|
||||
it('detects --format=value and -f=value (equals form)', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format=sarif'))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format=json'))).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-f=sarif'))).toBe(true);
|
||||
});
|
||||
|
||||
it('detects -fsarif and -fjson (Commander combined short form)', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-fsarif'))).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-fjson'))).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-ftext'))).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for --format text (the default)', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format', 'text'))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format=text'))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-f', 'text'))).toBe(false);
|
||||
});
|
||||
|
||||
it('still detects --json after a non-structured --format text', () => {
|
||||
// Regression: an earlier implementation returned eagerly on --format and missed
|
||||
// a later --json. Matters because Commander allows redundant flags.
|
||||
expect(
|
||||
requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format', 'text', '--json')),
|
||||
).toBe(true);
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '-f=text', '--json'))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('detects the latest format when multiple --format flags appear', () => {
|
||||
expect(
|
||||
requestsStructuredCodeScanOutput(
|
||||
argv('code-scans', 'run', '--format', 'text', '--format', 'sarif'),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
requestsStructuredCodeScanOutput(
|
||||
argv('code-scans', 'run', '--format=sarif', '--format=text'),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not misinterpret a value-eating flag's argument as a structured-output flag", () => {
|
||||
// `--base --json` — the literal `--json` is the value of `--base`, not a real flag.
|
||||
expect(
|
||||
requestsStructuredCodeScanOutput(argv('code-scans', 'run', '.', '--base', '--json')),
|
||||
).toBe(false);
|
||||
expect(
|
||||
requestsStructuredCodeScanOutput(argv('code-scans', 'run', '.', '--config', '--format')),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('stops scanning at the conventional `--` positional separator', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '.', '--', '--json'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('returns false for a bare --format with no value', () => {
|
||||
expect(requestsStructuredCodeScanOutput(argv('code-scans', 'run', '--format'))).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user