chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,621 @@
|
||||
name: "showcase / eval"
|
||||
|
||||
# SECURITY — residual trust model (read before editing):
|
||||
#
|
||||
# This workflow executes `showcase/bin/showcase eval` against PR-HEAD code.
|
||||
# Hardening layers mirror test_e2e-showcase-on-demand.yml:
|
||||
# - `getCollaboratorPermissionLevel` gate limits the trigger to users with
|
||||
# write (or higher) access — third-party commenters cannot spawn runs.
|
||||
# - workflow-level `permissions: contents: read` means the eval job's
|
||||
# GITHUB_TOKEN cannot mutate the repo; the `post-result` job gets write
|
||||
# perms scoped to just the final PR comment.
|
||||
# - `persist-credentials: false` on `actions/checkout` prevents the token
|
||||
# from leaking to PR-HEAD build hooks.
|
||||
# - `env:`-based pattern for UNTRUSTED values (comment body) prevents shell
|
||||
# injection.
|
||||
# - Slug whitelist (`^[a-z0-9-]+$`) prevents path traversal.
|
||||
#
|
||||
# Known TOCTOU — comment-trigger vs resolved HEAD SHA:
|
||||
# Same gap as test_e2e-showcase-on-demand.yml. The `pulls.get` call resolves
|
||||
# whatever HEAD is current at job start, not at comment time. The permission
|
||||
# gate + code-review social contract are the mitigations.
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "PR number to evaluate"
|
||||
required: true
|
||||
type: string
|
||||
check_run_id:
|
||||
description: "Check Run ID to update with results"
|
||||
required: false
|
||||
type: string
|
||||
level:
|
||||
description: "Eval depth level"
|
||||
required: false
|
||||
default: "d5"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: showcase-eval-${{ github.event.inputs.pr_number || github.event.issue.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
if: >
|
||||
github.event.issue.pull_request
|
||||
&& startsWith(github.event.comment.body, '/eval')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
outputs:
|
||||
pr_sha: ${{ steps.pr-ref.outputs.sha }}
|
||||
pr_number: ${{ steps.pr-ref.outputs.pr_number }}
|
||||
level: ${{ steps.parse.outputs.level }}
|
||||
scope_flag: ${{ steps.parse.outputs.scope_flag }}
|
||||
scope_display: ${{ steps.parse.outputs.scope_display }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
- name: Check commenter has write access
|
||||
id: auth
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
username: context.payload.comment.user.login,
|
||||
});
|
||||
const level = perm.permission;
|
||||
if (!['admin', 'write'].includes(level)) {
|
||||
core.setFailed(`User ${context.payload.comment.user.login} has '${level}' access — write access required to trigger /eval.`);
|
||||
return;
|
||||
}
|
||||
core.info(`User ${context.payload.comment.user.login} has '${level}' access — authorized.`);
|
||||
|
||||
- name: Parse /eval command
|
||||
id: parse
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Extract the first line of the comment to parse the command.
|
||||
FIRST_LINE=$(printf '%s' "$COMMENT_BODY" | head -n1)
|
||||
|
||||
# Parse: /eval → d5 affected
|
||||
# /eval d5 → d5 affected
|
||||
# /eval d5 all → d5 all
|
||||
# /eval d5 mastra,agno → d5 specific slugs
|
||||
ARGS=$(printf '%s' "$FIRST_LINE" | sed 's|^/eval[[:space:]]*||')
|
||||
|
||||
# Default level
|
||||
LEVEL="d5"
|
||||
SCOPE=""
|
||||
SCOPE_FLAG=""
|
||||
SCOPE_DISPLAY=""
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
# Bare /eval — d5 affected
|
||||
SCOPE_FLAG="--scope affected"
|
||||
SCOPE_DISPLAY="affected integrations"
|
||||
else
|
||||
# First token is the level (only d5 supported for now)
|
||||
LEVEL_TOKEN=$(printf '%s' "$ARGS" | awk '{print $1}')
|
||||
REST=$(printf '%s' "$ARGS" | sed "s|^${LEVEL_TOKEN}[[:space:]]*||")
|
||||
|
||||
# Validate level
|
||||
case "$LEVEL_TOKEN" in
|
||||
d5) LEVEL="d5" ;;
|
||||
*)
|
||||
echo "::error::Unknown eval level '$LEVEL_TOKEN'. Supported: d5"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "$REST" ]; then
|
||||
# /eval d5 — affected
|
||||
SCOPE_FLAG="--scope affected"
|
||||
SCOPE_DISPLAY="affected integrations"
|
||||
elif [ "$REST" = "all" ]; then
|
||||
# /eval d5 all
|
||||
SCOPE_FLAG="--scope all"
|
||||
SCOPE_DISPLAY="all integrations"
|
||||
else
|
||||
# /eval d5 mastra,agno → specific slugs
|
||||
# Validate each slug against ^[a-z0-9-]+$ to prevent injection
|
||||
IFS=',' read -ra SLUGS <<< "$REST"
|
||||
for s in "${SLUGS[@]}"; do
|
||||
s=$(printf '%s' "$s" | xargs) # trim whitespace
|
||||
case "$s" in
|
||||
''|*[!a-z0-9-]*)
|
||||
echo "::error::Invalid slug '$s' — must match ^[a-z0-9-]+$"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
# Reassemble validated slugs into a clean comma-separated string
|
||||
# (trims whitespace the user may have typed, e.g. "mastra, agno")
|
||||
CLEAN_REST=$(printf '%s' "$REST" | tr -d ' ')
|
||||
SCOPE_FLAG="--slug $CLEAN_REST"
|
||||
SCOPE_DISPLAY="$CLEAN_REST"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "level=$LEVEL" >> "$GITHUB_OUTPUT"
|
||||
echo "scope_flag=$SCOPE_FLAG" >> "$GITHUB_OUTPUT"
|
||||
echo "scope_display=$SCOPE_DISPLAY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve PR HEAD ref
|
||||
id: pr-ref
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: context.issue.number,
|
||||
});
|
||||
if (pr.state !== 'open') {
|
||||
core.setFailed(`PR #${pr.number} is ${pr.state} (not open). Refusing to run eval on a non-open PR.`);
|
||||
return;
|
||||
}
|
||||
core.setOutput('sha', pr.head.sha);
|
||||
core.setOutput('pr_number', pr.number);
|
||||
|
||||
- name: React with rocket emoji
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: context.payload.comment.id,
|
||||
content: 'rocket',
|
||||
});
|
||||
|
||||
- name: Post running status comment
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
env:
|
||||
LEVEL: ${{ steps.parse.outputs.level }}
|
||||
SCOPE_DISPLAY: ${{ steps.parse.outputs.scope_display }}
|
||||
with:
|
||||
script: |
|
||||
const level = process.env.LEVEL;
|
||||
const scope = process.env.SCOPE_DISPLAY;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: [
|
||||
`<!-- showcase-eval-status -->`,
|
||||
`### Showcase Eval`,
|
||||
``,
|
||||
`| | |`,
|
||||
`|---|---|`,
|
||||
`| **Status** | Running... |`,
|
||||
`| **Level** | \`${level}\` |`,
|
||||
`| **Scope** | ${scope} |`,
|
||||
`| **Run** | [View workflow](${runUrl}) |`,
|
||||
].join('\n'),
|
||||
});
|
||||
|
||||
dispatch-gate:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
pr_sha: ${{ steps.resolve.outputs.sha }}
|
||||
pr_number: ${{ github.event.inputs.pr_number }}
|
||||
level: ${{ github.event.inputs.level || 'd5' }}
|
||||
scope_flag: "--scope affected"
|
||||
scope_display: "affected"
|
||||
check_run_id: ${{ github.event.inputs.check_run_id }}
|
||||
steps:
|
||||
- name: Resolve PR HEAD SHA
|
||||
id: resolve
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
const pr = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: Number(process.env.PR_NUMBER),
|
||||
});
|
||||
if (pr.data.state !== 'open') {
|
||||
core.setFailed(`PR #${process.env.PR_NUMBER} is not open`);
|
||||
return;
|
||||
}
|
||||
core.setOutput('sha', pr.data.head.sha);
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.inputs.pr_number }}
|
||||
|
||||
eval:
|
||||
needs: [gate, dispatch-gate]
|
||||
if: always() && (needs.gate.result == 'success' || needs.dispatch-gate.result == 'success')
|
||||
runs-on: depot-ubuntu-24.04-16
|
||||
timeout-minutes: 45
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
PR_SHA: ${{ needs.gate.outputs.pr_sha || needs.dispatch-gate.outputs.pr_sha }}
|
||||
PR_NUMBER: ${{ needs.gate.outputs.pr_number || needs.dispatch-gate.outputs.pr_number }}
|
||||
EVAL_LEVEL: ${{ needs.gate.outputs.level || needs.dispatch-gate.outputs.level || 'd5' }}
|
||||
EVAL_SCOPE_FLAG: ${{ needs.gate.outputs.scope_flag || needs.dispatch-gate.outputs.scope_flag }}
|
||||
CHECK_RUN_ID: ${{ needs.dispatch-gate.outputs.check_run_id || '' }}
|
||||
|
||||
outputs:
|
||||
result_json: ${{ steps.run-eval.outputs.result_json }}
|
||||
exit_code: ${{ steps.run-eval.outputs.exit_code }}
|
||||
stderr_excerpt: ${{ steps.run-eval.outputs.stderr_excerpt }}
|
||||
|
||||
steps:
|
||||
- name: Checkout PR HEAD
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
ref: ${{ env.PR_SHA }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 22.x
|
||||
|
||||
# Omit `version:` so pnpm/action-setup inherits from the repo's
|
||||
# `packageManager` field in package.json (via corepack).
|
||||
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --ignore-scripts
|
||||
|
||||
- name: Install Playwright chromium
|
||||
run: npx playwright install chromium --with-deps
|
||||
|
||||
- name: Run showcase eval
|
||||
id: run-eval
|
||||
run: |
|
||||
set -o pipefail
|
||||
|
||||
# Build the command. EVAL_SCOPE_FLAG may contain spaces (e.g. "--slug mastra,agno")
|
||||
# so we intentionally leave it unquoted for word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
CMD="showcase/bin/showcase eval --${EVAL_LEVEL} ${EVAL_SCOPE_FLAG} --parallel 8 --json --baseline compare --timeout 60000 --ci"
|
||||
echo "::group::Running: $CMD"
|
||||
|
||||
EXIT_CODE=0
|
||||
# Capture both stdout (JSON results) and stderr separately.
|
||||
# Tee stderr to a file for excerpt extraction on failure.
|
||||
$CMD > eval-results.json 2> eval-stderr.log || EXIT_CODE=$?
|
||||
|
||||
echo "::endgroup::"
|
||||
echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ -f eval-results.json ] && [ -s eval-results.json ]; then
|
||||
# GitHub outputs have a 1MB limit; truncate if needed
|
||||
RESULT_SIZE=$(wc -c < eval-results.json)
|
||||
if [ "$RESULT_SIZE" -gt 900000 ]; then
|
||||
echo "::warning::eval-results.json exceeds 900KB ($RESULT_SIZE bytes), truncating for output"
|
||||
head -c 900000 eval-results.json > eval-results-truncated.json
|
||||
echo "result_json<<GHEOF" >> "$GITHUB_OUTPUT"
|
||||
cat eval-results-truncated.json >> "$GITHUB_OUTPUT"
|
||||
echo "GHEOF" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "result_json<<GHEOF" >> "$GITHUB_OUTPUT"
|
||||
cat eval-results.json >> "$GITHUB_OUTPUT"
|
||||
echo "GHEOF" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
else
|
||||
echo 'result_json={}' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Capture last 50 lines of stderr for failure reporting
|
||||
if [ -f eval-stderr.log ] && [ -s eval-stderr.log ]; then
|
||||
echo "stderr_excerpt<<GHEOF" >> "$GITHUB_OUTPUT"
|
||||
tail -n 50 eval-stderr.log >> "$GITHUB_OUTPUT"
|
||||
echo "GHEOF" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "stderr_excerpt=" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Propagate the exit code so the job status reflects eval outcome
|
||||
exit $EXIT_CODE
|
||||
|
||||
- name: Upload eval artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: showcase-eval-results
|
||||
path: |
|
||||
eval-results.json
|
||||
eval-stderr.log
|
||||
retention-days: 14
|
||||
if-no-files-found: ignore
|
||||
|
||||
post-result:
|
||||
needs: [gate, dispatch-gate, eval]
|
||||
if: always() && (needs.gate.result == 'success' || needs.dispatch-gate.result == 'success')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
checks: write
|
||||
|
||||
steps:
|
||||
- name: Post eval results to PR
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
env:
|
||||
EVAL_STATUS: ${{ needs.eval.result }}
|
||||
RESULT_JSON: ${{ needs.eval.outputs.result_json }}
|
||||
STDERR_EXCERPT: ${{ needs.eval.outputs.stderr_excerpt }}
|
||||
EXIT_CODE: ${{ needs.eval.outputs.exit_code }}
|
||||
LEVEL: ${{ needs.gate.outputs.level || needs.dispatch-gate.outputs.level || 'd5' }}
|
||||
SCOPE_DISPLAY: ${{ needs.gate.outputs.scope_display || needs.dispatch-gate.outputs.scope_display || 'affected' }}
|
||||
PR_NUMBER: ${{ needs.gate.outputs.pr_number || needs.dispatch-gate.outputs.pr_number }}
|
||||
with:
|
||||
script: |
|
||||
const evalStatus = process.env.EVAL_STATUS;
|
||||
const resultJson = process.env.RESULT_JSON || '{}';
|
||||
const stderrExcerpt = process.env.STDERR_EXCERPT || '';
|
||||
const exitCode = process.env.EXIT_CODE || 'unknown';
|
||||
const level = process.env.LEVEL;
|
||||
const scope = process.env.SCOPE_DISPLAY;
|
||||
const prNumber = parseInt(process.env.PR_NUMBER, 10);
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
let body = '';
|
||||
|
||||
if (evalStatus === 'success') {
|
||||
// Parse JSON results and build markdown table
|
||||
let results;
|
||||
try {
|
||||
results = JSON.parse(resultJson);
|
||||
} catch (e) {
|
||||
// JSON parse failed — report raw
|
||||
body = [
|
||||
`<!-- showcase-eval-result -->`,
|
||||
`### Showcase Eval Results`,
|
||||
``,
|
||||
`| | |`,
|
||||
`|---|---|`,
|
||||
`| **Verdict** | :warning: PARSE ERROR |`,
|
||||
`| **Level** | \`${level}\` |`,
|
||||
`| **Scope** | ${scope} |`,
|
||||
`| **Run** | [View workflow](${runUrl}) |`,
|
||||
``,
|
||||
`Could not parse eval JSON output:`,
|
||||
'```',
|
||||
e.message,
|
||||
'```',
|
||||
``,
|
||||
`<details><summary>Raw output</summary>`,
|
||||
``,
|
||||
'```json',
|
||||
resultJson.substring(0, 50000),
|
||||
'```',
|
||||
``,
|
||||
`</details>`,
|
||||
].join('\n');
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Build results table from the JSON.
|
||||
// Expected shape: { summary: { total, pass, fail, skip, duration_ms },
|
||||
// results: { slug: { testName: { status, duration_ms, error? } } } }
|
||||
const summary = results.summary || {};
|
||||
const resultsMap = results.results || {};
|
||||
const total = summary.total || 0;
|
||||
const passed = summary.pass || 0;
|
||||
const failed = summary.fail || 0;
|
||||
const skipped = summary.skip || 0;
|
||||
|
||||
const verdict = failed === 0
|
||||
? ':white_check_mark: **SAFE TO MERGE**'
|
||||
: `:x: **FAILURES DETECTED** (${failed}/${total} failed)`;
|
||||
|
||||
// Build per-integration results table from nested object
|
||||
let tableRows = '';
|
||||
const rows = [];
|
||||
for (const [slug, tests] of Object.entries(resultsMap)) {
|
||||
for (const [testName, r] of Object.entries(tests)) {
|
||||
const icon = r.status === 'pass' ? ':white_check_mark:'
|
||||
: r.status === 'fail' ? ':x:'
|
||||
: r.status === 'skip' ? ':fast_forward:'
|
||||
: r.status === 'error' ? ':boom:'
|
||||
: r.status === 'build_failed' ? ':hammer:'
|
||||
: r.status === 'unhealthy' ? ':warning:'
|
||||
: ':question:';
|
||||
const duration = r.duration_ms ? `${(r.duration_ms / 1000).toFixed(1)}s` : '-';
|
||||
const detail = r.error ? r.error.substring(0, 120) : '-';
|
||||
rows.push(`| ${icon} | \`${slug}\` | ${testName} | ${r.status || 'unknown'} | ${duration} | ${detail} |`);
|
||||
}
|
||||
}
|
||||
if (rows.length > 0) {
|
||||
tableRows = rows.join('\n');
|
||||
}
|
||||
|
||||
body = [
|
||||
`<!-- showcase-eval-result -->`,
|
||||
`### Showcase Eval Results`,
|
||||
``,
|
||||
`| | |`,
|
||||
`|---|---|`,
|
||||
`| **Verdict** | ${verdict} |`,
|
||||
`| **Level** | \`${level}\` |`,
|
||||
`| **Scope** | ${scope} |`,
|
||||
`| **Total** | ${total} |`,
|
||||
`| **Passed** | ${passed} |`,
|
||||
`| **Failed** | ${failed} |`,
|
||||
`| **Skipped** | ${skipped} |`,
|
||||
`| **Run** | [View workflow](${runUrl}) |`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
if (tableRows) {
|
||||
body += [
|
||||
`#### Per-Integration Results`,
|
||||
``,
|
||||
`| | Integration | Test | Status | Duration | Details |`,
|
||||
`|---|---|---|---|---|---|`,
|
||||
tableRows,
|
||||
``,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// Collapsible full JSON
|
||||
body += [
|
||||
`<details><summary>Full JSON details</summary>`,
|
||||
``,
|
||||
'```json',
|
||||
JSON.stringify(results, null, 2).substring(0, 60000),
|
||||
'```',
|
||||
``,
|
||||
`</details>`,
|
||||
].join('\n');
|
||||
|
||||
} else {
|
||||
// Eval failed — post error with stderr excerpt
|
||||
body = [
|
||||
`<!-- showcase-eval-result -->`,
|
||||
`### Showcase Eval Results`,
|
||||
``,
|
||||
`| | |`,
|
||||
`|---|---|`,
|
||||
`| **Verdict** | :x: **EVAL FAILED** (exit code: ${exitCode}) |`,
|
||||
`| **Level** | \`${level}\` |`,
|
||||
`| **Scope** | ${scope} |`,
|
||||
`| **Run** | [View workflow](${runUrl}) |`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
if (stderrExcerpt) {
|
||||
body += [
|
||||
`<details><summary>Error output (last 50 lines)</summary>`,
|
||||
``,
|
||||
'```',
|
||||
stderrExcerpt.substring(0, 30000),
|
||||
'```',
|
||||
``,
|
||||
`</details>`,
|
||||
``,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// If we got partial JSON, include it
|
||||
if (resultJson && resultJson !== '{}') {
|
||||
body += [
|
||||
`<details><summary>Partial JSON output</summary>`,
|
||||
``,
|
||||
'```json',
|
||||
resultJson.substring(0, 30000),
|
||||
'```',
|
||||
``,
|
||||
`</details>`,
|
||||
].join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body,
|
||||
});
|
||||
|
||||
- name: Generate devops-bot token
|
||||
id: bot-token
|
||||
if: needs.dispatch-gate.outputs.check_run_id != ''
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
app-id: 1108748
|
||||
private-key: ${{ secrets.DEVOPS_BOT_PRIVATE_KEY }}
|
||||
permission-checks: write
|
||||
|
||||
- name: Update Check Run with results
|
||||
if: needs.dispatch-gate.outputs.check_run_id != ''
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
github-token: ${{ steps.bot-token.outputs.token }}
|
||||
script: |
|
||||
const checkRunId = Number(process.env.CHECK_RUN_ID);
|
||||
const resultJson = process.env.RESULT_JSON || '{}';
|
||||
const evalStatus = '${{ needs.eval.result }}';
|
||||
|
||||
let conclusion = 'failure';
|
||||
let title = 'Showcase Eval — error';
|
||||
let summary = 'The evaluation encountered an error.';
|
||||
|
||||
try {
|
||||
const results = JSON.parse(resultJson);
|
||||
const s = results.summary || {};
|
||||
|
||||
if (evalStatus === 'success' && s.fail === 0) {
|
||||
conclusion = 'success';
|
||||
title = `${s.pass}/${s.total} passed (${(s.duration_ms / 1000).toFixed(1)}s)`;
|
||||
} else if (s.total === 0) {
|
||||
conclusion = 'neutral';
|
||||
title = 'No showcase integrations affected';
|
||||
} else {
|
||||
conclusion = 'failure';
|
||||
title = `${s.fail} failed, ${s.pass} passed`;
|
||||
}
|
||||
|
||||
const lines = ['## Eval Results\n'];
|
||||
lines.push('| Integration | Status |');
|
||||
lines.push('|-------------|--------|');
|
||||
if (results.results) {
|
||||
for (const [slug, tests] of Object.entries(results.results)) {
|
||||
const statuses = Object.values(tests);
|
||||
const pass = statuses.filter(t => t.status === 'pass').length;
|
||||
const total = statuses.length;
|
||||
const icon = pass === total ? '✅' : '❌';
|
||||
lines.push(`| ${slug} | ${icon} ${pass}/${total} |`);
|
||||
}
|
||||
}
|
||||
lines.push(`\n**Total:** ${s.pass} passed, ${s.fail} failed, ${s.skip} skipped (${(s.duration_ms / 1000).toFixed(1)}s)`);
|
||||
summary = lines.join('\n');
|
||||
} catch (e) {
|
||||
summary = `Parse error: ${e.message}`;
|
||||
}
|
||||
|
||||
await github.rest.checks.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
check_run_id: checkRunId,
|
||||
status: 'completed',
|
||||
conclusion,
|
||||
output: { title, summary },
|
||||
actions: [{
|
||||
label: 'Re-run Eval',
|
||||
description: 'Run D5 evaluation',
|
||||
identifier: 'run-eval',
|
||||
}],
|
||||
});
|
||||
env:
|
||||
CHECK_RUN_ID: ${{ needs.dispatch-gate.outputs.check_run_id }}
|
||||
RESULT_JSON: ${{ needs.eval.outputs.result_json }}
|
||||
Reference in New Issue
Block a user