wehub-resource-sync
59a0a3844c
PR Test AMD / cancel-on-close (push) Has been skipped
PR Test NVIDIA ARM / scan (push) Has been skipped
PR Test NVIDIA / cancel-on-close (push) Has been skipped
PR Test AMD / scan (push) Has been skipped
PR Test NVIDIA ARM / cancel-on-close (push) Has been skipped
PR Test NVIDIA / scan (push) Has been skipped
Release Docker Images / build (cu129-torch-2.11.0) (push) Has been skipped
Release Docker Images / build (cu130-torch-2.11.0) (push) Has been skipped
Release PyPI / publish (push) Has been skipped
Scheduler Python Test / test (push) Successful in 27m19s
Docs / build (push) Successful in 28m8s
Scheduler C++ Test / test (push) Successful in 28m19s
Scheduler C++ Test / test-flat (push) Successful in 28m18s
Docs / deploy (push) Has been cancelled
PR Test AMD / finish (push) Has been cancelled
PR Test NVIDIA / finish (push) Has been cancelled
PR Test NVIDIA ARM / finish (push) Has been cancelled
PR Test NVIDIA ARM / ${{ matrix.name }} (${{ matrix.runner }}) (push) Has been cancelled
PR Test AMD / ${{ matrix.name }} (${{ matrix.runner }}) (push) Has been cancelled
PR Test NVIDIA / ${{ matrix.name }} (${{ matrix.runner }}) (push) Has been cancelled
351 lines
12 KiB
YAML
351 lines
12 KiB
YAML
name: Retry Failed Approved PR CI
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "47 * * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
dry_run:
|
|
description: "Log eligible reruns without starting them"
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
pull-requests: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
rerun-failed:
|
|
name: Rerun failed approved-PR jobs
|
|
if: github.repository == 'lightseekorg/tokenspeed'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Rerun failed vendor CI jobs
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const workflows = [
|
|
"pr-test-nvidia.yml",
|
|
"pr-test-amd.yml",
|
|
"pr-test-nvidia-arm.yml",
|
|
];
|
|
const maxAttempts = 3;
|
|
const maxRunAgeMs = 30 * 24 * 60 * 60 * 1000;
|
|
const dryRun =
|
|
context.eventName === "workflow_dispatch" &&
|
|
String(context.payload.inputs?.dry_run) === "true";
|
|
const { owner, repo } = context.repo;
|
|
const summaryRows = [];
|
|
|
|
function addSummaryRow(pull, workflowId, run, result) {
|
|
summaryRows.push([
|
|
`<a href="${pull.html_url}">#${pull.number}</a>`,
|
|
workflowId || "—",
|
|
run
|
|
? `<a href="${run.html_url}">${run.id}</a>`
|
|
: "—",
|
|
result,
|
|
]);
|
|
}
|
|
|
|
async function getPullRequest(number) {
|
|
const query = `
|
|
query($owner: String!, $repo: String!, $number: Int!) {
|
|
repository(owner: $owner, name: $repo) {
|
|
pullRequest(number: $number) {
|
|
state
|
|
isDraft
|
|
headRefOid
|
|
reviewDecision
|
|
author {
|
|
login
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`;
|
|
const result = await github.graphql(query, {
|
|
owner,
|
|
repo,
|
|
number,
|
|
});
|
|
return result.repository.pullRequest;
|
|
}
|
|
|
|
async function hasFallbackApproval(number, authorLogin) {
|
|
const reviews = await github.paginate(
|
|
github.rest.pulls.listReviews,
|
|
{
|
|
owner,
|
|
repo,
|
|
pull_number: number,
|
|
per_page: 100,
|
|
},
|
|
);
|
|
const latestDecisiveReview = new Map();
|
|
|
|
for (const review of reviews) {
|
|
if (
|
|
!review.user ||
|
|
review.user.login === authorLogin ||
|
|
!["APPROVED", "CHANGES_REQUESTED"].includes(review.state)
|
|
) {
|
|
continue;
|
|
}
|
|
const previous = latestDecisiveReview.get(review.user.id);
|
|
if (
|
|
!previous ||
|
|
Date.parse(review.submitted_at) >=
|
|
Date.parse(previous.submitted_at)
|
|
) {
|
|
latestDecisiveReview.set(review.user.id, review);
|
|
}
|
|
}
|
|
|
|
const decisions = [...latestDecisiveReview.values()].map(
|
|
(review) => review.state,
|
|
);
|
|
return (
|
|
decisions.includes("APPROVED") &&
|
|
!decisions.includes("CHANGES_REQUESTED")
|
|
);
|
|
}
|
|
|
|
async function isEligible(number, pullRequest) {
|
|
if (
|
|
!pullRequest ||
|
|
pullRequest.state !== "OPEN" ||
|
|
pullRequest.isDraft
|
|
) {
|
|
return false;
|
|
}
|
|
if (pullRequest.reviewDecision === "APPROVED") {
|
|
return true;
|
|
}
|
|
if (pullRequest.reviewDecision !== null) {
|
|
core.info(
|
|
"PR #" + number + ": review decision is " +
|
|
pullRequest.reviewDecision + ".",
|
|
);
|
|
return false;
|
|
}
|
|
|
|
const approved = await hasFallbackApproval(
|
|
number,
|
|
pullRequest.author?.login,
|
|
);
|
|
if (!approved) {
|
|
core.info("PR #" + number + ": no effective approval.");
|
|
}
|
|
return approved;
|
|
}
|
|
|
|
try {
|
|
const pulls = await github.paginate(github.rest.pulls.list, {
|
|
owner,
|
|
repo,
|
|
state: "open",
|
|
per_page: 100,
|
|
});
|
|
|
|
core.info(
|
|
"Scanning " + pulls.length + " open pull request(s)" +
|
|
(dryRun ? " in dry-run mode." : "."),
|
|
);
|
|
|
|
for (const pull of pulls) {
|
|
try {
|
|
const pullRequest = await getPullRequest(pull.number);
|
|
if (!(await isEligible(pull.number, pullRequest))) {
|
|
continue;
|
|
}
|
|
|
|
const selectedRuns = [];
|
|
for (const workflowId of workflows) {
|
|
let runs;
|
|
try {
|
|
runs = await github.paginate(
|
|
github.rest.actions.listWorkflowRuns,
|
|
{
|
|
owner,
|
|
repo,
|
|
workflow_id: workflowId,
|
|
event: "pull_request",
|
|
head_sha: pullRequest.headRefOid,
|
|
per_page: 100,
|
|
},
|
|
);
|
|
} catch (error) {
|
|
core.warning(
|
|
"PR #" + pull.number + ": unable to list runs for " +
|
|
workflowId + ": " + error.message,
|
|
);
|
|
addSummaryRow(
|
|
pull,
|
|
workflowId,
|
|
null,
|
|
"Failed to inspect",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
const run = runs
|
|
.filter(
|
|
(candidate) =>
|
|
candidate.head_sha === pullRequest.headRefOid &&
|
|
candidate.event === "pull_request",
|
|
)
|
|
.sort(
|
|
(left, right) =>
|
|
Date.parse(right.created_at) -
|
|
Date.parse(left.created_at),
|
|
)[0];
|
|
if (run) {
|
|
selectedRuns.push({ workflowId, run });
|
|
} else {
|
|
core.info(
|
|
"PR #" + pull.number + ": " + workflowId +
|
|
" has no run for the current head; not applicable.",
|
|
);
|
|
}
|
|
}
|
|
|
|
if (selectedRuns.length === 0) {
|
|
core.info(
|
|
"PR #" + pull.number + ": no vendor CI runs found.",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
const failedRuns = selectedRuns.filter(
|
|
({ run }) =>
|
|
run.status === "completed" &&
|
|
run.conclusion === "failure",
|
|
);
|
|
if (failedRuns.length === 0) {
|
|
core.info(
|
|
"PR #" + pull.number + ": vendor CI has no failures.",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Keep the approval and head SHA checks close to the
|
|
// mutations. A push can still race this small window, just as
|
|
// it can in the latest-main retry workflow.
|
|
const currentPullRequest = await getPullRequest(pull.number);
|
|
if (
|
|
currentPullRequest?.headRefOid !== pullRequest.headRefOid ||
|
|
!(await isEligible(pull.number, currentPullRequest))
|
|
) {
|
|
core.info(
|
|
"PR #" + pull.number +
|
|
": eligibility or head changed; skipping reruns.",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
for (const { workflowId, run } of failedRuns) {
|
|
const attempt = run.run_attempt || 1;
|
|
if (attempt >= maxAttempts) {
|
|
core.warning(
|
|
"PR #" + pull.number + ": " + workflowId + " run " +
|
|
run.id + " failed on attempt " + attempt +
|
|
"; retry limit reached.",
|
|
);
|
|
addSummaryRow(
|
|
pull,
|
|
workflowId,
|
|
run,
|
|
"Retry limit reached",
|
|
);
|
|
continue;
|
|
}
|
|
if (
|
|
Date.now() - Date.parse(run.created_at) >
|
|
maxRunAgeMs
|
|
) {
|
|
core.warning(
|
|
"PR #" + pull.number + ": " + workflowId + " run " +
|
|
run.id + " is older than 30 days; skipping.",
|
|
);
|
|
addSummaryRow(
|
|
pull,
|
|
workflowId,
|
|
run,
|
|
"Run older than 30 days",
|
|
);
|
|
continue;
|
|
}
|
|
|
|
if (dryRun) {
|
|
core.notice(
|
|
"PR #" + pull.number + ": would rerun failed jobs for " +
|
|
workflowId + " run " + run.id + " after attempt " +
|
|
attempt + ".",
|
|
);
|
|
addSummaryRow(pull, workflowId, run, "Dry run");
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
await github.request(
|
|
"POST /repos/{owner}/{repo}/actions/runs/{run_id}/rerun-failed-jobs",
|
|
{ owner, repo, run_id: run.id },
|
|
);
|
|
core.notice(
|
|
"PR #" + pull.number + ": rerunning failed jobs for " +
|
|
workflowId + " run " + run.id + " after attempt " +
|
|
attempt + ".",
|
|
);
|
|
addSummaryRow(
|
|
pull,
|
|
workflowId,
|
|
run,
|
|
"Rerun started",
|
|
);
|
|
} catch (error) {
|
|
core.warning(
|
|
"PR #" + pull.number + ": unable to rerun failed jobs " +
|
|
"for " + workflowId + " run " + run.id + ": " +
|
|
error.message,
|
|
);
|
|
addSummaryRow(
|
|
pull,
|
|
workflowId,
|
|
run,
|
|
"Failed to start",
|
|
);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
core.warning(
|
|
"PR #" + pull.number + ": scan failed: " + error.message,
|
|
);
|
|
addSummaryRow(pull, null, null, "Failed to inspect");
|
|
}
|
|
}
|
|
} finally {
|
|
core.summary.addHeading("Approved PR retry summary");
|
|
if (summaryRows.length > 0) {
|
|
core.summary.addTable([
|
|
[
|
|
{ data: "PR", header: true },
|
|
{ data: "Workflow", header: true },
|
|
{ data: "Run", header: true },
|
|
{ data: "Result", header: true },
|
|
],
|
|
...summaryRows,
|
|
]);
|
|
} else {
|
|
core.summary.addRaw("No reruns were eligible.");
|
|
}
|
|
await core.summary.write();
|
|
}
|