name: Nightly Mutation on: schedule: - cron: "17 3 * * *" workflow_dispatch: permissions: contents: read jobs: stryker: name: Stryker mutation (batch ${{ matrix.batch.name }} — advisory) runs-on: ubuntu-latest # Mutation testing is expensive. History of the budget: # - Full 8-module set TIMED OUT at the 180min cap (run 27705123780 = exactly 180min). # The two god-files chatCore.ts/combo.ts dominated ~2/3 of the mutants and were # removed from stryker.conf.json `mutate`. # - The remaining 6 modules in 3 batches: auth.ts and accountFallback.ts are large; the # a (auth+publicCreds) and b (accountFallback+error) batches still ran near the 180min # cap (the perTest dry-run over ~130 covering test files is itself costly), so the two # big modules are now ISOLATED into their own batches (a=auth, b=accountFallback). # - Onda 3 / Fase 9 T5 re-add: combo.ts was split into 11 leaves; the 8 well-covered # combo/* leaves are back in `mutate` (covered by the 24 combo-*.test.ts), grouped into # 2 batches (d=heavy, e=light). After #4204 (D7b) merged, the reset-aware quota pair # quotaScoring/quotaStrategies was added as batch f. A covering-test audit then added the # 6 chatCore/* leaves with direct unit coverage as batch g. A follow-up then wrote dedicated # unit tests for 6 more leaves and added them as batch h. A final follow-up added dedicated # tests (no mock.module — fetch-override + crafted inputs + temp-DATA_DIR) for telemetryHelpers # + memorySkillsInjection + semanticCache (its cache-HIT block now has a setCachedResponse # fixture) as batch i — ALL 15/15 chatCore leaves are now mutated. See # _mutate_godfiles_excluded_comment in stryker.conf.json. # 9 PARALLEL batches, each overriding the mutate set via `--mutate` (Stryker 9 CLI: # `-m, --mutate `; the conf's `mutate[]` remains the local-run default/union). # - Cold-seeding budget (per-batch `timeout-minutes: ${{ matrix.batch.timeout || 180 }}`): # a COLD run must COMPLETE once to write stryker-incremental.json (Stryker writes it only on # a successful finish); a job cancelled at the cap writes nothing, so the next run is cold # again — an infinite never-seeds loop. actions/cache is also branch-scoped, so each branch # (incl. release) must seed its OWN cache via a run with enough headroom. Measured cold runs # Measured cold totals (run 27801802713, extrapolated from the % at the 180/350 cancel point): # auth ~375min (2301 mutants — EXCEEDS the 360min job max even isolated), accountFallback # ~358min (1441), the c security quartet ~348min (1163), d ~197min (1316), g=142, h=132, e=66, # f=45, i=33. The widely-covered modules blow the budget because the tap-runner re-runs every # covering test file per mutant (a perTest fixed cost over ~138 test files, times thousands of # mutants). A flat timeout bump cannot rescue auth (>360min max) — so the three over-budget # batches are SPLIT so each half fits: auth->a1/a2 and accountFallback->b1/b2 by mutation range # (`file:startLine-endLine`), the c quartet->c1/c2 by module pair. Splitting also seeds each # sub-batch's own incremental cache, after which nightlies re-test only changed mutants. # Full coverage every night in parallel; wall-clock = the slowest batch's cold run until seeded. # Runs at stryker concurrency=4 with per-process DATA_DIR isolation # (tests/_setup/isolateDataDir.ts) — see _concurrency_comment in stryker.conf.json. strategy: fail-fast: false matrix: batch: # Per-batch `timeout` (minutes) tiers the cold-seeding budget by measured cost; batches # without the key default to 180. Cold-run profiling (run 27801802713) showed the # widely-covered modules need FAR more than the 180 cap because the tap-runner re-runs every # covering test file per mutant: auth ~375min (2301 mutants, EXCEEDS the 360min GitHub job # max even isolated), accountFallback ~358min, the c security quartet ~348min — none fit a # single job. So auth/accountFallback are split by MUTATION RANGE (`file:startLine-endLine`, # ~half the mutants each) into a1/a2, b1/b2; the c quartet is split by MODULE pair into # c1/c2. d (3 combo modules, ~197min) stays whole. Split-heavy batches get 300min headroom # for their cold seeding run; once each batch completes once and writes # stryker-incremental.json, later runs re-test only changed mutants and finish far faster. # g/h (142/132min cold) keep a 240 buffer; e/f/i (33-66min) keep the 180 default. - name: a1 mutate: "src/sse/services/auth.ts:1-1109" timeout: 300 - name: a2 mutate: "src/sse/services/auth.ts:1110-2218" timeout: 300 - name: b1 mutate: "open-sse/services/accountFallback.ts:1-863" timeout: 300 - name: b2 mutate: "open-sse/services/accountFallback.ts:864-1726" timeout: 300 - name: c1 mutate: "src/server/authz/routeGuard.ts,src/shared/utils/circuitBreaker.ts" timeout: 300 - name: c2 mutate: "open-sse/utils/error.ts,open-sse/utils/publicCreds.ts" timeout: 300 - name: d mutate: "open-sse/services/combo/comboStructure.ts,open-sse/services/combo/autoStrategy.ts,open-sse/services/combo/validateQuality.ts" timeout: 300 - name: e mutate: "open-sse/services/combo/shadowRouting.ts,open-sse/services/combo/targetSorters.ts,open-sse/services/combo/comboPredicates.ts,open-sse/services/combo/rrState.ts,open-sse/services/combo/comboData.ts" - name: f mutate: "open-sse/services/combo/quotaScoring.ts,open-sse/services/combo/quotaStrategies.ts" - name: g mutate: "open-sse/handlers/chatCore/comboContextCache.ts,open-sse/handlers/chatCore/idempotency.ts,open-sse/handlers/chatCore/passthroughHelpers.ts,open-sse/handlers/chatCore/responseHeaders.ts,open-sse/handlers/chatCore/sanitization.ts,open-sse/handlers/chatCore/upstreamTimeouts.ts" timeout: 240 - name: h mutate: "open-sse/handlers/chatCore/headers.ts,open-sse/handlers/chatCore/logTruncation.ts,open-sse/handlers/chatCore/memoryExtraction.ts,open-sse/handlers/chatCore/nonStreamingSse.ts,open-sse/handlers/chatCore/passthroughToolNames.ts,open-sse/handlers/chatCore/executorHelpers.ts" timeout: 240 - name: i mutate: "open-sse/handlers/chatCore/telemetryHelpers.ts,open-sse/handlers/chatCore/memorySkillsInjection.ts,open-sse/handlers/chatCore/semanticCache.ts" # Per-batch budget: split-heavy batches (a1/a2/b1/b2/c1/c2/d) override to 300min, g/h to 240min; # the rest default to 180min. `matrix.batch.timeout` is null for batches without the key -> `|| 180`. # NOTE: a1+a2 both mutate auth.ts (disjoint line ranges) and b1+b2 both mutate accountFallback.ts; # when merging the per-batch mutation.json for radiography/scores, same-file mutants from sibling # ranges must be UNIONED (scripts/check/check-mutation-ratchet.mjs::measureMutationScores and # scripts/quality/mutation-radiography.mjs both merge per file). timeout-minutes: ${{ matrix.batch.timeout || 180 }} steps: - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-node@v6 with: node-version: "24" cache: npm - run: npm ci - name: Restore Stryker incremental cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: reports/mutation/stryker-incremental.json key: stryker-incremental-${{ matrix.batch.name }}-${{ github.run_id }} restore-keys: stryker-incremental-${{ matrix.batch.name }}- - name: Run Stryker (advisory) id: stryker continue-on-error: true env: BATCH_MUTATE: ${{ matrix.batch.mutate }} run: npx stryker run --mutate "$BATCH_MUTATE" - name: Upload mutation report if: always() uses: actions/upload-artifact@v7 with: name: mutation-report-${{ matrix.batch.name }} path: reports/mutation/ if-no-files-found: warn retention-days: 14 # Aggregation gate (T3): each split batch emits a PARTIAL view of a mutated file # (auth.ts lives in a1+a2, accountFallback in b1+b2), so a PER-BATCH ratchet would # only ever see half a file vs the whole-file baseline. This job runs AFTER every # batch, downloads all reports, and ratchets the MERGED per-module scores # (check-mutation-ratchet UNIONS same-file mutants across reports) against the # dedicatedGate `mutationScore.*` floors in quality-baseline.json (seeded ~2pt below # the first full measurement). Blocking: a module dropping below its floor fails the # run. Missing reports (e.g. an artifact-upload flake) are skipped, never failed. mutation-ratchet: name: Mutation score ratchet (blocking) needs: stryker if: always() runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v6 with: persist-credentials: false - uses: actions/setup-node@v6 with: node-version: "24" - name: Download all mutation reports uses: actions/download-artifact@v8 with: pattern: mutation-report-* path: reports/all - name: Ratchet merged per-module mutation scores run: node scripts/check/check-mutation-ratchet.mjs reports/all/*/mutation.json --ratchet