wehub-resource-sync
439 lines
20 KiB
JavaScript
439 lines
20 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* stats.mjs — Lifetime pipeline stats aggregator (zero-token). #1604
|
|
*
|
|
* One JSON contract for "how is my pipeline doing, lifetime": tracker roll-up,
|
|
* cumulative funnel, lifetime scanner totals from scan-history.tsv, portal
|
|
* coverage from portals.yml, and follow-up compliance. Reads durable data
|
|
* files only — no LLM cost anywhere.
|
|
*
|
|
* Run: node stats.mjs (JSON to stdout)
|
|
* node stats.mjs --summary (human-readable table)
|
|
*
|
|
* Sections degrade to null when their source file is missing, and
|
|
* `metadata.sources` says which files were found — a fresh clone with zero
|
|
* user data emits the full contract shape with null sections.
|
|
*
|
|
* `runs` aggregates data/scan-runs.tsv (per-run counters written by scan.mjs,
|
|
* #1604 PR-2) — null until the first non-dry scan creates the file.
|
|
*/
|
|
|
|
import { readFileSync, existsSync } from 'fs';
|
|
import { join, dirname } from 'path';
|
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
import yaml from 'js-yaml';
|
|
import { resolveColumns, parseTrackerRow } from './tracker-parse.mjs';
|
|
import { normalizeStatus } from './followup-cadence.mjs';
|
|
|
|
const ROOT = dirname(fileURLToPath(import.meta.url));
|
|
const APPS_FILE = join(ROOT, 'data', 'applications.md');
|
|
const SCAN_HISTORY_FILE = join(ROOT, 'data', 'scan-history.tsv');
|
|
const FOLLOWUPS_FILE = join(ROOT, 'data', 'follow-ups.md');
|
|
const SCAN_RUNS_FILE = join(ROOT, 'data', 'scan-runs.tsv');
|
|
const PORTALS_FILE = join(ROOT, 'portals.yml');
|
|
|
|
const CANONICAL_STATUSES = ['Evaluated', 'Applied', 'Responded', 'Interview', 'Offer', 'Rejected', 'Discarded', 'SKIP'];
|
|
|
|
// In-flight applications. Deliberately NARROWER than the dashboard's
|
|
// ActiveApps (which also counts Evaluated): an evaluated-but-never-sent row is
|
|
// a candidate, not an application in flight.
|
|
const ACTIVE_STATUSES = new Set(['Applied', 'Responded', 'Interview', 'Offer']);
|
|
|
|
// Rows that count toward avgScoreApplied — jobs the user actually pursued.
|
|
// Plain avgScore mixes in SKIP/Discarded and understates real fit.
|
|
const PURSUED_STATUSES = new Set(['Applied', 'Responded', 'Interview', 'Offer', 'Rejected']);
|
|
|
|
const round1 = (n) => Math.round(n * 10) / 10;
|
|
const pct = (part, total) => (total > 0 ? round1((part / total) * 100) : 0);
|
|
|
|
/** Canonical display form ("aplicado" → "Applied", "skip" → "SKIP"); unknown → "Unknown" (counted, never dropped). */
|
|
function canonicalStatus(raw) {
|
|
const norm = normalizeStatus(String(raw ?? ''));
|
|
if (norm === 'skip') return 'SKIP';
|
|
const cased = norm.charAt(0).toUpperCase() + norm.slice(1);
|
|
return CANONICAL_STATUSES.includes(cased) ? cased : 'Unknown';
|
|
}
|
|
|
|
// ── Tracker roll-up ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Roll up applications.md: counts per canonical status, score stats, pdf and
|
|
* report coverage, in-flight count. Header-aware via tracker-parse.mjs; CRLF
|
|
* input is normalized first (Windows checkouts).
|
|
*
|
|
* @param {string} content - Raw applications.md text.
|
|
*/
|
|
export function computeTrackerStats(content) {
|
|
const lines = String(content ?? '').replace(/\r/g, '').split('\n');
|
|
const colmap = resolveColumns(lines);
|
|
const byStatus = Object.fromEntries(CANONICAL_STATUSES.map((s) => [s, 0]));
|
|
let total = 0, scoreSum = 0, scoreCount = 0, topScore = 0, withPdf = 0, withReport = 0, activeApps = 0;
|
|
let pursuedSum = 0, pursuedCount = 0;
|
|
for (const line of lines) {
|
|
const row = parseTrackerRow(line, colmap);
|
|
if (!row) continue;
|
|
total++;
|
|
const status = canonicalStatus(row.status);
|
|
byStatus[status] = (byStatus[status] || 0) + 1;
|
|
if (ACTIVE_STATUSES.has(status)) activeApps++;
|
|
const score = parseFloat(String(row.score || '').replace(/\*/g, ''));
|
|
if (!Number.isNaN(score) && score > 0) {
|
|
scoreSum += score;
|
|
scoreCount++;
|
|
if (score > topScore) topScore = score;
|
|
if (PURSUED_STATUSES.has(status)) { pursuedSum += score; pursuedCount++; }
|
|
}
|
|
if ((row.pdf || '').includes('✅')) withPdf++;
|
|
if (/\[.*\]\(.*\)/.test(row.report || '')) withReport++;
|
|
}
|
|
return {
|
|
total,
|
|
byStatus,
|
|
avgScore: scoreCount > 0 ? round1(scoreSum / scoreCount) : null,
|
|
avgScoreApplied: pursuedCount > 0 ? round1(pursuedSum / pursuedCount) : null,
|
|
topScore: topScore > 0 ? topScore : null,
|
|
pdfPct: pct(withPdf, total),
|
|
reportPct: pct(withReport, total),
|
|
activeApps,
|
|
};
|
|
}
|
|
|
|
/** Map of tracker number → canonical status (for follow-up compliance). */
|
|
export function trackerStatusByNum(content) {
|
|
const lines = String(content ?? '').replace(/\r/g, '').split('\n');
|
|
const colmap = resolveColumns(lines);
|
|
const byNum = new Map();
|
|
for (const line of lines) {
|
|
const row = parseTrackerRow(line, colmap);
|
|
if (row) byNum.set(row.num, canonicalStatus(row.status));
|
|
}
|
|
return byNum;
|
|
}
|
|
|
|
// ── Cumulative funnel ───────────────────────────────────────────────
|
|
|
|
/**
|
|
* Cumulative funnel: everX = "reached stage X or beyond, ever". The math
|
|
* mirrors the dashboard's ComputeProgressMetrics (career.go): Rejected counts
|
|
* into everApplied (a rejection proves a submission), and each later stage
|
|
* sums itself plus everything beyond it. Rates are relative to everApplied.
|
|
*
|
|
* Keys are deliberately NOT bare status names — `tracker.byStatus.Applied` is
|
|
* "currently in Applied" while `everApplied` is "ever applied"; the same word
|
|
* for two different numbers would read as a bug.
|
|
*
|
|
* Known limitation: statuses are snapshots, so a Rejected row that never got a
|
|
* response is indistinguishable from one rejected after interviews — middle
|
|
* stages are lower bounds until status-transition logging exists (#1428).
|
|
*
|
|
* This is the canonical funnel definition for career-ops going forward;
|
|
* dashboard/web consuming this JSON instead of keeping independent copies is
|
|
* a named follow-up in #1604.
|
|
*/
|
|
export function computeFunnel(byStatus) {
|
|
const n = (k) => byStatus[k] || 0;
|
|
const everApplied = n('Applied') + n('Responded') + n('Interview') + n('Offer') + n('Rejected');
|
|
const everResponded = n('Responded') + n('Interview') + n('Offer');
|
|
const everInterview = n('Interview') + n('Offer');
|
|
const everOffer = n('Offer');
|
|
return {
|
|
everApplied,
|
|
everResponded,
|
|
everInterview,
|
|
everOffer,
|
|
responseRate: pct(everResponded, everApplied),
|
|
interviewRate: pct(everInterview, everApplied),
|
|
offerRate: pct(everOffer, everApplied),
|
|
smallSample: everApplied < 10,
|
|
};
|
|
}
|
|
|
|
// ── Lifetime scanner totals ─────────────────────────────────────────
|
|
|
|
/** ISO week key ("2026-W27") for a YYYY-MM-DD string; null for bad input. Exported for the year-boundary tests. */
|
|
export function isoWeek(dateStr) {
|
|
const d = new Date(`${dateStr}T00:00:00Z`);
|
|
if (Number.isNaN(d.getTime())) return null;
|
|
const day = (d.getUTCDay() + 6) % 7; // Mon=0
|
|
d.setUTCDate(d.getUTCDate() - day + 3); // nearest Thursday decides the ISO year
|
|
const jan4 = new Date(Date.UTC(d.getUTCFullYear(), 0, 4));
|
|
const week = 1 + Math.round(((d - jan4) / 86400000 - 3 + ((jan4.getUTCDay() + 6) % 7)) / 7);
|
|
return `${d.getUTCFullYear()}-W${String(week).padStart(2, '0')}`;
|
|
}
|
|
|
|
/**
|
|
* Lifetime totals from scan-history.tsv. Malformed rows (no URL in column 0)
|
|
* are skipped rather than crashing — a torn row from an interrupted append
|
|
* must never poison the aggregate. Extra trailing columns (e.g. the
|
|
* fingerprint column, #1597) are ignored.
|
|
*
|
|
* @param {string} content - Raw scan-history.tsv text.
|
|
* @param {{weeks?: number}} [opts] - How many trailing weeks of addedPerWeek to keep.
|
|
*/
|
|
export function computeScanStats(content, { weeks = 8 } = {}) {
|
|
const lines = String(content ?? '').replace(/\r/g, '').split('\n').filter((l) => l.trim());
|
|
const byPortal = {}, byStatus = {};
|
|
const companies = new Set();
|
|
const weekCounts = new Map();
|
|
let totalRecorded = 0, added = 0, firstSeen = null, lastSeen = null;
|
|
for (const line of lines) {
|
|
const cols = line.split('\t');
|
|
if (cols[0] === 'url') continue; // header
|
|
if (!/^https?:\/\//.test(cols[0])) continue; // torn/malformed row
|
|
totalRecorded++;
|
|
const [, date, portal, , company, statusRaw] = cols;
|
|
const status = (statusRaw || 'added').trim() || 'added';
|
|
byStatus[status] = (byStatus[status] || 0) + 1;
|
|
if (portal) byPortal[portal] = (byPortal[portal] || 0) + 1;
|
|
if (company) companies.add(company.toLowerCase());
|
|
if (status === 'added') {
|
|
added++;
|
|
const wk = isoWeek(date);
|
|
if (wk) weekCounts.set(wk, (weekCounts.get(wk) || 0) + 1);
|
|
}
|
|
if (date && /^\d{4}-\d{2}-\d{2}$/.test(date)) {
|
|
if (!firstSeen || date < firstSeen) firstSeen = date;
|
|
if (!lastSeen || date > lastSeen) lastSeen = date;
|
|
}
|
|
}
|
|
const addedPerWeek = [...weekCounts.entries()]
|
|
.sort((a, b) => (a[0] < b[0] ? -1 : 1))
|
|
.slice(-weeks)
|
|
.map(([week, count]) => ({ week, count }));
|
|
return { totalRecorded, added, byStatus, byPortal, distinctCompanies: companies.size, firstSeen, lastSeen, addedPerWeek };
|
|
}
|
|
|
|
/** Lowercased set of company names that ever appeared in scan-history.tsv. */
|
|
export function scanCompanyNames(content) {
|
|
const names = new Set();
|
|
for (const line of String(content ?? '').replace(/\r/g, '').split('\n')) {
|
|
const cols = line.split('\t');
|
|
if (!/^https?:\/\//.test(cols[0] || '')) continue;
|
|
const company = (cols[4] || '').trim();
|
|
if (company) names.add(company.toLowerCase());
|
|
}
|
|
return [...names];
|
|
}
|
|
|
|
// ── Portal coverage ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Configured-vs-producing portal coverage.
|
|
*
|
|
* producingPct = share of configured tracked_companies whose name has EVER
|
|
* appeared as a scanned job's company. A low number usually means "no matching
|
|
* openings from that company yet", NOT a broken portal — the --summary label
|
|
* carries the same caveat. Matching is exact-lowercase between portals.yml
|
|
* `name` and the scanner's company string; that undercounts when names differ
|
|
* ("Acme" vs "Acme, Inc."). Documented v1 contract — no silent fuzzy-matching.
|
|
*
|
|
* @param {string} portalsYmlContent - Raw portals.yml text.
|
|
* @param {object|null} scanStats - Result of computeScanStats (for activePortals).
|
|
* @param {string[]} [producingCompanyNames] - From scanCompanyNames().
|
|
*/
|
|
export function computePortalStats(portalsYmlContent, scanStats, producingCompanyNames = []) {
|
|
let cfg;
|
|
try {
|
|
cfg = yaml.load(String(portalsYmlContent ?? '')) || {};
|
|
} catch {
|
|
return null; // malformed YAML → no portal section rather than a crash
|
|
}
|
|
const companies = Array.isArray(cfg.tracked_companies) ? cfg.tracked_companies : [];
|
|
const boards = Array.isArray(cfg.job_boards) ? cfg.job_boards : [];
|
|
const configuredNames = new Set(
|
|
companies.map((c) => String(c?.name || '').toLowerCase()).filter(Boolean),
|
|
);
|
|
const producing = new Set(producingCompanyNames.map((n) => String(n).toLowerCase()));
|
|
let producingCompanies = 0;
|
|
for (const name of configuredNames) if (producing.has(name)) producingCompanies++;
|
|
return {
|
|
configuredCompanies: companies.length,
|
|
configuredBoards: boards.length,
|
|
activePortals: Object.keys(scanStats?.byPortal || {}).length,
|
|
producingCompanies,
|
|
producingPct: pct(producingCompanies, configuredNames.size),
|
|
};
|
|
}
|
|
|
|
// ── Follow-up compliance ────────────────────────────────────────────
|
|
|
|
/**
|
|
* Follow-up compliance from follow-ups.md (same table shape followup-cadence
|
|
* parses: | num | appNum | date | company | role | channel | contact | notes |).
|
|
* appliedWithoutFollowup counts tracker rows currently in Applied with zero
|
|
* logged follow-ups — the rows the followup mode would flag as aging silently.
|
|
*
|
|
* @param {string} followupsContent - Raw follow-ups.md text.
|
|
* @param {Map<number,string>} trackerByNum - From trackerStatusByNum().
|
|
*/
|
|
export function computeFollowupStats(followupsContent, trackerByNum) {
|
|
const byApp = new Map();
|
|
let totalFollowups = 0;
|
|
for (const line of String(followupsContent ?? '').replace(/\r/g, '').split('\n')) {
|
|
if (!line.startsWith('|')) continue;
|
|
const parts = line.split('|').map((s) => s.trim());
|
|
if (parts.length < 8) continue;
|
|
const num = parseInt(parts[1], 10);
|
|
const appNum = parseInt(parts[2], 10);
|
|
if (Number.isNaN(num) || Number.isNaN(appNum)) continue; // header/separator
|
|
totalFollowups++;
|
|
byApp.set(appNum, (byApp.get(appNum) || 0) + 1);
|
|
}
|
|
let appliedWithoutFollowup = 0;
|
|
for (const [num, status] of trackerByNum) {
|
|
if (status === 'Applied' && !byApp.has(num)) appliedWithoutFollowup++;
|
|
}
|
|
return {
|
|
totalFollowups,
|
|
appsWithFollowups: byApp.size,
|
|
appliedWithoutFollowup,
|
|
avgPerApp: byApp.size > 0 ? round1(totalFollowups / byApp.size) : 0,
|
|
};
|
|
}
|
|
|
|
// ── Scan-run trends ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Aggregate data/scan-runs.tsv (written by scan.mjs, one row per non-dry run).
|
|
*
|
|
* Header-name parsing, NEVER positional: columns may be appended in later
|
|
* schema versions and a positional slice would silently miscount from then on.
|
|
* Torn rows (crash mid-append) and rows with a bad timestamp are skipped;
|
|
* failed runs count in totalRuns/failedRuns but are excluded from averages so
|
|
* an aborted run doesn't drag the trend down.
|
|
*
|
|
* @param {string} content - Raw scan-runs.tsv text.
|
|
* @returns {object|null} null for empty/unknown-schema files.
|
|
*/
|
|
export function computeRunStats(content) {
|
|
const lines = String(content ?? '').replace(/\r/g, '').split('\n').filter((l) => l.trim());
|
|
if (lines.length < 2) return null;
|
|
const header = lines[0].split('\t');
|
|
const idx = Object.fromEntries(header.map((h, i) => [h, i]));
|
|
if (idx.timestamp == null || idx.found == null) return null; // unknown schema
|
|
const filterCols = header.filter((h) => h.startsWith('filtered_'));
|
|
const rows = [];
|
|
for (const line of lines.slice(1)) {
|
|
const cols = line.split('\t');
|
|
if (cols.length < header.length) continue; // torn row
|
|
if (!/^\d{4}-\d{2}-\d{2}/.test(cols[idx.timestamp] || '')) continue;
|
|
const num = (name) => { const v = Number(cols[idx[name]]); return Number.isNaN(v) ? 0 : v; };
|
|
rows.push({
|
|
date: cols[idx.timestamp].slice(0, 10),
|
|
status: idx.status != null ? cols[idx.status] : 'completed',
|
|
found: num('found'),
|
|
filtered: filterCols.reduce((a, h) => a + num(h), 0),
|
|
newAdded: num('new_added'),
|
|
});
|
|
}
|
|
if (rows.length === 0) return null;
|
|
const completed = rows.filter((r) => r.status !== 'failed');
|
|
const sum = (arr, k) => arr.reduce((a, r) => a + r[k], 0);
|
|
return {
|
|
totalRuns: rows.length,
|
|
failedRuns: rows.length - completed.length,
|
|
lastRunDate: rows.map((r) => r.date).sort().at(-1),
|
|
avgFoundPerRun: completed.length ? round1(sum(completed, 'found') / completed.length) : 0,
|
|
avgNewPerRun: completed.length ? round1(sum(completed, 'newAdded') / completed.length) : 0,
|
|
filterRemovalPct: pct(sum(completed, 'filtered'), sum(completed, 'found')),
|
|
};
|
|
}
|
|
|
|
// ── Assembler + CLI ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Assemble the full stats contract. Every section is null when its source
|
|
* file is missing; metadata.sources records what was found.
|
|
*/
|
|
export function computeAllStats({
|
|
appsFile = APPS_FILE,
|
|
scanHistoryFile = SCAN_HISTORY_FILE,
|
|
followupsFile = FOLLOWUPS_FILE,
|
|
scanRunsFile = SCAN_RUNS_FILE,
|
|
portalsFile = PORTALS_FILE,
|
|
} = {}) {
|
|
const read = (f) => (existsSync(f) ? readFileSync(f, 'utf-8') : null);
|
|
const apps = read(appsFile);
|
|
const scanHist = read(scanHistoryFile);
|
|
const fups = read(followupsFile);
|
|
const portals = read(portalsFile);
|
|
const runs = read(scanRunsFile);
|
|
const tracker = apps ? computeTrackerStats(apps) : null;
|
|
const scan = scanHist ? computeScanStats(scanHist) : null;
|
|
return {
|
|
metadata: {
|
|
generatedAt: new Date().toISOString().slice(0, 10),
|
|
sources: {
|
|
tracker: !!apps,
|
|
scanHistory: !!scanHist,
|
|
followups: !!fups,
|
|
portals: !!portals,
|
|
scanRuns: !!runs,
|
|
},
|
|
},
|
|
tracker,
|
|
funnel: tracker ? computeFunnel(tracker.byStatus) : null,
|
|
scan,
|
|
portals: portals ? computePortalStats(portals, scan, scanHist ? scanCompanyNames(scanHist) : []) : null,
|
|
followups: fups && apps ? computeFollowupStats(fups, trackerStatusByNum(apps)) : null,
|
|
runs: runs ? computeRunStats(runs) : null,
|
|
};
|
|
}
|
|
|
|
/** Human-readable table. pdfPct/reportPct are JSON-only by design — no job-search decision follows from them. */
|
|
function printSummary(stats) {
|
|
const line = '━'.repeat(45);
|
|
console.log(`\n${line}`);
|
|
console.log(`Pipeline Stats — ${stats.metadata.generatedAt}`);
|
|
console.log(line);
|
|
const t = stats.tracker;
|
|
if (t) {
|
|
const fit = t.avgScore != null
|
|
? ` | avg fit ${t.avgScore}/5${t.avgScoreApplied != null ? ` (pursued roles ${t.avgScoreApplied}/5)` : ''} | top ${t.topScore}`
|
|
: '';
|
|
console.log(`Tracker: ${t.total} total | ${t.activeApps} active${fit}`);
|
|
const statusLine = Object.entries(t.byStatus).filter(([, c]) => c > 0).map(([s, c]) => `${s} ${c}`).join(' · ');
|
|
if (statusLine) console.log(`Status: ${statusLine}`);
|
|
} else {
|
|
console.log('Tracker: — no data (data/applications.md missing)');
|
|
}
|
|
const f = stats.funnel;
|
|
if (f) {
|
|
const small = f.smallSample ? ' (small sample — rates indicative only)' : '';
|
|
console.log(`Funnel: ever applied ${f.everApplied} → responded ${f.everResponded} (${f.responseRate}%) → interview ${f.everInterview} (${f.interviewRate}%) → offer ${f.everOffer} (${f.offerRate}%)${small}`);
|
|
}
|
|
const s = stats.scan;
|
|
if (s) {
|
|
const portalsLine = Object.entries(s.byPortal).sort((a, b) => b[1] - a[1]).slice(0, 5).map(([p, c]) => `${p} ${c}`).join(' · ');
|
|
console.log(`Scanner: ${s.totalRecorded} jobs recorded${s.firstSeen ? ` since ${s.firstSeen}` : ''} | ${s.added} added | ${s.distinctCompanies} companies${portalsLine ? ` | ${portalsLine}` : ''}`);
|
|
} else {
|
|
console.log('Scanner: — no data (data/scan-history.tsv missing)');
|
|
}
|
|
const p = stats.portals;
|
|
if (p) {
|
|
console.log(`Portals: ${p.configuredCompanies} companies + ${p.configuredBoards} boards configured | ${p.producingCompanies} have produced a match (${p.producingPct}%) — low ≠ broken, may just be no openings`);
|
|
} else {
|
|
console.log('Portals: — no data (portals.yml missing)');
|
|
}
|
|
const fu = stats.followups;
|
|
if (fu) {
|
|
console.log(`Follow-ups: ${fu.totalFollowups} sent across ${fu.appsWithFollowups} apps | ${fu.appliedWithoutFollowup} Applied apps with none | avg ${fu.avgPerApp}/app`);
|
|
} else {
|
|
console.log('Follow-ups: — no data (data/follow-ups.md missing)');
|
|
}
|
|
const r = stats.runs;
|
|
if (r) {
|
|
const failed = r.failedRuns > 0 ? ` | ${r.failedRuns} failed` : '';
|
|
console.log(`Runs: ${r.totalRuns} recorded (last ${r.lastRunDate})${failed} | avg ${r.avgFoundPerRun} found / ${r.avgNewPerRun} new per run | filters remove ${r.filterRemovalPct}%`);
|
|
} else {
|
|
console.log('Runs: — no data (data/scan-runs.tsv missing; created by the next scan)');
|
|
}
|
|
console.log('');
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
const stats = computeAllStats();
|
|
if (process.argv.includes('--summary')) printSummary(stats);
|
|
else console.log(JSON.stringify(stats, null, 2));
|
|
}
|