wehub-resource-sync
117 lines
4.5 KiB
JavaScript
117 lines
4.5 KiB
JavaScript
/**
|
|
* Live probe for Piece 4 (pre-send validation) and Piece 5 (config-health
|
|
* banner). Exercises both happy and unhappy paths in a single run.
|
|
*
|
|
* Setup expectation: dev electron running with ENABLE_CDP=1, active
|
|
* model is DeepSeek (which is the case after the prior repro scripts).
|
|
*/
|
|
const { attach } = require("./e2e-attach");
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const os = require("os");
|
|
|
|
const ENV_FILE = path.join(os.homedir(), "AppData", "Local", "hermes", ".env");
|
|
const ENV_BACKUP = ENV_FILE + ".probe-validation-bk";
|
|
|
|
function readEnvKeys() {
|
|
const m = {};
|
|
for (const line of fs.readFileSync(ENV_FILE, "utf-8").split("\n")) {
|
|
const mm = line.match(/^([A-Z][A-Z0-9_]*)=(.*)$/);
|
|
if (mm) m[mm[1]] = mm[2];
|
|
}
|
|
return m;
|
|
}
|
|
|
|
(async () => {
|
|
const { browser, page } = await attach();
|
|
|
|
// --- A. Happy path: validateChatReadiness should be OK as-configured ---
|
|
const okState = await page.evaluate(async () => {
|
|
return await window.hermesAPI.validateChatReadiness();
|
|
});
|
|
console.log("[A] happy path validateChatReadiness:", JSON.stringify(okState));
|
|
|
|
// --- B. Remove DEEPSEEK_API_KEY from .env AND clear any deepseek
|
|
// credential-pool entries from auth.json. Both sources have
|
|
// to be empty for the validator to flag "missing key", since
|
|
// the engine accepts credentials from either location
|
|
// (the #367-driven extension to the validation path).
|
|
fs.copyFileSync(ENV_FILE, ENV_BACKUP);
|
|
const AUTH = path.join(
|
|
os.homedir(),
|
|
"AppData",
|
|
"Local",
|
|
"hermes",
|
|
"auth.json",
|
|
);
|
|
const AUTH_BACKUP = AUTH + ".probe-validation-auth-bk";
|
|
if (fs.existsSync(AUTH)) fs.copyFileSync(AUTH, AUTH_BACKUP);
|
|
const before = fs.readFileSync(ENV_FILE, "utf-8");
|
|
const without = before.replace(/^DEEPSEEK_API_KEY=.*\n?/m, "");
|
|
fs.writeFileSync(ENV_FILE, without);
|
|
if (fs.existsSync(AUTH)) {
|
|
const auth = JSON.parse(fs.readFileSync(AUTH, "utf-8"));
|
|
if (auth.providers) delete auth.providers.deepseek;
|
|
if (auth.credential_pool) delete auth.credential_pool.deepseek;
|
|
fs.writeFileSync(AUTH, JSON.stringify(auth, null, 2));
|
|
}
|
|
console.log("[setup] removed DEEPSEEK_API_KEY from .env + cleared deepseek from auth.json");
|
|
|
|
// The readEnv() in main process has a 5s TTL cache. Bust by setting and
|
|
// unsetting some random key, OR just wait — but easier to flip it via
|
|
// setEnv which invalidates the cache.
|
|
await page.evaluate(async () => {
|
|
await window.hermesAPI.setEnv("__VALIDATION_PROBE__", String(Date.now()));
|
|
});
|
|
// Now call validation again
|
|
const blockedState = await page.evaluate(async () => {
|
|
return await window.hermesAPI.validateChatReadiness();
|
|
});
|
|
console.log("[B] missing-key validateChatReadiness:", JSON.stringify(blockedState));
|
|
|
|
// --- C. Config-health audit: should report MODEL_KEY_MISSING + maybe more ---
|
|
const health = await page.evaluate(async () => {
|
|
return await window.hermesAPI.getConfigHealth();
|
|
});
|
|
console.log("[C] config-health summary:", JSON.stringify(health.summary));
|
|
console.log("[C] issue codes:", health.issues.map((i) => `${i.severity}:${i.code}`).join(", "));
|
|
|
|
// --- Restore .env + auth.json ---
|
|
fs.copyFileSync(ENV_BACKUP, ENV_FILE);
|
|
fs.unlinkSync(ENV_BACKUP);
|
|
if (fs.existsSync(AUTH_BACKUP)) {
|
|
fs.copyFileSync(AUTH_BACKUP, AUTH);
|
|
fs.unlinkSync(AUTH_BACKUP);
|
|
}
|
|
// Clean up the marker we set
|
|
await page.evaluate(async () => {
|
|
await window.hermesAPI.setEnv("__VALIDATION_PROBE__", "");
|
|
});
|
|
console.log("[teardown] .env + auth.json restored");
|
|
|
|
// --- Verdicts ---
|
|
console.log();
|
|
const aOk = okState.ok === true;
|
|
const bBlocked = blockedState.ok === false &&
|
|
blockedState.code === "MISSING_API_KEY" &&
|
|
blockedState.expectedEnvKey === "DEEPSEEK_API_KEY";
|
|
const cFlagged = (health.issues || []).some(
|
|
(i) => i.code === "MODEL_KEY_MISSING" && i.context?.expectedKey === "DEEPSEEK_API_KEY",
|
|
);
|
|
console.log(`[VERDICT A] ${aOk ? "✅" : "🔴"} happy path → ok:true`);
|
|
console.log(`[VERDICT B] ${bBlocked ? "✅" : "🔴"} missing-key → blocked with MISSING_API_KEY + DEEPSEEK_API_KEY`);
|
|
console.log(`[VERDICT C] ${cFlagged ? "✅" : "🔴"} config-health reports MODEL_KEY_MISSING for DEEPSEEK_API_KEY`);
|
|
|
|
await browser.close();
|
|
})().catch((e) => {
|
|
// Best-effort restore on failure
|
|
try {
|
|
if (fs.existsSync(ENV_BACKUP)) {
|
|
fs.copyFileSync(ENV_BACKUP, ENV_FILE);
|
|
fs.unlinkSync(ENV_BACKUP);
|
|
}
|
|
} catch {}
|
|
console.error("FAILED:", e.stack || e.message || e);
|
|
process.exit(1);
|
|
});
|