fathah--hermes-desktop/scripts/repro-nous-detection-367.js

/**
 * Live verification of the #367 Bug 4 fix in PR #369.
 *
 * Setup: switch the active model's provider to "nous" with no
 * NOUS_API_KEY in .env and no Nous evidence in auth.json. Then:
 *
 *   1. validateChatReadiness() should return ok:false with
 *      MISSING_API_KEY / NOUS_API_KEY.
 *   2. getConfigHealth() should report a MODEL_KEY_MISSING issue
 *      with context.provider="nous" and context.expectedKey="NOUS_API_KEY".
 *   3. After dropping a properly-shaped credential into auth.json,
 *      both checks should clear.
 *
 * Pre-fix (the gap this commit closes): all checks fail open and the
 * user gets no warning until the chat itself errors with "Hermes is
 * not logged into Nous Portal".
 */
const { attach } = require("./e2e-attach");
const fs = require("fs");
const path = require("path");
const os = require("os");

const HERMES_HOME = path.join(os.homedir(), "AppData", "Local", "hermes");
const CONFIG = path.join(HERMES_HOME, "config.yaml");
const ENV_FILE = path.join(HERMES_HOME, ".env");
const AUTH = path.join(HERMES_HOME, "auth.json");

const CONFIG_BAK = CONFIG + ".nous-test-bk";
const ENV_BAK = ENV_FILE + ".nous-test-bk";
const AUTH_BAK = AUTH + ".nous-test-bk";

function backup() {
  fs.copyFileSync(CONFIG, CONFIG_BAK);
  fs.copyFileSync(ENV_FILE, ENV_BAK);
  if (fs.existsSync(AUTH)) fs.copyFileSync(AUTH, AUTH_BAK);
}
function restore() {
  if (fs.existsSync(CONFIG_BAK)) {
    fs.copyFileSync(CONFIG_BAK, CONFIG);
    fs.unlinkSync(CONFIG_BAK);
  }
  if (fs.existsSync(ENV_BAK)) {
    fs.copyFileSync(ENV_BAK, ENV_FILE);
    fs.unlinkSync(ENV_BAK);
  }
  if (fs.existsSync(AUTH_BAK)) {
    fs.copyFileSync(AUTH_BAK, AUTH);
    fs.unlinkSync(AUTH_BAK);
  } else if (fs.existsSync(AUTH)) {
    fs.unlinkSync(AUTH);
  }
}

(async () => {
  backup();

  try {
    // Mutate config: provider=nous, no NOUS_API_KEY, no auth.json
    const cfg = fs.readFileSync(CONFIG, "utf-8");
    const cfgNew = cfg
      .replace(/^( *provider: ).*$/m, '$1"nous"')
      .replace(/^( *default: ).*$/m, '$1"hermes-4"');
    fs.writeFileSync(CONFIG, cfgNew);
    // Strip NOUS_API_KEY if present
    const env = fs.readFileSync(ENV_FILE, "utf-8");
    fs.writeFileSync(ENV_FILE, env.replace(/^NOUS_API_KEY=.*\n?/m, ""));
    // Remove nous entries from auth.json if any
    if (fs.existsSync(AUTH)) {
      const auth = JSON.parse(fs.readFileSync(AUTH, "utf-8"));
      if (auth.providers) delete auth.providers.nous;
      if (auth.credential_pool) delete auth.credential_pool.nous;
      fs.writeFileSync(AUTH, JSON.stringify(auth, null, 2));
    }

    const { browser, page } = await attach();

    // Bust the readEnv/getModelConfig caches by writing a dummy env entry
    await page.evaluate(async () => {
      await window.hermesAPI.setEnv("__NOUS_PROBE__", String(Date.now()));
      // Also trigger a model config reload — getModelConfig is cached
      // for 5s; invalidating it requires either a write or a wait.
    });
    // Trigger model config write to bust mc cache
    await page.evaluate(async () => {
      const mc = await window.hermesAPI.getModelConfig();
      await window.hermesAPI.setModelConfig(mc.provider, mc.model, mc.baseUrl);
    });

    // 1. validateChatReadiness
    const readiness = await page.evaluate(async () => {
      return await window.hermesAPI.validateChatReadiness();
    });
    console.log("[A] validateChatReadiness:", JSON.stringify(readiness));

    // 2. getConfigHealth
    const health = await page.evaluate(async () => {
      return await window.hermesAPI.getConfigHealth();
    });
    const modelKeyIssue = (health.issues || []).find(
      (i) => i.code === "MODEL_KEY_MISSING",
    );
    console.log(
      "[B] MODEL_KEY_MISSING issue:",
      JSON.stringify(modelKeyIssue || null),
    );

    // 3. Drop a properly-shaped credential pool entry → re-check
    const auth = fs.existsSync(AUTH)
      ? JSON.parse(fs.readFileSync(AUTH, "utf-8"))
      : { version: 1 };
    auth.credential_pool = auth.credential_pool || {};
    auth.credential_pool.nous = [
      {
        id: "test-nous-1",
        label: "Test Nous Key",
        auth_type: "api_key",
        access_token: "sk-nous-test-properly-shaped",
        base_url: "https://inference-api.nousresearch.com/v1",
        priority: 0,
      },
    ];
    fs.writeFileSync(AUTH, JSON.stringify(auth, null, 2));
    // Cache bust again
    await page.evaluate(async () => {
      await window.hermesAPI.setEnv("__NOUS_PROBE2__", String(Date.now()));
    });
    const readinessAfter = await page.evaluate(async () => {
      return await window.hermesAPI.validateChatReadiness();
    });
    const healthAfter = await page.evaluate(async () => {
      return await window.hermesAPI.rerunConfigHealth();
    });
    const modelKeyIssueAfter = (healthAfter.issues || []).find(
      (i) => i.code === "MODEL_KEY_MISSING",
    );
    console.log("[C] validateChatReadiness after adding pool entry:", JSON.stringify(readinessAfter));
    console.log(
      "[D] MODEL_KEY_MISSING issue after adding pool entry:",
      JSON.stringify(modelKeyIssueAfter || null),
    );

    await browser.close();

    console.log();
    const aBlocks = readiness.ok === false &&
      readiness.code === "MISSING_API_KEY" &&
      readiness.expectedEnvKey === "NOUS_API_KEY";
    const bFlags = modelKeyIssue &&
      modelKeyIssue.context?.expectedKey === "NOUS_API_KEY" &&
      modelKeyIssue.context?.provider === "nous";
    const cAllows = readinessAfter.ok === true;
    const dClears = !modelKeyIssueAfter;
    console.log(`[VERDICT A] ${aBlocks ? "✅" : "🔴"} validateChatReadiness blocks with MISSING_API_KEY / NOUS_API_KEY`);
    console.log(`[VERDICT B] ${bFlags ? "✅" : "🔴"} config-health flags MODEL_KEY_MISSING for nous`);
    console.log(`[VERDICT C] ${cAllows ? "✅" : "🔴"} validateChatReadiness clears after a properly-shaped pool entry`);
    console.log(`[VERDICT D] ${dClears ? "✅" : "🔴"} config-health MODEL_KEY_MISSING clears after pool entry`);
  } finally {
    restore();
    console.log("[teardown] config.yaml + .env + auth.json restored");
  }
})().catch((e) => {
  try { restore(); } catch {}
  console.error("FAILED:", e.stack || e.message || e);
  process.exit(1);
});