elizaos--eliza/plugins/plugin-computeruse/test/computeruse-cross-platform.e2e.test.ts

/**
 * Cross-platform e2e smoke test for plugin-computeruse.
 *
 * Boots the ComputerUseService against a minimal mock runtime, registers the
 * service, and drives a short action sequence:
 *   screenshot → mouse_move(coords) → type("hello") → screenshot
 *
 * Asserts on observable outcomes only (success flags + non-empty screenshot
 * buffers). No vi.mock, no monkey-patching of platform code — the actual
 * cross-platform driver runs.
 *
 * Skip rules:
 *   - Linux: skipped unless DISPLAY is set (CI must use Xvfb / xvfb-run)
 *   - Windows: runs when the selected driver and desktop session are available
 *   - macOS: runs normally (Accessibility/Screen Recording permissions are a
 *     prerequisite; if denied the assertion-level check is converted to skip).
 *
 * Driver:
 *   - Default uses ELIZA_COMPUTERUSE_DRIVER=nutjs (cross-platform native).
 *   - When the @nut-tree-fork/nut-js native module is unavailable (deps not
 *     installed yet, or unsupported arch), the test gracefully skips.
 */

import { platform } from "node:os";
import type { IAgentRuntime } from "@elizaos/core";
import { describe, expect, it } from "vitest";
import {
  loadFailureReason,
  isAvailable as nutAvailable,
} from "../src/platform/nut-driver.js";
import { ComputerUseService } from "../src/services/computer-use-service.js";
import type { ComputerActionResult } from "../src/types.js";
import { assertScreenshotBase64NotBlank } from "./helpers/screenshot-quality.ts";

const os = platform();
const requestedDriver = (
  process.env.ELIZA_COMPUTERUSE_DRIVER ?? "nutjs"
).toLowerCase();

function shouldSkip(): { skip: boolean; reason: string } {
  if (os === "win32") {
    if (requestedDriver !== "nutjs") {
      return {
        skip: true,
        reason:
          "Windows cross-platform smoke uses the nutjs driver; legacy PowerShell coverage is validated by platform capability tests.",
      };
    }
  }
  if (os === "linux" && !process.env.DISPLAY) {
    return {
      skip: true,
      reason: "Linux requires DISPLAY (run under Xvfb / xvfb-run in CI)",
    };
  }
  if (requestedDriver === "nutjs" && !nutAvailable()) {
    return {
      skip: true,
      reason: `nutjs native module unavailable: ${loadFailureReason() ?? "unknown"}`,
    };
  }
  return { skip: false, reason: "" };
}

function createMockRuntime(): IAgentRuntime {
  const settings: Record<string, string> = {
    COMPUTER_USE_APPROVAL_MODE: "full_control",
    COMPUTER_USE_SCREENSHOT_AFTER_ACTION: "true",
  };
  return {
    character: {},
    getSetting(key: string) {
      return settings[key];
    },
    getService() {
      return null;
    },
  } as IAgentRuntime;
}

describe("plugin-computeruse cross-platform driver e2e", () => {
  const { skip, reason } = shouldSkip();

  it.skipIf(skip)(
    "drives screenshot → mouse_move → type → screenshot via the selected driver",
    async ({ skip }) => {
      const service = (await ComputerUseService.start(
        createMockRuntime(),
      )) as ComputerUseService;
      try {
        const screenshotBefore = (await service.executeCommand(
          "screenshot",
        )) as ComputerActionResult;
        if (screenshotBefore.permissionDenied) {
          // macOS denies Screen Recording until granted in System Settings.
          // CI without that permission cannot prove driver wiring; treat as
          // an environment skip rather than a fail.
          skip(
            `Desktop screenshot permission denied: ${
              screenshotBefore.error ?? "unknown"
            }`,
          );
        }
        if (!screenshotBefore.success) {
          skip(
            `Desktop screenshot unavailable in this environment: ${
              screenshotBefore.error ?? "unknown"
            }`,
          );
        }
        expect(screenshotBefore.success).toBe(true);
        assertScreenshotBase64NotBlank(
          screenshotBefore.screenshot,
          "screenshot before computer-use actions",
        );

        const move = (await service.executeCommand("mouse_move", {
          coordinate: [100, 100],
        })) as ComputerActionResult;
        if (move.permissionDenied) return;
        expect(move.success).toBe(true);

        const typeResult = (await service.executeCommand("type", {
          text: "hello",
        })) as ComputerActionResult;
        if (typeResult.permissionDenied) return;
        expect(typeResult.success).toBe(true);

        const screenshotAfter = (await service.executeCommand(
          "screenshot",
        )) as ComputerActionResult;
        if (screenshotAfter.permissionDenied) return;
        expect(screenshotAfter.success).toBe(true);
        assertScreenshotBase64NotBlank(
          screenshotAfter.screenshot,
          "screenshot after computer-use actions",
        );
      } finally {
        await service.stop();
      }
    },
    60_000,
  );

  if (skip) {
    it("skip-reason recorded for environment", () => {
      expect(reason.length).toBeGreaterThan(0);
    });
  }
});