elizaos--eliza/plugins/plugin-openrouter/__tests__/models.live.test.ts

/**
 * Live integration test (real OpenRouter API, gated on `OPENROUTER_API_KEY`):
 * boots a real AgentRuntime with `@elizaos/plugin-sql` and exercises TEXT_SMALL,
 * TEXT_LARGE, structured output via responseSchema, IMAGE_DESCRIPTION, and
 * TEXT_EMBEDDING through `runtime.useModel`.
 */
import type { IAgentRuntime } from "@elizaos/core";
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { openrouterPlugin } from "../index";

const hasApiKey = Boolean(process.env.OPENROUTER_API_KEY);

async function createTestRuntime(settings: Record<string, string> = {}): Promise<{
  runtime: IAgentRuntime;
  cleanup: () => Promise<void>;
}> {
  if (!settings.OPENROUTER_API_KEY && !process.env.OPENROUTER_API_KEY) {
    throw new Error("OPENROUTER_API_KEY is required to create the OpenRouter test runtime");
  }

  const {
    createDatabaseAdapter,
    DatabaseMigrationService,
    plugin: sqlPluginInstance,
  } = await import("@elizaos/plugin-sql");
  const { AgentRuntime, createCharacter } = await import("@elizaos/core");
  const { v4: uuidv4 } = await import("uuid");

  const agentId = uuidv4() as `${string}-${string}-${string}-${string}-${string}`;

  // Create the adapter using the exported function with in-memory database
  const adapter = createDatabaseAdapter({ dataDir: "memory://" }, agentId);
  await adapter.init();

  // Run migrations to create the schema
  const migrationService = new DatabaseMigrationService();
  const db = (adapter as { getDatabase(): () => unknown }).getDatabase();
  await migrationService.initializeWithDatabase(db);
  migrationService.discoverAndRegisterPluginSchemas([sqlPluginInstance]);
  await migrationService.runAllPluginMigrations();

  const character = createCharacter({
    name: "Test Assistant",
    bio: ["A test assistant for testing purposes"],
    system: "You are a helpful assistant.",
    plugins: [],
    settings: {},
    secrets: {
      ...settings,
      OPENROUTER_API_KEY: settings.OPENROUTER_API_KEY || process.env.OPENROUTER_API_KEY,
    },
    messageExamples: [],
    postExamples: [],
    topics: ["testing"],
    adjectives: ["helpful"],
    style: { all: [], chat: [], post: [] },
  });

  // Create the agent in the database first
  await adapter.createAgent({
    id: agentId,
    ...character,
    createdAt: Date.now(),
    updatedAt: Date.now(),
  });

  const runtime = new AgentRuntime({
    agentId,
    character,
    adapter,
    plugins: [],
  });

  await runtime.initialize();

  const cleanup = async () => {
    try {
      await runtime.stop();
      await adapter.close();
    } catch {
      // Ignore cleanup errors
    }
  };

  return { runtime, cleanup };
}

describe.skipIf(!hasApiKey)("OpenRouter Plugin", () => {
  let runtime: IAgentRuntime;
  let cleanup: () => Promise<void>;

  beforeAll(async () => {
    if (!hasApiKey) {
      return;
    }

    const result = await createTestRuntime();
    runtime = result.runtime;
    cleanup = result.cleanup;

    // Initialize plugin
    if (openrouterPlugin.init) {
      await openrouterPlugin.init({}, runtime);
    }
  });

  afterAll(async () => {
    if (cleanup) {
      await cleanup();
    }
  });

  describe("TEXT_SMALL Model", () => {
    test("should generate text with TEXT_SMALL model", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const prompt = "Hello, how are you today?";

      const textHandler = openrouterPlugin.models?.TEXT_SMALL;
      if (!textHandler) throw new Error("TEXT_SMALL model handler is not registered");
      const response = await textHandler(runtime, { prompt });

      expect(response).toBeDefined();
      expect(typeof response).toBe("string");
      expect(response.length).toBeGreaterThan(0);
    }, 30000);
  });

  describe("TEXT_LARGE Model", () => {
    test("should generate text with TEXT_LARGE model", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const prompt = "Explain quantum computing in simple terms.";

      const textHandler = openrouterPlugin.models?.TEXT_LARGE;
      if (!textHandler) throw new Error("TEXT_LARGE model handler is not registered");
      const response = await textHandler(runtime, { prompt });

      expect(response).toBeDefined();
      expect(typeof response).toBe("string");
      expect(response.length).toBeGreaterThan(0);
    }, 30000);
  });

  describe("Structured output via TEXT_* (native tool calling)", () => {
    test("should generate JSON via TEXT_SMALL with responseSchema", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const handler = openrouterPlugin.models?.TEXT_SMALL;
      if (!handler) throw new Error("TEXT_SMALL model handler is not registered");

      const response = await handler(runtime, {
        prompt: "Create a JSON object representing a person with name, age, and hobbies.",
        responseSchema: {
          type: "object",
          properties: {
            name: { type: "string" },
            age: { type: "number" },
            hobbies: { type: "array", items: { type: "string" } },
          },
          required: ["name", "age", "hobbies"],
        },
      } as Parameters<typeof handler>[1]);

      expect(response).toEqual(
        expect.objectContaining({
          text: expect.any(String),
        })
      );
    }, 30000);

    test("should generate JSON via TEXT_LARGE with responseSchema", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const handler = openrouterPlugin.models?.TEXT_LARGE;
      if (!handler) throw new Error("TEXT_LARGE model handler is not registered");

      const response = await handler(runtime, {
        prompt: "Create a detailed JSON object representing a complex product catalog.",
        responseSchema: { type: "object" },
      } as Parameters<typeof handler>[1]);

      expect(response).toEqual(
        expect.objectContaining({
          text: expect.any(String),
        })
      );
    }, 500000);
  });

  describe("IMAGE_DESCRIPTION Model", () => {
    test("should describe an image with IMAGE_DESCRIPTION model", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      // Use a public domain test image
      const imageUrl =
        "https://upload.wikimedia.org/wikipedia/commons/thumb/9/9a/Gull_portrait_ca_usa.jpg/1280px-Gull_portrait_ca_usa.jpg";

      const imageDescHandler = openrouterPlugin.models?.IMAGE_DESCRIPTION;
      if (!imageDescHandler) throw new Error("IMAGE_DESCRIPTION model handler is not registered");
      const response = await imageDescHandler(runtime, imageUrl);

      expect(response).toBeDefined();
      expect(response).toHaveProperty("title");
      expect(response).toHaveProperty("description");
      expect(typeof response.title).toBe("string");
      expect(typeof response.description).toBe("string");
      expect(response.title.length).toBeGreaterThan(0);
      expect(response.description.length).toBeGreaterThan(0);
    }, 500000);

    test("should describe an image with custom prompt", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const imageUrl =
        "https://upload.wikimedia.org/wikipedia/commons/thumb/9/9a/Gull_portrait_ca_usa.jpg/1280px-Gull_portrait_ca_usa.jpg";
      const customPrompt =
        "Identify the species of bird in this image and provide detailed characteristics.";

      const imageDescHandler = openrouterPlugin.models?.IMAGE_DESCRIPTION;
      if (!imageDescHandler) throw new Error("IMAGE_DESCRIPTION model handler is not registered");
      const response = await imageDescHandler(runtime, {
        imageUrl,
        prompt: customPrompt,
      });

      expect(response).toBeDefined();
      expect(response).toHaveProperty("title");
      expect(response).toHaveProperty("description");
      expect(typeof response.title).toBe("string");
      expect(typeof response.description).toBe("string");
      expect(response.title.length).toBeGreaterThan(0);
      expect(response.description.length).toBeGreaterThan(0);
    }, 500000);
  });

  describe("TEXT_EMBEDDING Model", () => {
    test("should generate embeddings with TEXT_EMBEDDING model", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const text = "Hello, this is a test for embeddings.";

      const embeddingHandler = openrouterPlugin.models?.TEXT_EMBEDDING;
      if (!embeddingHandler) throw new Error("TEXT_EMBEDDING model handler is not registered");
      const embedding = await embeddingHandler(runtime, { text });

      expect(embedding).toBeDefined();
      expect(Array.isArray(embedding)).toBe(true);
      expect(embedding.length).toBeGreaterThan(0);
      expect(typeof embedding[0]).toBe("number");
    }, 30000);

    test("should handle string input for embeddings", async () => {
      if (!hasApiKey) {
        console.warn("Skipping test: OPENROUTER_API_KEY not set");
        return;
      }

      const text = "Testing string input for embeddings.";

      const embeddingHandler = openrouterPlugin.models?.TEXT_EMBEDDING;
      if (!embeddingHandler) throw new Error("TEXT_EMBEDDING model handler is not registered");
      const embedding = await embeddingHandler(runtime, text);

      expect(embedding).toBeDefined();
      expect(Array.isArray(embedding)).toBe(true);
      expect(embedding.length).toBeGreaterThan(0);
      expect(typeof embedding[0]).toBe("number");
    }, 30000);
  });
});