kong--kong/spec/03-plugins/38-ai-proxy/00-config_spec.lua

local PLUGIN_NAME = "ai-proxy"


-- helper function to validate data against a schema
local validate do
  local validate_entity = require("spec.helpers").validate_plugin_config_schema
  local plugin_schema = require("kong.plugins."..PLUGIN_NAME..".schema")

  function validate(data)
    return validate_entity(data, plugin_schema)
  end
end


local SELF_HOSTED_MODELS = {
  "mistral",
  "llama2",
}


describe(PLUGIN_NAME .. ": (schema)", function()


  for i, v in ipairs(SELF_HOSTED_MODELS) do
    local op = it
    if v == "mistral" then -- mistral.ai now has managed service too!
      op = pending
    end
    op("requires upstream_url when using self-hosted " .. v .. " model", function()
      local config = {
        route_type = "llm/v1/chat",
        auth = {
          header_name = "Authorization",
          header_value = "Bearer token",
        },
        model = {
          name = "llama-2-7b-chat-hf",
          provider = v,
          options = {
            max_tokens = 256,
            temperature = 1.0,
          },
        },
      }

      if v == "llama2" then
        config.model.options.llama2_format = "raw"
      end

      if v == "mistral" then
        config.model.options.mistral_format = "ollama"
      end

      local ok, err = validate(config)

      assert.not_nil(err["config"]["@entity"])
      assert.not_nil(err["config"]["@entity"][1])
      assert.equal(err["config"]["@entity"][1], "must set 'model.options.upstream_url' for self-hosted providers/models")
      assert.is_falsy(ok)
    end)

    it("does not require API auth for self-hosted " .. v .. " model", function()
      local config = {
        route_type = "llm/v1/chat",
        model = {
          name = "llama-2-7b-chat-hf",
          provider = v,
          options = {
            max_tokens = 256,
            temperature = 1.0,
            upstream_url = "http://nowhere",
          },
        },
      }

      if v == "llama2" then
        config.model.options.llama2_format = "raw"
      end

      if v == "mistral" then
        config.model.options.mistral_format = "ollama"
      end

      local ok, err = validate(config)

      assert.is_truthy(ok)
      assert.is_falsy(err)
    end)
  end

  it("requires [anthropic_version] field when anthropic provider is used", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        header_name = "x-api-key",
        header_value = "anthropic_key",
      },
      model = {
        name = "anthropic-chat",
        provider = "anthropic",
        options = {
          max_tokens = 256,
          temperature = 1.0,
        },
      },
    }

    local ok, err = validate(config)

    assert.not_nil(err["config"]["@entity"])
    assert.not_nil(err["config"]["@entity"][1])
    assert.equal(err["config"]["@entity"][1], "must set 'model.options.anthropic_version' for anthropic provider")
    assert.is_falsy(ok)
  end)

  it("do not support log statistics when /chat route_type is used for anthropic provider", function()
    local config = {
      route_type = "llm/v1/completions",
      auth = {
        header_name = "x-api-key",
        header_value = "anthropic_key",
      },
      model = {
        name = "anthropic-chat",
        provider = "anthropic",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          anthropic_version = "2021-09-01",
        },
      },
      logging = {
        log_statistics = true,
      },
    }

    local ok, err = validate(config)
    assert.is_falsy(ok)
    assert.not_nil(err["config"]["@entity"])
    assert.not_nil(err["config"]["@entity"][1])
    assert.not_nil(err["config"]["@entity"][1], "anthropic does not support statistics when route_type is llm/v1/completions")
  end)

  it("requires [azure_instance] field when azure provider is used", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        header_name = "Authorization",
        header_value = "Bearer token",
      },
      model = {
        name = "azure-chat",
        provider = "azure",
        options = {
          max_tokens = 256,
          temperature = 1.0,
        },
      },
    }

    local ok, err = validate(config)

    assert.not_nil(err["config"]["@entity"])
    assert.not_nil(err["config"]["@entity"][1])
    assert.equal(err["config"]["@entity"][1], "must set 'model.options.azure_instance' for azure provider")
    assert.is_falsy(ok)
  end)

  it("requires both [config.auth.header_name] and [config.auth.header_value] to be set", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        header_name = "Authorization",
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.equals(err["config"]["@entity"][1], "all or none of these fields must be set: 'auth.header_name', 'auth.header_value'")
    assert.is_falsy(ok)
  end)

  it("requires both [config.auth.header_name] and [config.auth.header_value] to be set", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        header_name = "Authorization",
        header_value = "Bearer token",
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(err)
    assert.is_truthy(ok)
  end)

  it("requires all of [config.auth.param_name] and [config.auth.param_value] and [config.auth.param_location] to be set", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.equals(err["config"]["@entity"][1], "all or none of these fields must be set: 'auth.param_name', 'auth.param_value', 'auth.param_location'")
  end)

  it("requires all of [config.auth.param_name] and [config.auth.param_value] and [config.auth.param_location] to be set", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(err)
    assert.is_truthy(ok)
  end)

  it("requires all auth parameters set in order to use both header and param types", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token"
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(err)
    assert.is_truthy(ok)
  end)

  it("bedrock model can not support ath.allowed_auth_override", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = true,
      },
      model = {
        name = "bedrock",
        provider = "bedrock",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.is_truthy(err)
  end)

  it("gemini model can not support ath.allowed_auth_override", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = true,
      },
      model = {
        name = "gemini",
        provider = "gemini",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.is_truthy(err)
  end)

  it("native llm_format can only use llm/v1/chat route_type", function()
    local config = {
      route_type = "llm/v1/completions",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = false,
      },
      model = {
        name = "gemini",
        provider = "gemini",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      llm_format = "gemini",
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.is_truthy(err)
    assert.same("native provider options in llm_format can only be used with the 'llm/v1/chat' route_type", err["@entity"][1])
    assert.same("value must be llm/v1/chat", err["config"]["route_type"])

    config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = false,
      },
      model = {
        name = "gemini",
        provider = "gemini",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      llm_format = "gemini",
    }

    ok, err = validate(config)

    assert.is_truthy(ok)
    assert.is_falsy(err)
  end)

  it("bedrock llm_format can only use bedrock provider", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = false,
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      llm_format = "bedrock",
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.is_truthy(err)
    assert.same("native llm_format 'bedrock' can only be used with the 'bedrock' model.provider", err["@entity"][1])
    assert.same("value must be bedrock", err["config"]["model"]["provider"])

    config = {
      route_type = "llm/v1/chat",
      model = {
        name = "bedrock",
        provider = "bedrock",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      auth = {
        allow_override = false,
      },
      llm_format = "bedrock",
    }

    ok, err = validate(config)

    assert.is_truthy(ok)
    assert.is_falsy(err)
  end)

  it("gemini llm_format can only use gemini provider", function()
    local config = {
      route_type = "llm/v1/chat",
      auth = {
        param_name = "apikey",
        param_value = "key",
        param_location = "query",
        header_name = "Authorization",
        header_value = "Bearer token",
        allow_override = false,
      },
      model = {
        name = "openai",
        provider = "openai",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      llm_format = "gemini",
    }

    local ok, err = validate(config)

    assert.is_falsy(ok)
    assert.is_truthy(err)
    assert.same("native llm_format 'gemini' can only be used with the 'gemini' model.provider", err["@entity"][1])
    assert.same("value must be gemini", err["config"]["model"]["provider"])

    config = {
      route_type = "llm/v1/chat",
      model = {
        name = "gemini",
        provider = "gemini",
        options = {
          max_tokens = 256,
          temperature = 1.0,
          upstream_url = "http://nowhere",
        },
      },
      auth = {
        allow_override = false,
      },
      llm_format = "gemini",
    }

    ok, err = validate(config)

    assert.is_truthy(ok)
    assert.is_falsy(err)
  end)
end)