wehub-resource-sync
1368 lines
49 KiB
Swift
1368 lines
49 KiB
Swift
import Foundation
|
|
#if canImport(FoundationNetworking)
|
|
import FoundationNetworking
|
|
#endif
|
|
import Testing
|
|
@testable import CodexBarCore
|
|
|
|
struct ModelsDevPricingTests {
|
|
@Test
|
|
func `parses models dev subset`() throws {
|
|
let catalog = try Self.fixtureCatalog()
|
|
|
|
#expect(catalog.providers["openai"]?.name == "OpenAI")
|
|
#expect(catalog.providers["anthropic"]?.models["claude-sonnet-4-6"]?.cost?.cacheWrite == 3.75)
|
|
#expect(catalog.providers["anthropic"]?.models["claude-sonnet-4-6"]?.limit?.context == 1_000_000)
|
|
}
|
|
|
|
@Test
|
|
func `looks up pricing by provider and model`() throws {
|
|
let catalog = try Self.fixtureCatalog()
|
|
|
|
let openAI = try #require(catalog.pricing(providerID: "openai", modelID: "shared-model"))
|
|
let anthropic = try #require(catalog.pricing(providerID: "anthropic", modelID: "shared-model"))
|
|
|
|
#expect(openAI.pricing.inputCostPerToken == 1 / 1_000_000.0)
|
|
#expect(openAI.pricing.outputCostPerToken == 2 / 1_000_000.0)
|
|
#expect(anthropic.pricing.inputCostPerToken == 3 / 1_000_000.0)
|
|
#expect(anthropic.pricing.outputCostPerToken == 4 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `does not fall back across providers`() throws {
|
|
let catalog = try Self.fixtureCatalog()
|
|
|
|
#expect(catalog.pricing(providerID: "openai", modelID: "claude-sonnet-4-6") == nil)
|
|
#expect(catalog.pricing(providerID: "anthropic", modelID: "gpt-4o-mini") == nil)
|
|
}
|
|
|
|
@Test
|
|
func `supports provider scoped model normalization`() throws {
|
|
let catalog = try Self.fixtureCatalog()
|
|
|
|
let anthropic = try #require(catalog.pricing(
|
|
providerID: "anthropic",
|
|
modelID: "us.anthropic.claude-sonnet-4-6"))
|
|
let vertex = try #require(catalog.pricing(
|
|
providerID: "google-vertex-anthropic",
|
|
modelID: "claude-sonnet-4-6"))
|
|
|
|
#expect(anthropic.normalizedModelID == "claude-sonnet-4-6")
|
|
#expect(vertex.normalizedModelID == "claude-sonnet-4-6")
|
|
#expect(vertex.pricing.inputCostPerToken == 3.1 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `converts models dev per million token prices to per token prices`() throws {
|
|
let pricing = try #require(try Self.fixtureCatalog().pricing(
|
|
providerID: "anthropic",
|
|
modelID: "claude-sonnet-4-6")?
|
|
.pricing)
|
|
|
|
#expect(pricing.inputCostPerToken == 3 / 1_000_000.0)
|
|
#expect(pricing.outputCostPerToken == 15 / 1_000_000.0)
|
|
#expect(pricing.cacheReadInputCostPerToken == 0.3 / 1_000_000.0)
|
|
#expect(pricing.cacheCreationInputCostPerToken == 3.75 / 1_000_000.0)
|
|
#expect(pricing.thresholdTokens == 200_000)
|
|
#expect(pricing.inputCostPerTokenAboveThreshold == 6 / 1_000_000.0)
|
|
#expect(pricing.outputCostPerTokenAboveThreshold == 22.5 / 1_000_000.0)
|
|
#expect(pricing.cacheReadInputCostPerTokenAboveThreshold == 0.6 / 1_000_000.0)
|
|
#expect(pricing.cacheCreationInputCostPerTokenAboveThreshold == 7.5 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `stale cache is still readable`() throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
let load = ModelsDevCache.load(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root)
|
|
|
|
#expect(load.artifact != nil)
|
|
#expect(load.isStale)
|
|
#expect(load.error == nil)
|
|
}
|
|
|
|
@Test
|
|
func `pipeline lookup reads cached pricing`() throws {
|
|
let root = try Self.cacheRoot()
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: Date(), cacheRoot: root)
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 0.15 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `network failure preserves last valid cache`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(result: .failure(MockError.failed))))
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 0.15 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `refresh preserves cache when fetched catalog drops cached provider`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
let partialCatalog = Data("""
|
|
{
|
|
"openai": { "id": 7, "models": [] },
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((partialCatalog, Self.response(status: 200))))))
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 0.15 / 1_000_000.0)
|
|
}
|
|
}
|
|
|
|
extension ModelsDevPricingTests {
|
|
@Test
|
|
func `unknown model refresh makes newly published pricing available`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 10000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-901),
|
|
cacheRoot: root)
|
|
let refreshed = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } }
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
let transport = TrackingTransport(result: .success((refreshed, Self.response(status: 200))))
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["gpt-new"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
#expect(outcome == .pricingAvailable)
|
|
#expect(transport.calls == 1)
|
|
#expect(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-new",
|
|
cacheRoot: root) != nil)
|
|
}
|
|
|
|
@Test
|
|
func `unknown model refresh is bounded per provider cache`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 20000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-901),
|
|
cacheRoot: root)
|
|
let transport = try TrackingTransport(result: .success((
|
|
JSONEncoder().encode(Self.fixtureCatalog()),
|
|
Self.response(status: 200))))
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
let first = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
let second = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["another-unknown-model"],
|
|
now: now.addingTimeInterval(60),
|
|
cacheRoot: root,
|
|
client: client)
|
|
|
|
#expect(first == .unavailable)
|
|
#expect(second == .unavailable)
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `known requested model does not mask an unresolved unknown model`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 25000)
|
|
let catalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"already-priced": { "id": "already-priced", "cost": { "input": 1, "output": 2 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"catalog-anchor": { "id": "catalog-anchor", "cost": { "input": 3, "output": 4 } }
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(
|
|
catalog: catalog,
|
|
fetchedAt: now.addingTimeInterval(-901),
|
|
cacheRoot: root)
|
|
let transport = try TrackingTransport(result: .success((
|
|
JSONEncoder().encode(catalog),
|
|
Self.response(status: 200))))
|
|
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["already-priced", "still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: transport))
|
|
|
|
#expect(outcome == .unavailable)
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `pricing added by a completed background refresh requests a rescan`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 30000)
|
|
let refreshed = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": { "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } }
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": { "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } }
|
|
}
|
|
}
|
|
""".utf8)
|
|
let refreshedCatalog = try JSONDecoder().decode(ModelsDevCatalog.self, from: refreshed)
|
|
ModelsDevCache.save(catalog: refreshedCatalog, fetchedAt: now, cacheRoot: root)
|
|
let transport = TrackingTransport(result: .failure(MockError.failed))
|
|
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["gpt-new"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: transport))
|
|
|
|
#expect(outcome == .pricingAvailable)
|
|
#expect(transport.calls == 0)
|
|
}
|
|
|
|
@Test
|
|
func `ttl and unknown model refreshes share one download`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 40000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1),
|
|
cacheRoot: root)
|
|
let transport = try TrackingTransport(
|
|
result: .success((JSONEncoder().encode(Self.fixtureCatalog()), Self.response(status: 200))),
|
|
delayNanoseconds: 100_000_000)
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
async let ttl: Void = ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
async let unknown = ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
_ = await (ttl, unknown)
|
|
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `completed ttl refresh bounds a following unknown model refresh`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 45000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1),
|
|
cacheRoot: root)
|
|
let transport = try TrackingTransport(result: .success((
|
|
JSONEncoder().encode(Self.fixtureCatalog()),
|
|
Self.response(status: 200))))
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
|
|
#expect(outcome == .unavailable)
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `failed ttl refresh bounds a following unknown model refresh within cooldown`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 46000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1),
|
|
cacheRoot: root)
|
|
let transport = TrackingTransport(result: .failure(MockError.failed))
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
|
|
#expect(outcome == .unavailable)
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `failed unknown model refresh bounds a following ttl refresh within cooldown`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 47000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1),
|
|
cacheRoot: root)
|
|
let transport = TrackingTransport(result: .failure(MockError.failed))
|
|
let client = ModelsDevClient(transport: transport)
|
|
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["still-unknown"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: client)
|
|
|
|
#expect(outcome == .unavailable)
|
|
#expect(transport.calls == 1)
|
|
}
|
|
|
|
@Test
|
|
func `ttl refresh rechecks cache freshness after coordination`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let now = Date(timeIntervalSince1970: 48000)
|
|
try ModelsDevCache.save(
|
|
catalog: Self.fixtureCatalog(),
|
|
fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1),
|
|
cacheRoot: root)
|
|
#expect(ModelsDevCache.load(now: now, cacheRoot: root).isStale)
|
|
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: now, cacheRoot: root)
|
|
let transport = TrackingTransport(result: .failure(MockError.failed))
|
|
let cacheIsCurrent = await ModelsDevPricingPipeline.refreshStaleCache(
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: transport))
|
|
|
|
#expect(cacheIsCurrent)
|
|
#expect(transport.calls == 0)
|
|
}
|
|
|
|
@Test
|
|
func `failed cache save does not report pricing available`() async {
|
|
let root = URL(fileURLWithPath: "/dev/null", isDirectory: true)
|
|
let now = Date(timeIntervalSince1970: 50000)
|
|
let refreshed = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": { "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } }
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": { "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } }
|
|
}
|
|
}
|
|
""".utf8)
|
|
|
|
let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded(
|
|
providerID: "openai",
|
|
modelIDs: ["gpt-new"],
|
|
now: now,
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((refreshed, Self.response(status: 200))))))
|
|
|
|
#expect(outcome == .unavailable)
|
|
}
|
|
|
|
@Test
|
|
func `refresh accepts model churn and preserves removed pricing as fallback`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
let partialCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"provider-a-new": {
|
|
"id": "provider-a-new",
|
|
"cost": { "input": 7, "output": 8 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((partialCatalog, Self.response(status: 200))))))
|
|
|
|
let oldLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
let newLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "provider-a-new",
|
|
cacheRoot: root))
|
|
let updatedLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "shared-model",
|
|
cacheRoot: root))
|
|
|
|
#expect(oldLookup.pricing.inputCostPerToken == 0.15 / 1_000_000.0)
|
|
#expect(newLookup.pricing.inputCostPerToken == 7 / 1_000_000.0)
|
|
#expect(updatedLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `accumulated fallback models do not freeze later refreshes`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"provider-a-old": { "id": "provider-a-old", "cost": { "input": 1, "output": 2 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-old": { "id": "provider-b-old", "cost": { "input": 3, "output": 4 } }
|
|
}
|
|
},
|
|
"stale-a": {
|
|
"id": "stale-a",
|
|
"models": {
|
|
"model-a": { "id": "model-a", "cost": { "input": 5, "output": 6 } }
|
|
}
|
|
},
|
|
"stale-b": {
|
|
"id": "stale-b",
|
|
"models": {
|
|
"model-b": { "id": "model-b", "cost": { "input": 7, "output": 8 } }
|
|
}
|
|
},
|
|
"stale-c": {
|
|
"id": "stale-c",
|
|
"models": {
|
|
"model-c": { "id": "model-c", "cost": { "input": 9, "output": 10 } }
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let fetchedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"provider-a-new": { "id": "provider-a-new", "cost": { "input": 11, "output": 12 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-new": { "id": "provider-b-new", "cost": { "input": 13, "output": 14 } }
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((fetchedCatalog, Self.response(status: 200))))))
|
|
|
|
let newLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "provider-a-new",
|
|
cacheRoot: root))
|
|
let fallbackLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "stale-a",
|
|
modelID: "model-a",
|
|
cacheRoot: root))
|
|
|
|
#expect(newLookup.pricing.inputCostPerToken == 11 / 1_000_000.0)
|
|
#expect(fallbackLookup.pricing.inputCostPerToken == 5 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `historical fallback does not overwrite a refreshed model that reuses its map key`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"rolling": { "id": "provider-a-old", "cost": { "input": 1, "output": 2 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": { "id": "provider-b-anchor", "cost": { "input": 3, "output": 4 } }
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let fetchedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"rolling": { "id": "provider-a-new", "cost": { "input": 99, "output": 100 } }
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": { "id": "provider-b-anchor", "cost": { "input": 3, "output": 4 } }
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((fetchedCatalog, Self.response(status: 200))))))
|
|
|
|
let freshLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "provider-a-new",
|
|
cacheRoot: root))
|
|
let fallbackLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "provider-a-old",
|
|
cacheRoot: root))
|
|
|
|
#expect(freshLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
#expect(fallbackLookup.pricing.inputCostPerToken == 1 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `refresh updates cache when fetched catalog renames model key but keeps id`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
let renamedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"renamed-model-key": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"renamed-vertex-key": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((renamedCatalog, Self.response(status: 200))))))
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.normalizedModelID == "gpt-4o-mini")
|
|
#expect(lookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `refresh preserves cache when fetched matching model is not priceable`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: old, cacheRoot: root)
|
|
|
|
let partialCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-4o-mini": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((partialCatalog, Self.response(status: 200))))))
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
let updatedLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "shared-model",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 0.15 / 1_000_000.0)
|
|
#expect(updatedLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `refresh updates cache when fetched catalog canonicalizes alias model id`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-4o-mini": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 0.15, "output": 0.6 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"snapshot-model@20250101": {
|
|
"id": "snapshot-model@20250101",
|
|
"cost": { "input": 3.1, "output": 15.1 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let canonicalizedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-4o-mini": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"claude-sonnet-4-6": {
|
|
"id": "claude-sonnet-4-6",
|
|
"cost": { "input": 99, "output": 99 }
|
|
},
|
|
"shared-model": {
|
|
"id": "shared-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"snapshot-model-20250101": {
|
|
"id": "snapshot-model-20250101",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((canonicalizedCatalog, Self.response(status: 200))))))
|
|
|
|
let aliasLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "google-vertex-anthropic",
|
|
modelID: "snapshot-model@20250101",
|
|
cacheRoot: root))
|
|
let canonicalLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "google-vertex-anthropic",
|
|
modelID: "snapshot-model-20250101",
|
|
cacheRoot: root))
|
|
|
|
#expect(aliasLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
#expect(canonicalLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `fallback merge treats default alias as the canonical base model`() throws {
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"base-model@default": {
|
|
"id": "base-model@default",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
let refreshedCatalog = try Self.catalog("""
|
|
{
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"base-model": {
|
|
"id": "base-model",
|
|
"cost": { "input": 99, "output": 100 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
let merged = refreshedCatalog.mergingFallbackPricing(from: cachedCatalog)
|
|
let aliasLookup = try #require(merged.pricing(
|
|
providerID: "anthropic",
|
|
modelID: "base-model@default"))
|
|
|
|
#expect(aliasLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
#expect(merged.providers["anthropic"]?.models.count == 1)
|
|
}
|
|
|
|
@Test
|
|
func `fallback merge treats provider version alias as the canonical base model`() throws {
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"openai/base-model-v1:0": {
|
|
"id": "openai/base-model-v1:0",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
let refreshedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"base-model": {
|
|
"id": "base-model",
|
|
"cost": { "input": 99, "output": 100 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
let merged = refreshedCatalog.mergingFallbackPricing(from: cachedCatalog)
|
|
let aliasLookup = try #require(merged.pricing(
|
|
providerID: "openai",
|
|
modelID: "openai/base-model-v1:0"))
|
|
|
|
#expect(aliasLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
#expect(merged.providers["openai"]?.models.count == 1)
|
|
}
|
|
|
|
@Test
|
|
func `refresh keeps historical pinned pricing while accepting a new snapshot`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"snapshot-model@20250101": {
|
|
"id": "snapshot-model@20250101",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let fetchedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"provider-a-anchor": {
|
|
"id": "provider-a-anchor",
|
|
"cost": { "input": 1, "output": 2 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": {
|
|
"id": "provider-b-anchor",
|
|
"cost": { "input": 3, "output": 4 }
|
|
}
|
|
}
|
|
},
|
|
"google-vertex-anthropic": {
|
|
"id": "google-vertex-anthropic",
|
|
"models": {
|
|
"snapshot-model@20250201": {
|
|
"id": "snapshot-model@20250201",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((fetchedCatalog, Self.response(status: 200))))))
|
|
|
|
let oldLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "google-vertex-anthropic",
|
|
modelID: "snapshot-model@20250101",
|
|
cacheRoot: root))
|
|
let newLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "google-vertex-anthropic",
|
|
modelID: "snapshot-model@20250201",
|
|
cacheRoot: root))
|
|
|
|
#expect(oldLookup.pricing.inputCostPerToken == 3 / 1_000_000.0)
|
|
#expect(newLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `refresh preserves dated snapshot when fetched catalog only keeps base model`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"historical-map-key": {
|
|
"id": "snapshot-model-2025-01-01",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": {
|
|
"id": "provider-b-anchor",
|
|
"cost": { "input": 3, "output": 4 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let fetchedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"snapshot-model": {
|
|
"id": "snapshot-model",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": {
|
|
"id": "provider-b-anchor",
|
|
"cost": { "input": 3, "output": 4 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((fetchedCatalog, Self.response(status: 200))))))
|
|
|
|
let snapshotLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "snapshot-model-2025-01-01",
|
|
cacheRoot: root))
|
|
let baseLookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "snapshot-model",
|
|
cacheRoot: root))
|
|
|
|
#expect(snapshotLookup.pricing.inputCostPerToken == 3 / 1_000_000.0)
|
|
#expect(baseLookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `compact snapshot alias prefers snapshot pricing over base pricing`() throws {
|
|
let catalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"snapshot-model": {
|
|
"id": "snapshot-model",
|
|
"cost": { "input": 99, "output": 100 }
|
|
},
|
|
"snapshot-model-20250101": {
|
|
"id": "snapshot-model-20250101",
|
|
"cost": { "input": 3, "output": 15 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
let lookup = try #require(catalog.pricing(
|
|
providerID: "openai",
|
|
modelID: "snapshot-model@20250101"))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 3 / 1_000_000.0)
|
|
#expect(lookup.normalizedModelID == "snapshot-model-20250101")
|
|
}
|
|
|
|
@Test
|
|
func `refresh ignores unpriceable models in old cache continuity check`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
let old = Date(timeIntervalSince1970: 1)
|
|
let cachedCatalog = try Self.catalog("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-4o-mini": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 0.15, "output": 0.6 }
|
|
},
|
|
"unpriced-model": {
|
|
"id": "unpriced-model"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
ModelsDevCache.save(catalog: cachedCatalog, fetchedAt: old, cacheRoot: root)
|
|
|
|
let fetchedCatalog = Data("""
|
|
{
|
|
"openai": {
|
|
"id": "openai",
|
|
"models": {
|
|
"gpt-4o-mini": {
|
|
"id": "gpt-4o-mini",
|
|
"cost": { "input": 99, "output": 99 }
|
|
}
|
|
}
|
|
},
|
|
"anthropic": {
|
|
"id": "anthropic",
|
|
"models": {
|
|
"provider-b-anchor": {
|
|
"id": "provider-b-anchor",
|
|
"cost": { "input": 3, "output": 4 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""".utf8)
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(timeIntervalSince1970: 1 + ModelsDevCache.ttlSeconds + 1),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: MockTransport(
|
|
result: .success((fetchedCatalog, Self.response(status: 200))))))
|
|
|
|
let lookup = try #require(ModelsDevPricingPipeline.lookup(
|
|
providerID: "openai",
|
|
modelID: "gpt-4o-mini",
|
|
cacheRoot: root))
|
|
|
|
#expect(lookup.pricing.inputCostPerToken == 99 / 1_000_000.0)
|
|
}
|
|
|
|
@Test
|
|
func `fresh cache does not refresh`() async throws {
|
|
let root = try Self.cacheRoot()
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: Date(), cacheRoot: root)
|
|
let transport = TrackingTransport(result: .failure(MockError.failed))
|
|
|
|
await ModelsDevPricingPipeline.refreshIfNeeded(
|
|
now: Date(),
|
|
cacheRoot: root,
|
|
client: ModelsDevClient(transport: transport))
|
|
|
|
#expect(transport.calls == 0)
|
|
}
|
|
|
|
@Test
|
|
func `corrupt cache is ignored safely`() throws {
|
|
let root = try Self.cacheRoot()
|
|
let url = ModelsDevCache.cacheFileURL(cacheRoot: root)
|
|
try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)
|
|
try Data("not json".utf8).write(to: url)
|
|
|
|
let load = ModelsDevCache.load(cacheRoot: root)
|
|
|
|
#expect(load.artifact == nil)
|
|
#expect(load.isStale)
|
|
#expect(load.error == .invalidJSON)
|
|
}
|
|
|
|
@Test
|
|
func `serves decoded catalog from memo while the file is unchanged`() throws {
|
|
let root = try Self.cacheRoot()
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: Date(), cacheRoot: root)
|
|
let url = ModelsDevCache.cacheFileURL(cacheRoot: root)
|
|
|
|
// Pin a whole-second modification date so the memo key (which compares modification dates) round-trips
|
|
// deterministically through the filesystem.
|
|
let pinnedDate = Date(timeIntervalSince1970: 1_700_000_000)
|
|
try FileManager.default.setAttributes([.modificationDate: pinnedDate], ofItemAtPath: url.path)
|
|
|
|
// Prime the in-memory memo with a successful decode.
|
|
let primed = ModelsDevCache.load(cacheRoot: root)
|
|
let cachedArtifact = try #require(primed.artifact)
|
|
|
|
// Corrupt the file contents while preserving its size and modification date, so the on-disk identity
|
|
// the memo keys on is unchanged. A re-decode would now fail; a memo hit returns the cached artifact.
|
|
let size = try #require(
|
|
try (FileManager.default.attributesOfItem(atPath: url.path)[.size]) as? NSNumber).intValue
|
|
try Data(repeating: 0, count: size).write(to: url)
|
|
try FileManager.default.setAttributes([.modificationDate: pinnedDate], ofItemAtPath: url.path)
|
|
|
|
let reloaded = ModelsDevCache.load(cacheRoot: root)
|
|
|
|
#expect(reloaded.error == nil)
|
|
#expect(reloaded.artifact == cachedArtifact)
|
|
}
|
|
|
|
@Test
|
|
func `saving a new catalog invalidates the memo`() throws {
|
|
let root = try Self.cacheRoot()
|
|
try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: Date(), cacheRoot: root)
|
|
#expect(ModelsDevCache.load(cacheRoot: root).artifact?.catalog.providers["openai"] != nil)
|
|
|
|
// Overwriting the cache must drop the memo so the next load reflects the freshly written catalog.
|
|
ModelsDevCache.save(catalog: ModelsDevCatalog(providers: [:]), fetchedAt: Date(), cacheRoot: root)
|
|
let reloaded = ModelsDevCache.load(cacheRoot: root)
|
|
|
|
#expect(reloaded.error == nil)
|
|
#expect(reloaded.artifact?.catalog.providers.isEmpty == true)
|
|
}
|
|
|
|
@Test
|
|
func `serves a failed load from memo while the file is unchanged`() throws {
|
|
let root = try Self.cacheRoot()
|
|
let url = ModelsDevCache.cacheFileURL(cacheRoot: root)
|
|
try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)
|
|
let validData = try Self.encodedArtifactData()
|
|
|
|
// Write invalid JSON of the same size as a valid encoding, with a pinned modification date, then prime
|
|
// the memo with the resulting failure.
|
|
let pinnedDate = Date(timeIntervalSince1970: 1_700_000_000)
|
|
try Data(repeating: 0x7B, count: validData.count).write(to: url)
|
|
try FileManager.default.setAttributes([.modificationDate: pinnedDate], ofItemAtPath: url.path)
|
|
#expect(ModelsDevCache.load(cacheRoot: root).error == .invalidJSON)
|
|
|
|
// Replace the bytes with a valid encoding of identical size + modification date. A re-read would now
|
|
// succeed, so a returned failure proves the unchanged-identity file was not read and decoded again.
|
|
try validData.write(to: url)
|
|
try FileManager.default.setAttributes([.modificationDate: pinnedDate], ofItemAtPath: url.path)
|
|
let reloaded = ModelsDevCache.load(cacheRoot: root)
|
|
|
|
#expect(reloaded.error == .invalidJSON)
|
|
#expect(reloaded.artifact == nil)
|
|
}
|
|
|
|
@Test
|
|
func `client fetches with mock transport`() async throws {
|
|
let data = try Self.fixtureData()
|
|
let client = ModelsDevClient(transport: MockTransport(result: .success((data, Self.response(status: 200)))))
|
|
|
|
let catalog = try await client.fetchCatalog()
|
|
|
|
#expect(catalog.providers["google-vertex-anthropic"]?.models["claude-sonnet-4-6"]?.cost?.input == 3.1)
|
|
}
|
|
|
|
@Test
|
|
func `client reports http and json failures`() async throws {
|
|
let data = try Self.fixtureData()
|
|
let httpClient = ModelsDevClient(transport: MockTransport(result: .success((data, Self.response(status: 500)))))
|
|
let jsonClient = ModelsDevClient(transport: MockTransport(
|
|
result: .success((Data("not json".utf8), Self.response(status: 200)))))
|
|
|
|
await #expect(throws: ModelsDevClient.Error.httpStatus(500)) {
|
|
_ = try await httpClient.fetchCatalog()
|
|
}
|
|
await #expect(throws: ModelsDevClient.Error.invalidJSON) {
|
|
_ = try await jsonClient.fetchCatalog()
|
|
}
|
|
}
|
|
|
|
private static func fixtureData() throws -> Data {
|
|
let url = try #require(Bundle.module.url(
|
|
forResource: "models-dev-subset",
|
|
withExtension: "json",
|
|
subdirectory: "Fixtures"))
|
|
return try Data(contentsOf: url)
|
|
}
|
|
|
|
private static func fixtureCatalog() throws -> ModelsDevCatalog {
|
|
try JSONDecoder().decode(ModelsDevCatalog.self, from: self.fixtureData())
|
|
}
|
|
|
|
/// A valid `ModelsDevCacheArtifact` encoding, written the same way `ModelsDevCache.save` writes the file.
|
|
private static func encodedArtifactData() throws -> Data {
|
|
let artifact = try ModelsDevCacheArtifact(
|
|
version: ModelsDevCache.artifactVersion,
|
|
fetchedAt: Date(timeIntervalSince1970: 0),
|
|
catalog: self.fixtureCatalog())
|
|
let encoder = JSONEncoder()
|
|
encoder.dateEncodingStrategy = .iso8601
|
|
return try encoder.encode(artifact)
|
|
}
|
|
|
|
private static func catalog(_ json: String) throws -> ModelsDevCatalog {
|
|
try JSONDecoder().decode(ModelsDevCatalog.self, from: Data(json.utf8))
|
|
}
|
|
|
|
private static func cacheRoot() throws -> URL {
|
|
let root = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("codexbar-modelsdev-tests-\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
|
|
return root
|
|
}
|
|
|
|
private static func response(status: Int) -> HTTPURLResponse {
|
|
HTTPURLResponse(
|
|
url: URL(string: "https://models.dev/api.json")!,
|
|
statusCode: status,
|
|
httpVersion: nil,
|
|
headerFields: nil)!
|
|
}
|
|
}
|
|
|
|
private enum MockError: Error {
|
|
case failed
|
|
}
|
|
|
|
private struct MockTransport: ModelsDevHTTPTransport {
|
|
let result: Result<(Data, URLResponse), Error>
|
|
|
|
func data(for _: URLRequest) async throws -> (Data, URLResponse) {
|
|
try self.result.get()
|
|
}
|
|
}
|
|
|
|
private final class TrackingTransport: ModelsDevHTTPTransport, @unchecked Sendable {
|
|
private let lock = NSLock()
|
|
private var callCount = 0
|
|
let result: Result<(Data, URLResponse), Error>
|
|
let delayNanoseconds: UInt64
|
|
|
|
var calls: Int {
|
|
self.lock.withLock { self.callCount }
|
|
}
|
|
|
|
init(result: Result<(Data, URLResponse), Error>, delayNanoseconds: UInt64 = 0) {
|
|
self.result = result
|
|
self.delayNanoseconds = delayNanoseconds
|
|
}
|
|
|
|
func data(for _: URLRequest) async throws -> (Data, URLResponse) {
|
|
self.lock.withLock { self.callCount += 1 }
|
|
if self.delayNanoseconds > 0 {
|
|
try await Task.sleep(nanoseconds: self.delayNanoseconds)
|
|
}
|
|
return try self.result.get()
|
|
}
|
|
}
|