wehub-resource-sync
411 lines
18 KiB
Swift
411 lines
18 KiB
Swift
import Foundation
|
|
import Testing
|
|
@testable import CodexBarCore
|
|
|
|
@Suite(.serialized)
|
|
struct ClaudeBaselineCharacterizationTests {
|
|
private func makeStubClaudeCLI(loggedIn: Bool = true, invocationLog: URL? = nil) throws -> String {
|
|
let loggedInJSON = loggedIn ? "true" : "false"
|
|
return try self.makeStubClaudeCLI(
|
|
authStatusScript: "printf '%s\\n' '{\"loggedIn\":\(loggedInJSON)}'",
|
|
invocationLog: invocationLog)
|
|
}
|
|
|
|
private func makeStubClaudeCLI(authStatusScript: String, invocationLog: URL? = nil) throws -> String {
|
|
let sample = """
|
|
Current session
|
|
12% used (Resets 11am)
|
|
Current week (all models)
|
|
40% used (Resets Nov 21)
|
|
Current week (Sonnet only)
|
|
5% used (Resets Nov 21)
|
|
Account: user@example.com
|
|
Org: Example Org
|
|
"""
|
|
let recordInvocation = invocationLog.map { "printf '%s\\n' \"$*\" >> '\($0.path)'" } ?? ""
|
|
let script = """
|
|
#!/bin/sh
|
|
\(recordInvocation)
|
|
if [ "$1" = "auth" ] && [ "$2" = "status" ]; then
|
|
\(authStatusScript)
|
|
exit 0
|
|
fi
|
|
cat <<'EOF'
|
|
\(sample)
|
|
EOF
|
|
"""
|
|
let url = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("claude-stub-\(UUID().uuidString)")
|
|
try Data(script.utf8).write(to: url)
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: url.path)
|
|
return url.path
|
|
}
|
|
|
|
private func makeContext(
|
|
runtime: ProviderRuntime,
|
|
sourceMode: ProviderSourceMode,
|
|
env: [String: String] = [:],
|
|
settings: ProviderSettingsSnapshot? = nil) -> ProviderFetchContext
|
|
{
|
|
let browserDetection = BrowserDetection(cacheTTL: 0)
|
|
return ProviderFetchContext(
|
|
runtime: runtime,
|
|
sourceMode: sourceMode,
|
|
includeCredits: false,
|
|
webTimeout: 1,
|
|
webDebugDumpHTML: false,
|
|
verbose: false,
|
|
env: env,
|
|
settings: settings,
|
|
fetcher: UsageFetcher(environment: env),
|
|
claudeFetcher: ClaudeUsageFetcher(browserDetection: browserDetection),
|
|
browserDetection: browserDetection)
|
|
}
|
|
|
|
private func strategyIDs(
|
|
runtime: ProviderRuntime,
|
|
sourceMode: ProviderSourceMode,
|
|
env: [String: String] = [:],
|
|
settings: ProviderSettingsSnapshot? = nil) async -> [String]
|
|
{
|
|
let descriptor = ProviderDescriptorRegistry.descriptor(for: .claude)
|
|
let context = self.makeContext(runtime: runtime, sourceMode: sourceMode, env: env, settings: settings)
|
|
let strategies = await descriptor.fetchPlan.pipeline.resolveStrategies(context)
|
|
return strategies.map(\.id)
|
|
}
|
|
|
|
private func fetchOutcome(
|
|
runtime: ProviderRuntime,
|
|
sourceMode: ProviderSourceMode,
|
|
env: [String: String] = [:],
|
|
settings: ProviderSettingsSnapshot? = nil) async -> ProviderFetchOutcome
|
|
{
|
|
let descriptor = ProviderDescriptorRegistry.descriptor(for: .claude)
|
|
let context = self.makeContext(runtime: runtime, sourceMode: sourceMode, env: env, settings: settings)
|
|
return await descriptor.fetchPlan.fetchOutcome(context: context, provider: .claude)
|
|
}
|
|
|
|
private func withNoOAuthCredentials<T>(operation: () async throws -> T) async rethrows -> T {
|
|
let missingCredentialsURL = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("missing-claude-creds-\(UUID().uuidString).json")
|
|
return try await KeychainCacheStore.withServiceOverrideForTesting("rat-110-\(UUID().uuidString)") {
|
|
KeychainCacheStore.setTestStoreForTesting(true)
|
|
defer { KeychainCacheStore.setTestStoreForTesting(false) }
|
|
return try await ClaudeOAuthCredentialsStore.withIsolatedMemoryCacheForTesting {
|
|
try await ClaudeOAuthCredentialsStore.withIsolatedCredentialsFileTrackingForTesting {
|
|
try await ClaudeOAuthCredentialsStore.withCredentialsURLOverrideForTesting(missingCredentialsURL) {
|
|
try await ClaudeOAuthCredentialsStore.withKeychainAccessOverrideForTesting(true) {
|
|
try await ClaudeOAuthCredentialsStore.withClaudeKeychainOverridesForTesting(
|
|
data: nil,
|
|
fingerprint: nil)
|
|
{
|
|
try await operation()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
func `app auto pipeline order is OAuth then CLI then web`() async {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: true,
|
|
cookieSource: .manual,
|
|
manualCookieHeader: "sessionKey=sk-ant-session-token"))
|
|
let env = [
|
|
ClaudeOAuthCredentialsStore.environmentTokenKey: "oauth-token",
|
|
ClaudeOAuthCredentialsStore.environmentScopesKey: "user:profile",
|
|
"CLAUDE_CLI_PATH": "/usr/bin/true",
|
|
]
|
|
let strategyIDs = await self.strategyIDs(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
#expect(strategyIDs == ["claude.oauth", "claude.cli", "claude.web"])
|
|
}
|
|
|
|
@Test
|
|
func `CLI auto pipeline order is web then CLI`() async {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .manual,
|
|
manualCookieHeader: "sessionKey=sk-ant-session-token"))
|
|
let env = [
|
|
"CLAUDE_CLI_PATH": "/usr/bin/true",
|
|
]
|
|
let strategyIDs = await self.strategyIDs(runtime: .cli, sourceMode: .auto, env: env, settings: settings)
|
|
#expect(strategyIDs == ["claude.web", "claude.cli"])
|
|
}
|
|
|
|
@Test
|
|
func `explicit CLI pipeline attempts strategy even when planner marks CLI unavailable`() async {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .cli,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
let env = [
|
|
"CLAUDE_CLI_PATH": "/definitely/missing/claude",
|
|
]
|
|
let descriptor = ProviderDescriptorRegistry.descriptor(for: .claude)
|
|
let context = self.makeContext(runtime: .app, sourceMode: .cli, env: env, settings: settings)
|
|
let strategies = await descriptor.fetchPlan.pipeline.resolveStrategies(context)
|
|
|
|
#expect(strategies.map(\.id) == ["claude.cli"])
|
|
#expect(await strategies[0].isAvailable(context))
|
|
}
|
|
|
|
@Test
|
|
func `auto pipeline records unavailable planned steps when planner has no executable source`() async {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: true,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
let env = ["CLAUDE_CLI_PATH": "/definitely/missing/claude"]
|
|
|
|
await self.withNoOAuthCredentials {
|
|
await ClaudeCLIResolver.withResolvedBinaryPathOverrideForTesting("/definitely/missing/claude") {
|
|
let strategyIDs = await self.strategyIDs(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
#expect(strategyIDs == ["claude.oauth", "claude.cli", "claude.web"])
|
|
|
|
let outcome = await self.fetchOutcome(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
#expect(outcome.attempts.map(\.strategyID) == ["claude.oauth", "claude.cli", "claude.web"])
|
|
#expect(outcome.attempts.map(\.wasAvailable) == [false, false, false])
|
|
|
|
switch outcome.result {
|
|
case let .failure(error as ProviderFetchError):
|
|
switch error {
|
|
case let .noAvailableStrategy(provider):
|
|
#expect(provider == .claude)
|
|
}
|
|
case let .failure(error):
|
|
Issue.record("Unexpected failure: \(error)")
|
|
case let .success(result):
|
|
Issue.record("Unexpected success: \(result.sourceLabel)")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
func `app background auto does not start logged out Claude CLI`() async throws {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
let invocationLog = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("claude-invocations-\(UUID().uuidString).log")
|
|
let stubCLIPath = try self.makeStubClaudeCLI(loggedIn: false, invocationLog: invocationLog)
|
|
let env = ["CLAUDE_CLI_PATH": stubCLIPath]
|
|
|
|
await self.withNoOAuthCredentials {
|
|
let outcome = await self.fetchOutcome(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
|
|
#expect(outcome.attempts.map(\.strategyID) == ["claude.oauth", "claude.cli", "claude.web"])
|
|
#expect(outcome.attempts.map(\.wasAvailable) == [false, false, false])
|
|
}
|
|
|
|
let invocations = try String(contentsOf: invocationLog, encoding: .utf8)
|
|
#expect(invocations == "auth status --json\n")
|
|
}
|
|
|
|
@Test(arguments: ["nonzero", "timeout", "malformed"])
|
|
func `app background auto falls back to web when auth status is unusable`(
|
|
failureMode: String) async throws
|
|
{
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .manual,
|
|
manualCookieHeader: "sessionKey=sk-ant-session-token"))
|
|
let invocationLog = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("claude-invocations-\(UUID().uuidString).log")
|
|
let authStatusScript = switch failureMode {
|
|
case "nonzero":
|
|
"exit 9"
|
|
case "timeout":
|
|
"sleep 6"
|
|
default:
|
|
"printf '%s\\n' 'not-json'"
|
|
}
|
|
let stubCLIPath = try self.makeStubClaudeCLI(
|
|
authStatusScript: authStatusScript,
|
|
invocationLog: invocationLog)
|
|
let env = ["CLAUDE_CLI_PATH": stubCLIPath]
|
|
let usageLoader: ClaudeWebFetchStrategy.UsageLoader = { _ in
|
|
ClaudeUsageSnapshot(
|
|
primary: RateWindow(
|
|
usedPercent: 20,
|
|
windowMinutes: 300,
|
|
resetsAt: nil,
|
|
resetDescription: nil),
|
|
secondary: nil,
|
|
opus: nil,
|
|
updatedAt: Date(timeIntervalSince1970: 1_800_000_100),
|
|
accountEmail: nil,
|
|
accountOrganization: nil,
|
|
loginMethod: nil,
|
|
rawText: nil)
|
|
}
|
|
|
|
let outcome = await self.withNoOAuthCredentials {
|
|
await ClaudeWebFetchStrategy.$usageLoaderOverrideForTesting.withValue(usageLoader) {
|
|
await self.fetchOutcome(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
}
|
|
}
|
|
let result = try outcome.result.get()
|
|
|
|
#expect(outcome.attempts.map(\.strategyID) == ["claude.oauth", "claude.cli", "claude.web"])
|
|
#expect(outcome.attempts.map(\.wasAvailable) == [false, false, true])
|
|
#expect(result.strategyID == "claude.web")
|
|
let invocations = try String(contentsOf: invocationLog, encoding: .utf8)
|
|
#expect(invocations == "auth status --json\n")
|
|
}
|
|
|
|
@Test
|
|
func `app user initiated auto preserves CLI fallback without auth preflight`() async throws {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
let invocationLog = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("claude-invocations-\(UUID().uuidString).log")
|
|
let stubCLIPath = try self.makeStubClaudeCLI(loggedIn: false, invocationLog: invocationLog)
|
|
let env = ["CLAUDE_CLI_PATH": stubCLIPath]
|
|
let descriptor = ProviderDescriptorRegistry.descriptor(for: .claude)
|
|
let context = self.makeContext(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
let strategies = await descriptor.fetchPlan.pipeline.resolveStrategies(context)
|
|
let cli = try #require(strategies.first { $0.id == "claude.cli" })
|
|
|
|
let cliAvailable = await ProviderInteractionContext.$current.withValue(.userInitiated) {
|
|
await cli.isAvailable(context)
|
|
}
|
|
|
|
#expect(cliAvailable)
|
|
#expect(!FileManager.default.fileExists(atPath: invocationLog.path))
|
|
}
|
|
|
|
@Test
|
|
func `app auto pipeline retains OAuth bootstrap strategy at startup`() async {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
|
|
await ClaudeOAuthCredentialsStore.withIsolatedMemoryCacheForTesting {
|
|
ClaudeOAuthCredentialsStore.invalidateCache()
|
|
ClaudeOAuthCredentialsStore._resetCredentialsFileTrackingForTesting()
|
|
ClaudeOAuthKeychainAccessGate.resetForTesting()
|
|
defer {
|
|
ClaudeOAuthCredentialsStore.invalidateCache()
|
|
ClaudeOAuthCredentialsStore._resetCredentialsFileTrackingForTesting()
|
|
ClaudeOAuthKeychainAccessGate.resetForTesting()
|
|
}
|
|
|
|
await self.withNoOAuthCredentials {
|
|
let strategyIDs = await ClaudeOAuthKeychainPromptPreference.withTaskOverrideForTesting(
|
|
.onlyOnUserAction)
|
|
{
|
|
await ProviderRefreshContext.$current.withValue(.startup) {
|
|
await ProviderInteractionContext.$current.withValue(.background) {
|
|
await self.strategyIDs(runtime: .app, sourceMode: .auto, settings: settings)
|
|
}
|
|
}
|
|
}
|
|
#expect(strategyIDs.first == "claude.oauth")
|
|
#expect(strategyIDs.contains("claude.oauth"))
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
func `auto pipeline CLI uses planned environment for execution`() async throws {
|
|
let settings = ProviderSettingsSnapshot.make(claude: .init(
|
|
usageDataSource: .auto,
|
|
webExtrasEnabled: false,
|
|
cookieSource: .off,
|
|
manualCookieHeader: nil))
|
|
let stubCLIPath = try self.makeStubClaudeCLI()
|
|
let env = ["CLAUDE_CLI_PATH": stubCLIPath]
|
|
|
|
await self.withNoOAuthCredentials {
|
|
let fetchOverride: @Sendable (String, TimeInterval, Bool) async throws
|
|
-> ClaudeStatusSnapshot = { binary, _, _ in
|
|
#expect(binary == stubCLIPath)
|
|
return ClaudeStatusSnapshot(
|
|
sessionPercentLeft: 88,
|
|
weeklyPercentLeft: 60,
|
|
opusPercentLeft: 95,
|
|
accountEmail: "user@example.com",
|
|
accountOrganization: "Example Org",
|
|
loginMethod: nil,
|
|
primaryResetDescription: "Resets 11am",
|
|
secondaryResetDescription: "Resets Nov 21",
|
|
opusResetDescription: "Resets Nov 21",
|
|
rawText: "stub")
|
|
}
|
|
let outcome = await ClaudeStatusProbe.$fetchOverride.withValue(fetchOverride) {
|
|
await self.fetchOutcome(runtime: .app, sourceMode: .auto, env: env, settings: settings)
|
|
}
|
|
|
|
#expect(outcome.attempts.map(\.strategyID) == ["claude.oauth", "claude.cli"])
|
|
#expect(outcome.attempts.map(\.wasAvailable) == [false, true])
|
|
|
|
switch outcome.result {
|
|
case let .success(result):
|
|
#expect(result.strategyID == "claude.cli")
|
|
#expect(result.sourceLabel == "claude")
|
|
#expect(result.usage.primary?.usedPercent == 12)
|
|
#expect(result.usage.secondary?.usedPercent == 40)
|
|
#expect(result.usage.tertiary?.usedPercent == 5)
|
|
#expect(result.usage.identity?.accountEmail == "user@example.com")
|
|
case let .failure(error):
|
|
Issue.record("Unexpected failure: \(error)")
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test(arguments: [
|
|
(ProviderSourceMode.oauth, "claude.oauth"),
|
|
(ProviderSourceMode.cli, "claude.cli"),
|
|
(ProviderSourceMode.web, "claude.web"),
|
|
])
|
|
func `explicit modes resolve single Claude strategy`(
|
|
sourceMode: ProviderSourceMode,
|
|
expectedStrategyID: String) async
|
|
{
|
|
let strategyIDs = await self.strategyIDs(runtime: .app, sourceMode: sourceMode)
|
|
#expect(strategyIDs == [expectedStrategyID])
|
|
}
|
|
|
|
@Test(arguments: [
|
|
(ProviderSourceMode.oauth, "claude.oauth"),
|
|
(ProviderSourceMode.cli, "claude.cli"),
|
|
(ProviderSourceMode.web, "claude.web"),
|
|
])
|
|
func `CLI explicit modes resolve single Claude strategy`(
|
|
sourceMode: ProviderSourceMode,
|
|
expectedStrategyID: String) async
|
|
{
|
|
let strategyIDs = await self.strategyIDs(runtime: .cli, sourceMode: sourceMode)
|
|
#expect(strategyIDs == [expectedStrategyID])
|
|
}
|
|
|
|
@Test
|
|
func `Claude OAuth token heuristics accept raw and bearer inputs`() {
|
|
#expect(TokenAccountSupportCatalog.isClaudeOAuthToken("sk-ant-oat-test-token"))
|
|
#expect(TokenAccountSupportCatalog.isClaudeOAuthToken("Bearer sk-ant-oat-test-token"))
|
|
}
|
|
|
|
@Test
|
|
func `Claude OAuth token heuristics reject cookie shaped inputs`() {
|
|
#expect(!TokenAccountSupportCatalog.isClaudeOAuthToken("sessionKey=sk-ant-session"))
|
|
#expect(!TokenAccountSupportCatalog.isClaudeOAuthToken("Cookie: sessionKey=sk-ant-session; foo=bar"))
|
|
}
|
|
}
|