commit 03e739d01e3286f7b7de7d8d1f64e2220e9c2953 Author: wehub-resource-sync Date: Mon Jul 13 13:11:38 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..832e80a --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +worker/node_modules/ +worker/.dev.vars +.DS_Store +*.xcuserstate +build/ +releases/ +.claude/ +coding-plans/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6946d44 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,167 @@ +# Clicky - Agent Instructions + + + + +## Overview + +macOS menu bar companion app. Lives entirely in the macOS status bar (no dock icon, no main window). Clicking the menu bar icon opens a custom floating panel with companion voice controls. Uses push-to-talk (ctrl+option) to capture voice input, transcribes it via AssemblyAI streaming, and sends the transcript + a screenshot of the user's screen to Claude. Claude responds with text (streamed via SSE) and voice (ElevenLabs TTS). A blue cursor overlay can fly to and point at UI elements Claude references on any connected monitor. + +All API keys live on a Cloudflare Worker proxy — nothing sensitive ships in the app. + +## Architecture + +- **App Type**: Menu bar-only (`LSUIElement=true`), no dock icon or main window +- **Framework**: SwiftUI (macOS native) with AppKit bridging for menu bar panel and cursor overlay +- **Pattern**: MVVM with `@StateObject` / `@Published` state management +- **AI Chat**: Claude (Sonnet 4.6 default, Opus 4.6 optional) via Cloudflare Worker proxy with SSE streaming +- **Speech-to-Text**: AssemblyAI real-time streaming (`u3-rt-pro` model) via websocket, with OpenAI and Apple Speech as fallbacks +- **Text-to-Speech**: ElevenLabs (`eleven_flash_v2_5` model) via Cloudflare Worker proxy +- **Screen Capture**: ScreenCaptureKit (macOS 14.2+), multi-monitor support +- **Voice Input**: Push-to-talk via `AVAudioEngine` + pluggable transcription-provider layer. System-wide keyboard shortcut via listen-only CGEvent tap. +- **Element Pointing**: Claude embeds `[POINT:x,y:label:screenN]` tags in responses. The overlay parses these, maps coordinates to the correct monitor, and animates the blue cursor along a bezier arc to the target. +- **Concurrency**: `@MainActor` isolation, async/await throughout +- **Analytics**: PostHog via `ClickyAnalytics.swift` + +### API Proxy (Cloudflare Worker) + +The app never calls external APIs directly. All requests go through a Cloudflare Worker (`worker/src/index.ts`) that holds the real API keys as secrets. + +| Route | Upstream | Purpose | +|-------|----------|---------| +| `POST /chat` | `api.anthropic.com/v1/messages` | Claude vision + streaming chat | +| `POST /tts` | `api.elevenlabs.io/v1/text-to-speech/{voiceId}` | ElevenLabs TTS audio | +| `POST /transcribe-token` | `streaming.assemblyai.com/v3/token` | Fetches a short-lived (480s) AssemblyAI websocket token | + +Worker secrets: `ANTHROPIC_API_KEY`, `ASSEMBLYAI_API_KEY`, `ELEVENLABS_API_KEY` +Worker vars: `ELEVENLABS_VOICE_ID` + +### Key Architecture Decisions + +**Menu Bar Panel Pattern**: The companion panel uses `NSStatusItem` for the menu bar icon and a custom borderless `NSPanel` for the floating control panel. This gives full control over appearance (dark, rounded corners, custom shadow) and avoids the standard macOS menu/popover chrome. The panel is non-activating so it doesn't steal focus. A global event monitor auto-dismisses it on outside clicks. + +**Cursor Overlay**: A full-screen transparent `NSPanel` hosts the blue cursor companion. It's non-activating, joins all Spaces, and never steals focus. The cursor position, response text, waveform, and pointing animations all render in this overlay via SwiftUI through `NSHostingView`. + +**Global Push-To-Talk Shortcut**: Background push-to-talk uses a listen-only `CGEvent` tap instead of an AppKit global monitor so modifier-based shortcuts like `ctrl + option` are detected more reliably while the app is running in the background. + +**Shared URLSession for AssemblyAI**: A single long-lived `URLSession` is shared across all AssemblyAI streaming sessions (owned by the provider, not the session). Creating and invalidating a URLSession per session corrupts the OS connection pool and causes "Socket is not connected" errors after a few rapid reconnections. + +**Transient Cursor Mode**: When "Show Clicky" is off, pressing the hotkey fades in the cursor overlay for the duration of the interaction (recording → response → TTS → optional pointing), then fades it out automatically after 1 second of inactivity. + +## Key Files + +| File | Lines | Purpose | +|------|-------|---------| +| `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. | +| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | +| `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. | +| `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | +| `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | +| `CompanionResponseOverlay.swift` | ~217 | SwiftUI view for the response text bubble and waveform displayed next to the cursor in the overlay. | +| `CompanionScreenCaptureUtility.swift` | ~132 | Multi-monitor screenshot capture using ScreenCaptureKit. Returns labeled image data for each connected display. | +| `BuddyDictationManager.swift` | ~866 | Push-to-talk voice pipeline. Handles microphone capture via `AVAudioEngine`, provider-aware permission checks, keyboard/button dictation sessions, transcript finalization, shortcut parsing, contextual keyterms, and live audio-level reporting for waveform feedback. | +| `BuddyTranscriptionProvider.swift` | ~100 | Protocol surface and provider factory for voice transcription backends. Resolves provider based on `VoiceTranscriptionProvider` in Info.plist — AssemblyAI, OpenAI, or Apple Speech. | +| `AssemblyAIStreamingTranscriptionProvider.swift` | ~478 | Streaming transcription provider. Fetches temp tokens from the Cloudflare Worker, opens an AssemblyAI v3 websocket, streams PCM16 audio, tracks turn-based transcripts, and delivers finalized text on key-up. Shares a single URLSession across all sessions. | +| `OpenAIAudioTranscriptionProvider.swift` | ~317 | Upload-based transcription provider. Buffers push-to-talk audio locally, uploads as WAV on release, returns finalized transcript. | +| `AppleSpeechTranscriptionProvider.swift` | ~147 | Local fallback transcription provider backed by Apple's Speech framework. | +| `BuddyAudioConversionSupport.swift` | ~108 | Audio conversion helpers. Converts live mic buffers to PCM16 mono audio and builds WAV payloads for upload-based providers. | +| `GlobalPushToTalkShortcutMonitor.swift` | ~132 | System-wide push-to-talk monitor. Owns the listen-only `CGEvent` tap and publishes press/release transitions. | +| `ClaudeAPI.swift` | ~291 | Claude vision API client with streaming (SSE) and non-streaming modes. TLS warmup optimization, image MIME detection, conversation history support. | +| `OpenAIAPI.swift` | ~142 | OpenAI GPT vision API client. | +| `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | +| `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | +| `DesignSystem.swift` | ~880 | Design system tokens — colors, corner radii, shared styles. All UI references `DS.Colors`, `DS.CornerRadius`, etc. | +| `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. | +| `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | +| `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. | +| `worker/src/index.ts` | ~142 | Cloudflare Worker proxy. Three routes: `/chat` (Claude), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). | + +## Build & Run + +```bash +# Open in Xcode +open leanring-buddy.xcodeproj + +# Select the leanring-buddy scheme, set signing team, Cmd+R to build and run + +# Known non-blocking warnings: Swift 6 concurrency warnings, +# deprecated onChange warning in OverlayWindow.swift. Do NOT attempt to fix these. +``` + +**Do NOT run `xcodebuild` from the terminal** — it invalidates TCC (Transparency, Consent, and Control) permissions and the app will need to re-request screen recording, accessibility, etc. + +## Cloudflare Worker + +```bash +cd worker +npm install + +# Add secrets +npx wrangler secret put ANTHROPIC_API_KEY +npx wrangler secret put ASSEMBLYAI_API_KEY +npx wrangler secret put ELEVENLABS_API_KEY + +# Deploy +npx wrangler deploy + +# Local dev (create worker/.dev.vars with your keys) +npx wrangler dev +``` + +## Code Style & Conventions + +### Variable and Method Naming + +IMPORTANT: Follow these naming rules strictly. Clarity is the top priority. + +- Be as clear and specific with variable and method names as possible +- **Optimize for clarity over concision.** A developer with zero context on the codebase should immediately understand what a variable or method does just from reading its name +- Use longer names when it improves clarity. Do NOT use single-character variable names +- Example: use `originalQuestionLastAnsweredDate` instead of `originalAnswered` +- When passing props or arguments to functions, keep the same names as the original variable. Do not shorten or abbreviate parameter names. If you have `currentCardData`, pass it as `currentCardData`, not `card` or `cardData` + +### Code Clarity + +- **Clear is better than clever.** Do not write functionality in fewer lines if it makes the code harder to understand +- Write more lines of code if additional lines improve readability and comprehension +- Make things so clear that someone with zero context would completely understand the variable names, method names, what things do, and why they exist +- When a variable or method name alone cannot fully explain something, add a comment explaining what is happening and why + +### Swift/SwiftUI Conventions + +- Use SwiftUI for all UI unless a feature is only supported in AppKit (e.g., `NSPanel` for floating windows) +- All UI state updates must be on `@MainActor` +- Use async/await for all asynchronous operations +- Comments should explain "why" not just "what", especially for non-obvious AppKit bridging +- AppKit `NSPanel`/`NSWindow` bridged into SwiftUI via `NSHostingView` +- All buttons must show a pointer cursor on hover +- For any interactive element, explicitly think through its hover behavior (cursor, visual feedback, and whether hover should communicate clickability) + +### Do NOT + +- Do not add features, refactor code, or make "improvements" beyond what was asked +- Do not add docstrings, comments, or type annotations to code you did not change +- Do not try to fix the known non-blocking warnings (Swift 6 concurrency, deprecated onChange) +- Do not rename the project directory or scheme (the "leanring" typo is intentional/legacy) +- Do not run `xcodebuild` from the terminal — it invalidates TCC permissions + +## Git Workflow + +- Branch naming: `feature/description` or `fix/description` +- Commit messages: imperative mood, concise, explain the "why" not the "what" +- Do not force-push to main + +## Self-Update Instructions + + + +When you make changes to this project that affect the information in this file, update this file to reflect those changes. Specifically: + +1. **New files**: Add new source files to the "Key Files" table with their purpose and approximate line count +2. **Deleted files**: Remove entries for files that no longer exist +3. **Architecture changes**: Update the architecture section if you introduce new patterns, frameworks, or significant structural changes +4. **Build changes**: Update build commands if the build process changes +5. **New conventions**: If the user establishes a new coding convention during a session, add it to the appropriate conventions section +6. **Line count drift**: If a file's line count changes significantly (>50 lines), update the approximate count in the Key Files table + +Do NOT update this file for minor edits, bug fixes, or changes that don't affect the documented architecture or conventions. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..14b9c4b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Farza + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ea1fba --- /dev/null +++ b/README.md @@ -0,0 +1,162 @@ +Update: April 27, 2026. + +Hi there! I'm Farza, the guy that made Clicky. + +The existing codebase remains open source. Tinker with it, make it yours, start a company out of it, do whatever you want I don't mind. But, for all the new stuff I'm hacking on, gonna keep it private. To get the latest Clicky, you can go [here](https://www.heyclicky.com/). + +I also tweeted about this [here](https://x.com/FarzaTV/status/2043402737828962489). + +Go crazy with this repo!! It's an MIT license. + +# Hi, this is Clicky. +It's an AI teacher that lives as a buddy next to your cursor. It can see your screen, talk to you, and even point at stuff. Kinda like having a real teacher next to you. + +Download it [here](https://www.clicky.so/) for free. + +Here's the [original tweet](https://x.com/FarzaTV/status/2041314633978659092) that kinda blew up for a demo for more context. + +![Clicky — an ai buddy that lives on your mac](clicky-demo.gif) + +This is the open-source version of Clicky for those that want to hack on it, build their own features, or just see how it works under the hood. + +## Get started with Claude Code + +The fastest way to get this running is with [Claude Code](https://docs.anthropic.com/en/docs/claude-code). + +Once you get Claude running, paste this: + +``` +Hi Claude. + +Clone https://github.com/farzaa/clicky.git into my current directory. + +Then read the CLAUDE.md. I want to get Clicky running locally on my Mac. + +Help me set up everything — the Cloudflare Worker with my own API keys, the proxy URLs, and getting it building in Xcode. Walk me through it. +``` + +That's it. It'll clone the repo, read the docs, and walk you through the whole setup. Once you're running you can just keep talking to it — build features, fix bugs, whatever. Go crazy. + +## Manual setup + +If you want to do it yourself, here's the deal. + +### Prerequisites + +- macOS 14.2+ (for ScreenCaptureKit) +- Xcode 15+ +- Node.js 18+ (for the Cloudflare Worker) +- A [Cloudflare](https://cloudflare.com) account (free tier works) +- API keys for: [Anthropic](https://console.anthropic.com), [AssemblyAI](https://www.assemblyai.com), [ElevenLabs](https://elevenlabs.io) + +### 1. Set up the Cloudflare Worker + +The Worker is a tiny proxy that holds your API keys. The app talks to the Worker, the Worker talks to the APIs. This way your keys never ship in the app binary. + +```bash +cd worker +npm install +``` + +Now add your secrets. Wrangler will prompt you to paste each one: + +```bash +npx wrangler secret put ANTHROPIC_API_KEY +npx wrangler secret put ASSEMBLYAI_API_KEY +npx wrangler secret put ELEVENLABS_API_KEY +``` + +For the ElevenLabs voice ID, open `wrangler.toml` and set it there (it's not sensitive): + +```toml +[vars] +ELEVENLABS_VOICE_ID = "your-voice-id-here" +``` + +Deploy it: + +```bash +npx wrangler deploy +``` + +It'll give you a URL like `https://your-worker-name.your-subdomain.workers.dev`. Copy that. + +### 2. Run the Worker locally (for development) + +If you want to test changes to the Worker without deploying: + +```bash +cd worker +npx wrangler dev +``` + +This starts a local server (usually `http://localhost:8787`) that behaves exactly like the deployed Worker. You'll need to create a `.dev.vars` file in the `worker/` directory with your keys: + +``` +ANTHROPIC_API_KEY=sk-ant-... +ASSEMBLYAI_API_KEY=... +ELEVENLABS_API_KEY=... +ELEVENLABS_VOICE_ID=... +``` + +Then update the proxy URLs in the Swift code to point to `http://localhost:8787` instead of the deployed Worker URL while developing. Grep for `clicky-proxy` to find them all. + +### 3. Update the proxy URLs in the app + +The app has the Worker URL hardcoded in a few places. Search for `your-worker-name.your-subdomain.workers.dev` and replace it with your Worker URL: + +```bash +grep -r "clicky-proxy" leanring-buddy/ +``` + +You'll find it in: +- `CompanionManager.swift` — Claude chat + ElevenLabs TTS +- `AssemblyAIStreamingTranscriptionProvider.swift` — AssemblyAI token endpoint + +### 4. Open in Xcode and run + +```bash +open leanring-buddy.xcodeproj +``` + +In Xcode: +1. Select the `leanring-buddy` scheme (yes, the typo is intentional, long story) +2. Set your signing team under Signing & Capabilities +3. Hit **Cmd + R** to build and run + +The app will appear in your menu bar (not the dock). Click the icon to open the panel, grant the permissions it asks for, and you're good. + +### Permissions the app needs + +- **Microphone** — for push-to-talk voice capture +- **Accessibility** — for the global keyboard shortcut (Control + Option) +- **Screen Recording** — for taking screenshots when you use the hotkey +- **Screen Content** — for ScreenCaptureKit access + +## Architecture + +If you want the full technical breakdown, read `CLAUDE.md`. But here's the short version: + +**Menu bar app** (no dock icon) with two `NSPanel` windows — one for the control panel dropdown, one for the full-screen transparent cursor overlay. Push-to-talk streams audio over a websocket to AssemblyAI, sends the transcript + screenshot to Claude via streaming SSE, and plays the response through ElevenLabs TTS. Claude can embed `[POINT:x,y:label:screenN]` tags in its responses to make the cursor fly to specific UI elements across multiple monitors. All three APIs are proxied through a Cloudflare Worker. + +## Project structure + +``` +leanring-buddy/ # Swift source (yes, the typo stays) + CompanionManager.swift # Central state machine + CompanionPanelView.swift # Menu bar panel UI + ClaudeAPI.swift # Claude streaming client + ElevenLabsTTSClient.swift # Text-to-speech playback + OverlayWindow.swift # Blue cursor overlay + AssemblyAI*.swift # Real-time transcription + BuddyDictation*.swift # Push-to-talk pipeline +worker/ # Cloudflare Worker proxy + src/index.ts # Three routes: /chat, /tts, /transcribe-token +CLAUDE.md # Full architecture doc (agents read this) +``` + +## Contributing + +PRs welcome. If you're using Claude Code, it already knows the codebase — just tell it what you want to build and point it at `CLAUDE.md`. + +Got feedback? DM me on X [@farzatv](https://x.com/farzatv). diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..6bb3b9f --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`farzaa/clicky` +- 原始仓库：https://github.com/farzaa/clicky +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/appcast.xml b/appcast.xml new file mode 100644 index 0000000..6047142 --- /dev/null +++ b/appcast.xml @@ -0,0 +1,30 @@ + + + + makesomething + + 2.0 + Sat, 14 Mar 2026 09:56:02 -0700 + 11 + 2.0 + 14.0 + + + + 1.9 + Sat, 14 Mar 2026 08:33:19 -0700 + 10 + 1.9 + 14.0 + + + + 1.8 + Wed, 11 Mar 2026 10:17:12 -0700 + 9 + 1.8 + 14.0 + + + + \ No newline at end of file diff --git a/clicky-demo.gif b/clicky-demo.gif new file mode 100644 index 0000000..55fe6b7 Binary files /dev/null and b/clicky-demo.gif differ diff --git a/dmg-background.png b/dmg-background.png new file mode 100644 index 0000000..8a6af51 Binary files /dev/null and b/dmg-background.png differ diff --git a/leanring-buddy.xcodeproj/project.pbxproj b/leanring-buddy.xcodeproj/project.pbxproj new file mode 100644 index 0000000..75e5726 --- /dev/null +++ b/leanring-buddy.xcodeproj/project.pbxproj @@ -0,0 +1,635 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + AA00BB032F6500030039DA55 /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = AA00BB022F6500020039DA55 /* Sparkle */; }; + AA00BB062F6500060039DA55 /* PostHog in Frameworks */ = {isa = PBXBuildFile; productRef = AA00BB052F6500050039DA55 /* PostHog */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 28F22CCD2F56440300A0FC59 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 28F22CB72F56440300A0FC59 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 28F22CBE2F56440300A0FC59; + remoteInfo = "leanring-buddy"; + }; + 28F22CD72F56440300A0FC59 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 28F22CB72F56440300A0FC59 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 28F22CBE2F56440300A0FC59; + remoteInfo = "leanring-buddy"; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + 28F22CBF2F56440300A0FC59 /* Clicky.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Clicky.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 28F22CCC2F56440300A0FC59 /* leanring-buddyTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "leanring-buddyTests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; + 28F22CD62F56440300A0FC59 /* leanring-buddyUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "leanring-buddyUITests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + 28F22CC12F56440300A0FC59 /* leanring-buddy */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = "leanring-buddy"; + sourceTree = ""; + }; + 28F22CCF2F56440300A0FC59 /* leanring-buddyTests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = "leanring-buddyTests"; + sourceTree = ""; + }; + 28F22CD92F56440300A0FC59 /* leanring-buddyUITests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = "leanring-buddyUITests"; + sourceTree = ""; + }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + 28F22CBC2F56440300A0FC59 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + AA00BB032F6500030039DA55 /* Sparkle in Frameworks */, + AA00BB062F6500060039DA55 /* PostHog in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CC92F56440300A0FC59 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CD32F56440300A0FC59 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 28F22CB62F56440300A0FC59 = { + isa = PBXGroup; + children = ( + 28F22CC12F56440300A0FC59 /* leanring-buddy */, + 28F22CCF2F56440300A0FC59 /* leanring-buddyTests */, + 28F22CD92F56440300A0FC59 /* leanring-buddyUITests */, + 28F22CC02F56440300A0FC59 /* Products */, + ); + sourceTree = ""; + }; + 28F22CC02F56440300A0FC59 /* Products */ = { + isa = PBXGroup; + children = ( + 28F22CBF2F56440300A0FC59 /* Clicky.app */, + 28F22CCC2F56440300A0FC59 /* leanring-buddyTests.xctest */, + 28F22CD62F56440300A0FC59 /* leanring-buddyUITests.xctest */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 28F22CBE2F56440300A0FC59 /* leanring-buddy */ = { + isa = PBXNativeTarget; + buildConfigurationList = 28F22CE02F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddy" */; + buildPhases = ( + 28F22CBB2F56440300A0FC59 /* Sources */, + 28F22CBC2F56440300A0FC59 /* Frameworks */, + 28F22CBD2F56440300A0FC59 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 28F22CC12F56440300A0FC59 /* leanring-buddy */, + ); + name = "leanring-buddy"; + packageProductDependencies = ( + AA00BB022F6500020039DA55 /* Sparkle */, + AA00BB052F6500050039DA55 /* PostHog */, + ); + productName = "leanring-buddy"; + productReference = 28F22CBF2F56440300A0FC59 /* Clicky.app */; + productType = "com.apple.product-type.application"; + }; + 28F22CCB2F56440300A0FC59 /* leanring-buddyTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 28F22CE32F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddyTests" */; + buildPhases = ( + 28F22CC82F56440300A0FC59 /* Sources */, + 28F22CC92F56440300A0FC59 /* Frameworks */, + 28F22CCA2F56440300A0FC59 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 28F22CCE2F56440300A0FC59 /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + 28F22CCF2F56440300A0FC59 /* leanring-buddyTests */, + ); + name = "leanring-buddyTests"; + packageProductDependencies = ( + ); + productName = "leanring-buddyTests"; + productReference = 28F22CCC2F56440300A0FC59 /* leanring-buddyTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; + 28F22CD52F56440300A0FC59 /* leanring-buddyUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 28F22CE62F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddyUITests" */; + buildPhases = ( + 28F22CD22F56440300A0FC59 /* Sources */, + 28F22CD32F56440300A0FC59 /* Frameworks */, + 28F22CD42F56440300A0FC59 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 28F22CD82F56440300A0FC59 /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + 28F22CD92F56440300A0FC59 /* leanring-buddyUITests */, + ); + name = "leanring-buddyUITests"; + packageProductDependencies = ( + ); + productName = "leanring-buddyUITests"; + productReference = 28F22CD62F56440300A0FC59 /* leanring-buddyUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 28F22CB72F56440300A0FC59 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 2620; + LastUpgradeCheck = 2620; + TargetAttributes = { + 28F22CBE2F56440300A0FC59 = { + CreatedOnToolsVersion = 26.2; + }; + 28F22CCB2F56440300A0FC59 = { + CreatedOnToolsVersion = 26.2; + TestTargetID = 28F22CBE2F56440300A0FC59; + }; + 28F22CD52F56440300A0FC59 = { + CreatedOnToolsVersion = 26.2; + TestTargetID = 28F22CBE2F56440300A0FC59; + }; + }; + }; + buildConfigurationList = 28F22CBA2F56440300A0FC59 /* Build configuration list for PBXProject "leanring-buddy" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 28F22CB62F56440300A0FC59; + minimizedProjectReferenceProxies = 1; + packageReferences = ( + AA00BB012F6500010039DA55 /* XCRemoteSwiftPackageReference "Sparkle" */, + AA00BB042F6500040039DA55 /* XCRemoteSwiftPackageReference "posthog-ios" */, + ); + preferredProjectObjectVersion = 77; + productRefGroup = 28F22CC02F56440300A0FC59 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 28F22CBE2F56440300A0FC59 /* leanring-buddy */, + 28F22CCB2F56440300A0FC59 /* leanring-buddyTests */, + 28F22CD52F56440300A0FC59 /* leanring-buddyUITests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 28F22CBD2F56440300A0FC59 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CCA2F56440300A0FC59 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CD42F56440300A0FC59 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 28F22CBB2F56440300A0FC59 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CC82F56440300A0FC59 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 28F22CD22F56440300A0FC59 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 28F22CCE2F56440300A0FC59 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 28F22CBE2F56440300A0FC59 /* leanring-buddy */; + targetProxy = 28F22CCD2F56440300A0FC59 /* PBXContainerItemProxy */; + }; + 28F22CD82F56440300A0FC59 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 28F22CBE2F56440300A0FC59 /* leanring-buddy */; + targetProxy = 28F22CD72F56440300A0FC59 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 28F22CDE2F56440300A0FC59 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 14.2; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 28F22CDF2F56440300A0FC59 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 14.2; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + }; + name = Release; + }; + 28F22CE12F56440300A0FC59 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = "leanring-buddy/leanring-buddy.entitlements"; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 2UDAY4J48G; + ENABLE_APP_SANDBOX = NO; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; + ENABLE_PREVIEWS = YES; + ENABLE_USER_SELECTED_FILES = readonly; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = "leanring-buddy/Info.plist"; + INFOPLIST_KEY_CFBundleDisplayName = Clicky; + INFOPLIST_KEY_CFBundleName = Clicky; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSScreenCaptureUsageDescription = "Clicky needs screen recording access to see your screen and help you."; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_NAME = Clicky; + REGISTER_APP_GROUPS = YES; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 28F22CE22F56440300A0FC59 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = "leanring-buddy/leanring-buddy.entitlements"; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 2UDAY4J48G; + ENABLE_APP_SANDBOX = NO; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; + ENABLE_PREVIEWS = YES; + ENABLE_USER_SELECTED_FILES = readonly; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = "leanring-buddy/Info.plist"; + INFOPLIST_KEY_CFBundleDisplayName = Clicky; + INFOPLIST_KEY_CFBundleName = Clicky; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSScreenCaptureUsageDescription = "Clicky needs screen recording access to see your screen and help you."; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_NAME = Clicky; + REGISTER_APP_GROUPS = YES; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + 28F22CE42F56440300A0FC59 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + GENERATE_INFOPLIST_FILE = YES; + MACOSX_DEPLOYMENT_TARGET = 14.2; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddyTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + }; + name = Debug; + }; + 28F22CE52F56440300A0FC59 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + GENERATE_INFOPLIST_FILE = YES; + MACOSX_DEPLOYMENT_TARGET = 14.2; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddyTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + }; + name = Release; + }; + 28F22CE72F56440300A0FC59 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddyUITests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + TEST_TARGET_NAME = "leanring-buddy"; + }; + name = Debug; + }; + 28F22CE82F56440300A0FC59 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 6D7X9GGZAW; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddyUITests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + TEST_TARGET_NAME = "leanring-buddy"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 28F22CBA2F56440300A0FC59 /* Build configuration list for PBXProject "leanring-buddy" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 28F22CDE2F56440300A0FC59 /* Debug */, + 28F22CDF2F56440300A0FC59 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 28F22CE02F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddy" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 28F22CE12F56440300A0FC59 /* Debug */, + 28F22CE22F56440300A0FC59 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 28F22CE32F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddyTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 28F22CE42F56440300A0FC59 /* Debug */, + 28F22CE52F56440300A0FC59 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 28F22CE62F56440300A0FC59 /* Build configuration list for PBXNativeTarget "leanring-buddyUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 28F22CE72F56440300A0FC59 /* Debug */, + 28F22CE82F56440300A0FC59 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + AA00BB012F6500010039DA55 /* XCRemoteSwiftPackageReference "Sparkle" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/sparkle-project/Sparkle"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 2.9.0; + }; + }; + AA00BB042F6500040039DA55 /* XCRemoteSwiftPackageReference "posthog-ios" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/PostHog/posthog-ios.git"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 3.0.0; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + AA00BB022F6500020039DA55 /* Sparkle */ = { + isa = XCSwiftPackageProductDependency; + package = AA00BB012F6500010039DA55 /* XCRemoteSwiftPackageReference "Sparkle" */; + productName = Sparkle; + }; + AA00BB052F6500050039DA55 /* PostHog */ = { + isa = XCSwiftPackageProductDependency; + package = AA00BB042F6500040039DA55 /* XCRemoteSwiftPackageReference "posthog-ios" */; + productName = PostHog; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = 28F22CB72F56440300A0FC59 /* Project object */; +} diff --git a/leanring-buddy.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/leanring-buddy.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/leanring-buddy.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000..d88adb2 --- /dev/null +++ b/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,33 @@ +{ + "originHash" : "3c6fb67fefedcfcd00708e24ca8088151f21dccfc0ade32ea80c406646277e89", + "pins" : [ + { + "identity" : "plcrashreporter", + "kind" : "remoteSourceControl", + "location" : "https://github.com/microsoft/plcrashreporter.git", + "state" : { + "revision" : "0254f941c646b1ed17b243654723d0f071e990d0", + "version" : "1.12.2" + } + }, + { + "identity" : "posthog-ios", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PostHog/posthog-ios.git", + "state" : { + "revision" : "09da1be6a614325a6a464c6d2017a9ac858d1b5a", + "version" : "3.47.0" + } + }, + { + "identity" : "sparkle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/sparkle-project/Sparkle", + "state" : { + "revision" : "21d8df80440b1ca3b65fa82e40782f1e5a9e6ba2", + "version" : "2.9.0" + } + } + ], + "version" : 3 +} diff --git a/leanring-buddy.xcodeproj/xcuserdata/julianalvarez.xcuserdatad/xcschemes/xcschememanagement.plist b/leanring-buddy.xcodeproj/xcuserdata/julianalvarez.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..6c13490 --- /dev/null +++ b/leanring-buddy.xcodeproj/xcuserdata/julianalvarez.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + leanring-buddy.xcscheme_^#shared#^_ + + orderHint + 0 + + + + diff --git a/leanring-buddy.xcodeproj/xcuserdata/thorfinn.xcuserdatad/xcschemes/xcschememanagement.plist b/leanring-buddy.xcodeproj/xcuserdata/thorfinn.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..6c13490 --- /dev/null +++ b/leanring-buddy.xcodeproj/xcuserdata/thorfinn.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + leanring-buddy.xcscheme_^#shared#^_ + + orderHint + 0 + + + + diff --git a/leanring-buddy/AGENTS.md b/leanring-buddy/AGENTS.md new file mode 100644 index 0000000..b59cb09 --- /dev/null +++ b/leanring-buddy/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS.md - leanring-buddy (Main App Target) + +## Source Files + +### FloatingSessionButton.swift +- `FloatingSessionButtonManager` — `@MainActor` class managing the `NSPanel` lifecycle + - `showFloatingButton()` — Creates/shows the panel in top-right of primary screen + - `hideFloatingButton()` — Hides panel (keeps it alive for quick re-show) + - `destroyFloatingButton()` — Removes panel permanently (session ended) + - `onFloatingButtonClicked` — Callback closure, set by ContentView to bring main window to front + - `floatingButtonPanel` — Exposed `NSPanel` reference for screenshot exclusion +- `FloatingButtonView` — Private SwiftUI view with gradient circle, scale+glow hover animation, pointer cursor + +### ContentView.swift +- Receives `FloatingSessionButtonManager` via `@EnvironmentObject` +- `isMainWindowCurrentlyFocused` — Tracks main window focus state +- `configureFloatingButtonManager()` — Wires up the click callback +- `startObservingMainWindowFocusChanges()` — Sets up `NSWindow` notification observers +- `updateFloatingButtonVisibility()` — Core logic: show if running + not focused, hide otherwise +- `bringMainWindowToFront()` — Activates app and orders main window front + +### ScreenshotManager.swift +- `floatingButtonWindowToExcludeFromCaptures` — `NSWindow?` reference set by ContentView +- `captureScreen()` — Matches the floating window to an `SCWindow` and excludes it from capture filter + +### leanring_buddyApp.swift +- Owns `FloatingSessionButtonManager` as `@StateObject` +- Injects it into ContentView via `.environmentObject()` diff --git a/leanring-buddy/AppBundleConfiguration.swift b/leanring-buddy/AppBundleConfiguration.swift new file mode 100644 index 0000000..6fd2337 --- /dev/null +++ b/leanring-buddy/AppBundleConfiguration.swift @@ -0,0 +1,28 @@ +// +// AppBundleConfiguration.swift +// leanring-buddy +// +// Shared helper for reading runtime configuration from the built app bundle. +// + +import Foundation + +enum AppBundleConfiguration { + static func stringValue(forKey key: String) -> String? { + if let value = Bundle.main.object(forInfoDictionaryKey: key) as? String { + let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedValue.isEmpty { + return trimmedValue + } + } + + guard let resourceInfoPath = Bundle.main.path(forResource: "Info", ofType: "plist"), + let resourceInfo = NSDictionary(contentsOfFile: resourceInfoPath), + let value = resourceInfo[key] as? String else { + return nil + } + + let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmedValue.isEmpty ? nil : trimmedValue + } +} diff --git a/leanring-buddy/AppleSpeechTranscriptionProvider.swift b/leanring-buddy/AppleSpeechTranscriptionProvider.swift new file mode 100644 index 0000000..600594f --- /dev/null +++ b/leanring-buddy/AppleSpeechTranscriptionProvider.swift @@ -0,0 +1,147 @@ +// +// AppleSpeechTranscriptionProvider.swift +// leanring-buddy +// +// Local fallback transcription provider backed by Apple's Speech framework. +// + +import AVFoundation +import Foundation +import Speech + +struct AppleSpeechTranscriptionProviderError: LocalizedError { + let message: String + + var errorDescription: String? { + message + } +} + +final class AppleSpeechTranscriptionProvider: BuddyTranscriptionProvider { + let displayName = "Apple Speech" + let requiresSpeechRecognitionPermission = true + let isConfigured = true + let unavailableExplanation: String? = nil + + func startStreamingSession( + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) async throws -> any BuddyStreamingTranscriptionSession { + guard let speechRecognizer = Self.makeBestAvailableSpeechRecognizer() else { + throw AppleSpeechTranscriptionProviderError(message: "dictation is not available on this mac.") + } + + return try AppleSpeechTranscriptionSession( + speechRecognizer: speechRecognizer, + onTranscriptUpdate: onTranscriptUpdate, + onFinalTranscriptReady: onFinalTranscriptReady, + onError: onError + ) + } + + private static func makeBestAvailableSpeechRecognizer() -> SFSpeechRecognizer? { + let preferredLocales = [ + Locale.autoupdatingCurrent, + Locale(identifier: "en-US") + ] + + for preferredLocale in preferredLocales { + if let speechRecognizer = SFSpeechRecognizer(locale: preferredLocale) { + return speechRecognizer + } + } + + return SFSpeechRecognizer() + } +} + +private final class AppleSpeechTranscriptionSession: NSObject, BuddyStreamingTranscriptionSession { + let finalTranscriptFallbackDelaySeconds: TimeInterval = 1.8 + + private let recognitionRequest: SFSpeechAudioBufferRecognitionRequest + private var recognitionTask: SFSpeechRecognitionTask? + private let onTranscriptUpdate: (String) -> Void + private let onFinalTranscriptReady: (String) -> Void + private let onError: (Error) -> Void + + private var latestRecognizedText = "" + private var hasRequestedFinalTranscript = false + private var hasDeliveredFinalTranscript = false + + init( + speechRecognizer: SFSpeechRecognizer, + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) throws { + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.onTranscriptUpdate = onTranscriptUpdate + self.onFinalTranscriptReady = onFinalTranscriptReady + self.onError = onError + + super.init() + + recognitionRequest.shouldReportPartialResults = true + recognitionRequest.taskHint = .dictation + recognitionRequest.addsPunctuation = true + + if speechRecognizer.supportsOnDeviceRecognition { + recognitionRequest.requiresOnDeviceRecognition = true + } + + recognitionTask = speechRecognizer.recognitionTask(with: recognitionRequest) { [weak self] result, error in + self?.handleRecognitionEvent(result: result, error: error) + } + } + + func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { + guard !hasRequestedFinalTranscript else { return } + recognitionRequest.append(audioBuffer) + } + + func requestFinalTranscript() { + guard !hasRequestedFinalTranscript else { return } + hasRequestedFinalTranscript = true + recognitionRequest.endAudio() + } + + func cancel() { + recognitionTask?.cancel() + recognitionTask = nil + } + + private func handleRecognitionEvent( + result: SFSpeechRecognitionResult?, + error: Error? + ) { + if let result { + latestRecognizedText = result.bestTranscription.formattedString + onTranscriptUpdate(latestRecognizedText) + + if result.isFinal { + deliverFinalTranscriptIfNeeded(latestRecognizedText) + return + } + } + + guard let error else { return } + + if hasRequestedFinalTranscript && !latestRecognizedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + deliverFinalTranscriptIfNeeded(latestRecognizedText) + } else { + onError(error) + } + } + + private func deliverFinalTranscriptIfNeeded(_ transcriptText: String) { + guard !hasDeliveredFinalTranscript else { return } + hasDeliveredFinalTranscript = true + onFinalTranscriptReady(transcriptText) + } + + deinit { + cancel() + } +} diff --git a/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift new file mode 100644 index 0000000..d21286b --- /dev/null +++ b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift @@ -0,0 +1,478 @@ +// +// AssemblyAIStreamingTranscriptionProvider.swift +// leanring-buddy +// +// Streaming AI transcription provider backed by AssemblyAI's websocket API. +// + +import AVFoundation +import Foundation + +struct AssemblyAIStreamingTranscriptionProviderError: LocalizedError { + let message: String + + var errorDescription: String? { + message + } +} + +final class AssemblyAIStreamingTranscriptionProvider: BuddyTranscriptionProvider { + /// URL for the Cloudflare Worker endpoint that returns a short-lived + /// AssemblyAI streaming token. The real API key never leaves the server. + private static let tokenProxyURL = "https://your-worker-name.your-subdomain.workers.dev/transcribe-token" + + let displayName = "AssemblyAI" + let requiresSpeechRecognitionPermission = false + + var isConfigured: Bool { true } + var unavailableExplanation: String? { nil } + + /// Single long-lived URLSession shared across all streaming sessions. + /// Creating and invalidating a URLSession per session corrupts the OS + /// connection pool and causes "Socket is not connected" errors after + /// a few rapid reconnections to the same host. + private let sharedWebSocketURLSession = URLSession(configuration: .default) + + func startStreamingSession( + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) async throws -> any BuddyStreamingTranscriptionSession { + // Fetch a fresh temporary token from the proxy before each session + let temporaryToken = try await fetchTemporaryToken() + print("🎙️ AssemblyAI: fetched temporary token (\(temporaryToken.prefix(20))...)") + + let session = AssemblyAIStreamingTranscriptionSession( + apiKey: nil, + temporaryToken: temporaryToken, + urlSession: sharedWebSocketURLSession, + keyterms: keyterms, + onTranscriptUpdate: onTranscriptUpdate, + onFinalTranscriptReady: onFinalTranscriptReady, + onError: onError + ) + + try await session.open() + return session + } + + /// Calls the Cloudflare Worker to get a short-lived AssemblyAI token. + private func fetchTemporaryToken() async throws -> String { + var request = URLRequest(url: URL(string: Self.tokenProxyURL)!) + request.httpMethod = "POST" + + let (data, response) = try await URLSession.shared.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + let body = String(data: data, encoding: .utf8) ?? "unknown" + throw AssemblyAIStreamingTranscriptionProviderError( + message: "Failed to fetch AssemblyAI token (HTTP \(statusCode)): \(body)" + ) + } + + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let token = json["token"] as? String else { + throw AssemblyAIStreamingTranscriptionProviderError( + message: "Invalid token response from proxy." + ) + } + + return token + } +} + +private final class AssemblyAIStreamingTranscriptionSession: NSObject, BuddyStreamingTranscriptionSession { + private struct MessageEnvelope: Decodable { + let type: String + } + + private struct TurnMessage: Decodable { + let type: String + let transcript: String? + let turn_order: Int? + let end_of_turn: Bool? + let turn_is_formatted: Bool? + } + + private struct ErrorMessage: Decodable { + let type: String + let error: String? + let message: String? + } + + private struct StoredTurnTranscript { + var transcriptText: String + var isFormatted: Bool + } + + private static let websocketBaseURLString = "wss://streaming.assemblyai.com/v3/ws" + private static let targetSampleRate = 16_000.0 + private static let explicitFinalTranscriptGracePeriodSeconds = 1.4 + + let finalTranscriptFallbackDelaySeconds: TimeInterval = 2.8 + + private let apiKey: String? + private let temporaryToken: String? + private let keyterms: [String] + private let onTranscriptUpdate: (String) -> Void + private let onFinalTranscriptReady: (String) -> Void + private let onError: (Error) -> Void + + private let stateQueue = DispatchQueue(label: "com.learningbuddy.assemblyai.state") + private let sendQueue = DispatchQueue(label: "com.learningbuddy.assemblyai.send") + private let audioPCM16Converter = BuddyPCM16AudioConverter(targetSampleRate: targetSampleRate) + private let urlSession: URLSession + + private var webSocketTask: URLSessionWebSocketTask? + private var readyContinuation: CheckedContinuation? + private var hasResolvedReadyContinuation = false + private var hasDeliveredFinalTranscript = false + private var isAwaitingExplicitFinalTranscript = false + private var latestTranscriptText = "" + private var activeTurnOrder: Int? + private var activeTurnTranscriptText = "" + private var storedTurnTranscriptsByOrder: [Int: StoredTurnTranscript] = [:] + private var explicitFinalTranscriptDeadlineWorkItem: DispatchWorkItem? + + init( + apiKey: String?, + temporaryToken: String?, + urlSession: URLSession, + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) { + self.apiKey = apiKey + self.temporaryToken = temporaryToken + self.urlSession = urlSession + self.keyterms = keyterms + self.onTranscriptUpdate = onTranscriptUpdate + self.onFinalTranscriptReady = onFinalTranscriptReady + self.onError = onError + } + + func open() async throws { + let websocketURL = try Self.makeWebsocketURL( + temporaryToken: temporaryToken, + keyterms: keyterms + ) + + var websocketRequest = URLRequest(url: websocketURL) + if let apiKey { + websocketRequest.setValue(apiKey, forHTTPHeaderField: "Authorization") + } + + let webSocketTask = urlSession.webSocketTask(with: websocketRequest) + self.webSocketTask = webSocketTask + webSocketTask.resume() + + receiveNextMessage() + + try await withCheckedThrowingContinuation { continuation in + stateQueue.async { + self.readyContinuation = continuation + } + } + } + + func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { + guard let audioPCM16Data = audioPCM16Converter.convertToPCM16Data(from: audioBuffer), + !audioPCM16Data.isEmpty else { + return + } + + sendQueue.async { [weak self] in + guard let self, let webSocketTask = self.webSocketTask else { return } + webSocketTask.send(.data(audioPCM16Data)) { [weak self] error in + if let error { + self?.failSession(with: error) + } + } + } + } + + func requestFinalTranscript() { + stateQueue.async { + guard !self.hasDeliveredFinalTranscript else { return } + self.isAwaitingExplicitFinalTranscript = true + self.scheduleExplicitFinalTranscriptDeadline() + } + + sendJSONMessage(["type": "ForceEndpoint"]) + } + + func cancel() { + stateQueue.async { + self.explicitFinalTranscriptDeadlineWorkItem?.cancel() + self.explicitFinalTranscriptDeadlineWorkItem = nil + } + + sendJSONMessage(["type": "Terminate"]) + webSocketTask?.cancel(with: .goingAway, reason: nil) + } + + private func receiveNextMessage() { + webSocketTask?.receive { [weak self] result in + guard let self else { return } + + switch result { + case .success(let message): + switch message { + case .string(let text): + self.handleIncomingTextMessage(text) + case .data(let data): + if let text = String(data: data, encoding: .utf8) { + self.handleIncomingTextMessage(text) + } + @unknown default: + break + } + + self.receiveNextMessage() + case .failure(let error): + self.failSession(with: error) + } + } + } + + private func handleIncomingTextMessage(_ text: String) { + guard let messageData = text.data(using: .utf8) else { return } + + do { + let envelope = try JSONDecoder().decode(MessageEnvelope.self, from: messageData) + + switch envelope.type.lowercased() { + case "begin": + resolveReadyContinuationIfNeeded(with: .success(())) + case "turn": + let turnMessage = try JSONDecoder().decode(TurnMessage.self, from: messageData) + handleTurnMessage(turnMessage) + case "termination": + resolveReadyContinuationIfNeeded(with: .success(())) + stateQueue.async { + if self.isAwaitingExplicitFinalTranscript && !self.hasDeliveredFinalTranscript { + self.deliverFinalTranscriptIfNeeded(self.bestAvailableTranscriptText()) + } + } + case "error": + let errorMessage = try JSONDecoder().decode(ErrorMessage.self, from: messageData) + let messageText = errorMessage.error ?? errorMessage.message ?? "AssemblyAI returned an error." + failSession(with: AssemblyAIStreamingTranscriptionProviderError(message: messageText)) + default: + break + } + } catch { + failSession(with: error) + } + } + + private func handleTurnMessage(_ turnMessage: TurnMessage) { + let transcriptText = turnMessage.transcript? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + + stateQueue.async { + let turnOrder = turnMessage.turn_order + ?? self.activeTurnOrder + ?? ((self.storedTurnTranscriptsByOrder.keys.max() ?? -1) + 1) + + if turnMessage.end_of_turn == true || turnMessage.turn_is_formatted == true { + self.activeTurnOrder = nil + self.activeTurnTranscriptText = "" + self.storeTurnTranscript( + transcriptText, + forTurnOrder: turnOrder, + isFormatted: turnMessage.turn_is_formatted == true + ) + } else { + self.activeTurnOrder = turnOrder + self.activeTurnTranscriptText = transcriptText + } + + let fullTranscriptText = self.composeFullTranscript() + self.latestTranscriptText = fullTranscriptText + + if !fullTranscriptText.isEmpty { + self.onTranscriptUpdate(fullTranscriptText) + } + + guard self.isAwaitingExplicitFinalTranscript else { return } + + if turnMessage.end_of_turn == true || turnMessage.turn_is_formatted == true { + self.explicitFinalTranscriptDeadlineWorkItem?.cancel() + self.explicitFinalTranscriptDeadlineWorkItem = nil + self.deliverFinalTranscriptIfNeeded(self.bestAvailableTranscriptText()) + } + } + } + + private func storeTurnTranscript( + _ transcriptText: String, + forTurnOrder turnOrder: Int, + isFormatted: Bool + ) { + guard !transcriptText.isEmpty else { return } + + if let existingTurnTranscript = storedTurnTranscriptsByOrder[turnOrder] { + if existingTurnTranscript.isFormatted && !isFormatted { + return + } + } + + storedTurnTranscriptsByOrder[turnOrder] = StoredTurnTranscript( + transcriptText: transcriptText, + isFormatted: isFormatted + ) + } + + private func composeFullTranscript() -> String { + let committedTranscriptSegments = storedTurnTranscriptsByOrder + .sorted(by: { $0.key < $1.key }) + .map(\.value.transcriptText) + .filter { !$0.isEmpty } + + var transcriptSegments = committedTranscriptSegments + + let trimmedActiveTurnTranscriptText = activeTurnTranscriptText + .trimmingCharacters(in: .whitespacesAndNewlines) + + if !trimmedActiveTurnTranscriptText.isEmpty { + transcriptSegments.append(trimmedActiveTurnTranscriptText) + } + + return transcriptSegments.joined(separator: " ") + } + + private func scheduleExplicitFinalTranscriptDeadline() { + explicitFinalTranscriptDeadlineWorkItem?.cancel() + + let deadlineWorkItem = DispatchWorkItem { [weak self] in + self?.stateQueue.async { + guard let self else { return } + self.deliverFinalTranscriptIfNeeded(self.bestAvailableTranscriptText()) + } + } + + explicitFinalTranscriptDeadlineWorkItem = deadlineWorkItem + + DispatchQueue.main.asyncAfter( + deadline: .now() + Self.explicitFinalTranscriptGracePeriodSeconds, + execute: deadlineWorkItem + ) + } + + private func deliverFinalTranscriptIfNeeded(_ transcriptText: String) { + guard !hasDeliveredFinalTranscript else { return } + hasDeliveredFinalTranscript = true + explicitFinalTranscriptDeadlineWorkItem?.cancel() + explicitFinalTranscriptDeadlineWorkItem = nil + onFinalTranscriptReady(transcriptText) + sendJSONMessage(["type": "Terminate"]) + } + + private func sendJSONMessage(_ payload: [String: Any]) { + guard let jsonData = try? JSONSerialization.data(withJSONObject: payload), + let jsonString = String(data: jsonData, encoding: .utf8) else { + return + } + + sendQueue.async { [weak self] in + guard let self, let webSocketTask = self.webSocketTask else { return } + webSocketTask.send(.string(jsonString)) { [weak self] error in + if let error { + self?.failSession(with: error) + } + } + } + } + + private func failSession(with error: Error) { + resolveReadyContinuationIfNeeded(with: .failure(error)) + stateQueue.async { + let latestTranscriptText = self.bestAvailableTranscriptText() + + if self.isAwaitingExplicitFinalTranscript + && !self.hasDeliveredFinalTranscript + && !latestTranscriptText.isEmpty { + print("[AssemblyAI] ⚠️ WebSocket error during active session, delivering partial transcript as fallback: \(error.localizedDescription)") + self.deliverFinalTranscriptIfNeeded(latestTranscriptText) + return + } + print("[AssemblyAI] ❌ Session failed with error: \(error.localizedDescription)") + + self.onError(error) + } + } + + private func bestAvailableTranscriptText() -> String { + let composedTranscriptText = composeFullTranscript() + .trimmingCharacters(in: .whitespacesAndNewlines) + + if !composedTranscriptText.isEmpty { + return composedTranscriptText + } + + return latestTranscriptText.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private func resolveReadyContinuationIfNeeded(with result: Result) { + stateQueue.async { + guard !self.hasResolvedReadyContinuation else { return } + self.hasResolvedReadyContinuation = true + + switch result { + case .success: + self.readyContinuation?.resume() + case .failure(let error): + self.readyContinuation?.resume(throwing: error) + } + + self.readyContinuation = nil + } + } + + private static func makeWebsocketURL( + temporaryToken: String?, + keyterms: [String] + ) throws -> URL { + guard var websocketURLComponents = URLComponents(string: websocketBaseURLString) else { + throw AssemblyAIStreamingTranscriptionProviderError( + message: "AssemblyAI websocket URL is invalid." + ) + } + + var queryItems = [ + URLQueryItem(name: "sample_rate", value: "16000"), + URLQueryItem(name: "encoding", value: "pcm_s16le"), + URLQueryItem(name: "format_turns", value: "true"), + URLQueryItem(name: "speech_model", value: "u3-rt-pro") + ] + + let normalizedKeyterms = keyterms + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + + if !normalizedKeyterms.isEmpty, + let keytermsData = try? JSONSerialization.data(withJSONObject: normalizedKeyterms), + let keytermsJSONString = String(data: keytermsData, encoding: .utf8) { + queryItems.append(URLQueryItem(name: "keyterms_prompt", value: keytermsJSONString)) + } + + if let temporaryToken { + queryItems.append(URLQueryItem(name: "token", value: temporaryToken)) + } + + websocketURLComponents.queryItems = queryItems + + guard let websocketURL = websocketURLComponents.url else { + throw AssemblyAIStreamingTranscriptionProviderError( + message: "AssemblyAI websocket URL could not be created." + ) + } + + return websocketURL + } +} diff --git a/leanring-buddy/Assets.xcassets/AccentColor.colorset/Contents.json b/leanring-buddy/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 0000000..eb87897 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/1024-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/1024-mac.png new file mode 100644 index 0000000..037d7ca Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/1024-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/128-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/128-mac.png new file mode 100644 index 0000000..c056299 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/128-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/16-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/16-mac.png new file mode 100644 index 0000000..c6dcb77 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/16-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/256-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/256-mac.png new file mode 100644 index 0000000..cbae977 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/256-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/32-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/32-mac.png new file mode 100644 index 0000000..a5088ec Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/32-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/512-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/512-mac.png new file mode 100644 index 0000000..1ca5e44 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/512-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/64-mac.png b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/64-mac.png new file mode 100644 index 0000000..ef90c27 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/64-mac.png differ diff --git a/leanring-buddy/Assets.xcassets/AppIcon.appiconset/Contents.json b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..5177872 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,74 @@ +{ + "images" : [ + { + "filename" : "1024-mac.png", + "idiom" : "ios-marketing", + "scale" : "1x", + "size" : "1024x1024" + }, + { + "filename" : "16-mac.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "16x16" + }, + { + "filename" : "32-mac.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "16x16" + }, + { + "filename" : "32-mac.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "32x32" + }, + { + "filename" : "64-mac.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "32x32" + }, + { + "filename" : "128-mac.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "128x128" + }, + { + "filename" : "256-mac.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "128x128" + }, + { + "filename" : "256-mac.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "256x256" + }, + { + "filename" : "512-mac.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "256x256" + }, + { + "filename" : "512-mac.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "512x512" + }, + { + "filename" : "1024-mac.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "512x512" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/Contents.json b/leanring-buddy/Assets.xcassets/Contents.json new file mode 100644 index 0000000..73c0059 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Contents.json b/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Contents.json new file mode 100644 index 0000000..17f6fd9 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "Image.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Image.png b/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Image.png new file mode 100644 index 0000000..4584882 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/add-a-project-in-codex-from-sidebar.imageset/Image.png differ diff --git a/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/Contents.json b/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/Contents.json new file mode 100644 index 0000000..93ca50f --- /dev/null +++ b/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "add-a-project-in-codex.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/add-a-project-in-codex.png b/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/add-a-project-in-codex.png new file mode 100644 index 0000000..68847ce Binary files /dev/null and b/leanring-buddy/Assets.xcassets/add-a-project-in-codex.imageset/add-a-project-in-codex.png differ diff --git a/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/Contents.json b/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/Contents.json new file mode 100644 index 0000000..37b9955 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "codex-app-screenshot.jpg", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/codex-app-screenshot.jpg b/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/codex-app-screenshot.jpg new file mode 100644 index 0000000..47548b9 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/codex-app-screenshot.imageset/codex-app-screenshot.jpg differ diff --git a/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/Contents.json b/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/Contents.json new file mode 100644 index 0000000..8b73e30 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "codex-home-screen.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/codex-home-screen.png b/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/codex-home-screen.png new file mode 100644 index 0000000..02e74f1 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/codex-home-screen.imageset/codex-home-screen.png differ diff --git a/leanring-buddy/Assets.xcassets/codex-permissions.imageset/Contents.json b/leanring-buddy/Assets.xcassets/codex-permissions.imageset/Contents.json new file mode 100644 index 0000000..5ca6299 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/codex-permissions.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "codex-permissions.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/codex-permissions.imageset/codex-permissions.png b/leanring-buddy/Assets.xcassets/codex-permissions.imageset/codex-permissions.png new file mode 100644 index 0000000..308b542 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/codex-permissions.imageset/codex-permissions.png differ diff --git a/leanring-buddy/Assets.xcassets/discord-logo.imageset/Contents.json b/leanring-buddy/Assets.xcassets/discord-logo.imageset/Contents.json new file mode 100644 index 0000000..65d8ed3 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/discord-logo.imageset/Contents.json @@ -0,0 +1,25 @@ +{ + "images" : [ + { + "filename" : "discord-logo.svg", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/leanring-buddy/Assets.xcassets/discord-logo.imageset/discord-logo.svg b/leanring-buddy/Assets.xcassets/discord-logo.imageset/discord-logo.svg new file mode 100644 index 0000000..f059720 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/discord-logo.imageset/discord-logo.svg @@ -0,0 +1,3 @@ + + + diff --git a/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/Contents.json b/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/Contents.json new file mode 100644 index 0000000..43657ae --- /dev/null +++ b/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "git-tools-prompt.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/git-tools-prompt.png b/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/git-tools-prompt.png new file mode 100644 index 0000000..f4aa6e4 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/git-tools-prompt.imageset/git-tools-prompt.png differ diff --git a/leanring-buddy/Assets.xcassets/google-logo.imageset/Contents.json b/leanring-buddy/Assets.xcassets/google-logo.imageset/Contents.json new file mode 100644 index 0000000..9ce91aa --- /dev/null +++ b/leanring-buddy/Assets.xcassets/google-logo.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "google-logo.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "original" + } +} diff --git a/leanring-buddy/Assets.xcassets/google-logo.imageset/google-logo.svg b/leanring-buddy/Assets.xcassets/google-logo.imageset/google-logo.svg new file mode 100644 index 0000000..e102c9d --- /dev/null +++ b/leanring-buddy/Assets.xcassets/google-logo.imageset/google-logo.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/Contents.json b/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/Contents.json new file mode 100644 index 0000000..5d0d423 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "inside-the-makesomething-project-folder.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/inside-the-makesomething-project-folder.png b/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/inside-the-makesomething-project-folder.png new file mode 100644 index 0000000..6a7ecf7 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/inside-the-makesomething-project-folder.imageset/inside-the-makesomething-project-folder.png differ diff --git a/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/Contents.json b/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/Contents.json new file mode 100644 index 0000000..5d5d66c --- /dev/null +++ b/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "filename" : "makesomething-project-folder-in-downloads.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/makesomething-project-folder-in-downloads.png b/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/makesomething-project-folder-in-downloads.png new file mode 100644 index 0000000..8039a73 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/makesomething-project-folder-in-downloads.imageset/makesomething-project-folder-in-downloads.png differ diff --git a/leanring-buddy/Assets.xcassets/steve.imageset/Contents.json b/leanring-buddy/Assets.xcassets/steve.imageset/Contents.json new file mode 100644 index 0000000..9cac693 --- /dev/null +++ b/leanring-buddy/Assets.xcassets/steve.imageset/Contents.json @@ -0,0 +1,12 @@ +{ + "images" : [ + { + "filename" : "steve.jpg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/leanring-buddy/Assets.xcassets/steve.imageset/steve.jpg b/leanring-buddy/Assets.xcassets/steve.imageset/steve.jpg new file mode 100644 index 0000000..19b2d40 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/steve.imageset/steve.jpg differ diff --git a/leanring-buddy/Assets.xcassets/steve.jpg b/leanring-buddy/Assets.xcassets/steve.jpg new file mode 100644 index 0000000..19b2d40 Binary files /dev/null and b/leanring-buddy/Assets.xcassets/steve.jpg differ diff --git a/leanring-buddy/BuddyAudioConversionSupport.swift b/leanring-buddy/BuddyAudioConversionSupport.swift new file mode 100644 index 0000000..02623c6 --- /dev/null +++ b/leanring-buddy/BuddyAudioConversionSupport.swift @@ -0,0 +1,108 @@ +// +// BuddyAudioConversionSupport.swift +// leanring-buddy +// +// Shared audio conversion helpers for voice transcription providers. +// + +import AVFoundation +import Foundation + +final class BuddyPCM16AudioConverter { + private let targetAudioFormat: AVAudioFormat + private var audioConverter: AVAudioConverter? + private var currentInputFormatDescription: String? + + init(targetSampleRate: Double) { + self.targetAudioFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: targetSampleRate, + channels: 1, + interleaved: true + )! + } + + func convertToPCM16Data(from audioBuffer: AVAudioPCMBuffer) -> Data? { + let inputFormatDescription = audioBuffer.format.settings.description + + if currentInputFormatDescription != inputFormatDescription { + audioConverter = AVAudioConverter(from: audioBuffer.format, to: targetAudioFormat) + currentInputFormatDescription = inputFormatDescription + } + + guard let audioConverter else { return nil } + + let sampleRateRatio = targetAudioFormat.sampleRate / audioBuffer.format.sampleRate + let outputFrameCapacity = AVAudioFrameCount( + (Double(audioBuffer.frameLength) * sampleRateRatio).rounded(.up) + 32 + ) + + guard let outputBuffer = AVAudioPCMBuffer( + pcmFormat: targetAudioFormat, + frameCapacity: outputFrameCapacity + ) else { + return nil + } + + var hasProvidedSourceBuffer = false + var conversionError: NSError? + + let conversionStatus = audioConverter.convert(to: outputBuffer, error: &conversionError) { _, outStatus in + if hasProvidedSourceBuffer { + outStatus.pointee = .noDataNow + return nil + } + + hasProvidedSourceBuffer = true + outStatus.pointee = .haveData + return audioBuffer + } + + guard conversionStatus != .error else { return nil } + guard let pcmDataPointer = outputBuffer.audioBufferList.pointee.mBuffers.mData else { return nil } + + let bytesPerFrame = Int(targetAudioFormat.streamDescription.pointee.mBytesPerFrame) + let byteCount = Int(outputBuffer.frameLength) * bytesPerFrame + guard byteCount > 0 else { return nil } + + return Data(bytes: pcmDataPointer, count: byteCount) + } +} + +enum BuddyWAVFileBuilder { + static func buildWAVData( + fromPCM16MonoAudio pcm16AudioData: Data, + sampleRate: Int, + channelCount: Int = 1, + bitsPerSample: Int = 16 + ) -> Data { + let byteRate = sampleRate * channelCount * bitsPerSample / 8 + let blockAlign = channelCount * bitsPerSample / 8 + let dataChunkSize = UInt32(pcm16AudioData.count) + let fileSize = UInt32(36) + dataChunkSize + + var wavData = Data() + + wavData.append("RIFF".data(using: .ascii)!) + wavData.append(littleEndianData(from: fileSize)) + wavData.append("WAVE".data(using: .ascii)!) + wavData.append("fmt ".data(using: .ascii)!) + wavData.append(littleEndianData(from: UInt32(16))) + wavData.append(littleEndianData(from: UInt16(1))) + wavData.append(littleEndianData(from: UInt16(channelCount))) + wavData.append(littleEndianData(from: UInt32(sampleRate))) + wavData.append(littleEndianData(from: UInt32(byteRate))) + wavData.append(littleEndianData(from: UInt16(blockAlign))) + wavData.append(littleEndianData(from: UInt16(bitsPerSample))) + wavData.append("data".data(using: .ascii)!) + wavData.append(littleEndianData(from: dataChunkSize)) + wavData.append(pcm16AudioData) + + return wavData + } + + private static func littleEndianData(from value: T) -> Data { + var littleEndianValue = value.littleEndian + return Data(bytes: &littleEndianValue, count: MemoryLayout.size) + } +} diff --git a/leanring-buddy/BuddyDictationManager.swift b/leanring-buddy/BuddyDictationManager.swift new file mode 100644 index 0000000..5bca267 --- /dev/null +++ b/leanring-buddy/BuddyDictationManager.swift @@ -0,0 +1,866 @@ +// +// BuddyDictationManager.swift +// leanring-buddy +// +// Shared push-to-talk dictation manager for the help chat and brainstorm buddy. +// Captures microphone audio with AVAudioEngine, routes it into the active +// transcription provider, and hands the final draft back to the active input bar. +// + +import AppKit +import AVFoundation +import Combine +import Foundation +import Speech + +enum BuddyPushToTalkShortcut { + enum ShortcutOption { + case shiftFunction + case controlOption + case shiftControl + case controlOptionSpace + case shiftControlSpace + + var displayText: String { + switch self { + case .shiftFunction: + return "shift + fn" + case .controlOption: + return "ctrl + option" + case .shiftControl: + return "shift + control" + case .controlOptionSpace: + return "ctrl + option + space" + case .shiftControlSpace: + return "shift + control + space" + } + } + + var keyCapsuleLabels: [String] { + switch self { + case .shiftFunction: + return ["shift", "fn"] + case .controlOption: + return ["ctrl", "option"] + case .shiftControl: + return ["shift", "control"] + case .controlOptionSpace: + return ["ctrl", "option", "space"] + case .shiftControlSpace: + return ["shift", "control", "space"] + } + } + + fileprivate var modifierOnlyFlags: NSEvent.ModifierFlags? { + switch self { + case .shiftFunction: + return [.shift, .function] + case .controlOption: + return [.control, .option] + case .shiftControl: + return [.shift, .control] + case .controlOptionSpace, .shiftControlSpace: + return nil + } + } + + fileprivate var spaceShortcutModifierFlags: NSEvent.ModifierFlags? { + switch self { + case .shiftFunction: + return nil + case .controlOption: + return nil + case .shiftControl: + return nil + case .controlOptionSpace: + return [.control, .option] + case .shiftControlSpace: + return [.shift, .control] + } + } + } + + enum ShortcutTransition { + case none + case pressed + case released + } + + private enum ShortcutEventType { + case flagsChanged + case keyDown + case keyUp + } + + static let currentShortcutOption: ShortcutOption = .controlOption + static let pushToTalkKeyCode: UInt16 = 49 // Space + static let pushToTalkDisplayText = currentShortcutOption.displayText + static let pushToTalkTooltipText = "push to talk (\(pushToTalkDisplayText))" + + static func shortcutTransition( + for event: NSEvent, + wasShortcutPreviouslyPressed: Bool + ) -> ShortcutTransition { + guard let shortcutEventType = shortcutEventType(for: event.type) else { return .none } + + return shortcutTransition( + for: shortcutEventType, + keyCode: event.keyCode, + modifierFlags: event.modifierFlags.intersection(.deviceIndependentFlagsMask), + wasShortcutPreviouslyPressed: wasShortcutPreviouslyPressed + ) + } + + static func shortcutTransition( + for eventType: CGEventType, + keyCode: UInt16, + modifierFlagsRawValue: UInt64, + wasShortcutPreviouslyPressed: Bool + ) -> ShortcutTransition { + guard let shortcutEventType = shortcutEventType(for: eventType) else { return .none } + + return shortcutTransition( + for: shortcutEventType, + keyCode: keyCode, + modifierFlags: NSEvent.ModifierFlags(rawValue: UInt(modifierFlagsRawValue)) + .intersection(.deviceIndependentFlagsMask), + wasShortcutPreviouslyPressed: wasShortcutPreviouslyPressed + ) + } + + private static func shortcutEventType(for eventType: NSEvent.EventType) -> ShortcutEventType? { + switch eventType { + case .flagsChanged: + return .flagsChanged + case .keyDown: + return .keyDown + case .keyUp: + return .keyUp + default: + return nil + } + } + + private static func shortcutEventType(for eventType: CGEventType) -> ShortcutEventType? { + switch eventType { + case .flagsChanged: + return .flagsChanged + case .keyDown: + return .keyDown + case .keyUp: + return .keyUp + default: + return nil + } + } + + private static func shortcutTransition( + for shortcutEventType: ShortcutEventType, + keyCode: UInt16, + modifierFlags: NSEvent.ModifierFlags, + wasShortcutPreviouslyPressed: Bool + ) -> ShortcutTransition { + if let modifierOnlyFlags = currentShortcutOption.modifierOnlyFlags { + guard shortcutEventType == .flagsChanged else { return .none } + + let isShortcutCurrentlyPressed = modifierFlags.contains(modifierOnlyFlags) + + if isShortcutCurrentlyPressed && !wasShortcutPreviouslyPressed { + return .pressed + } + + if !isShortcutCurrentlyPressed && wasShortcutPreviouslyPressed { + return .released + } + + return .none + } + + guard let pushToTalkModifierFlags = currentShortcutOption.spaceShortcutModifierFlags else { + return .none + } + + let matchesModifierFlags = modifierFlags.isSuperset(of: pushToTalkModifierFlags) + + if shortcutEventType == .keyDown + && keyCode == pushToTalkKeyCode + && matchesModifierFlags + && !wasShortcutPreviouslyPressed { + return .pressed + } + + if shortcutEventType == .keyUp + && keyCode == pushToTalkKeyCode + && wasShortcutPreviouslyPressed { + return .released + } + + return .none + } +} + +enum BuddyDictationPermissionProblem { + case microphoneAccessDenied + case speechRecognitionDenied +} + +private enum BuddyDictationStartSource { + case microphoneButton + case keyboardShortcut +} + +private struct BuddyDictationDraftCallbacks { + let updateDraftText: (String) -> Void + let submitDraftText: (String) -> Void +} + +@MainActor +final class BuddyDictationManager: NSObject, ObservableObject { + private static let defaultFinalTranscriptFallbackDelaySeconds: TimeInterval = 2.4 + private static let recordedAudioPowerHistoryLength = 44 + private static let recordedAudioPowerHistoryBaselineLevel: CGFloat = 0.02 + private static let recordedAudioPowerHistorySampleIntervalSeconds: TimeInterval = 0.07 + + @Published private(set) var isRecordingFromMicrophoneButton = false + @Published private(set) var isRecordingFromKeyboardShortcut = false + @Published private(set) var isKeyboardShortcutSessionActiveOrFinalizing = false + @Published private(set) var isFinalizingTranscript = false + @Published private(set) var isPreparingToRecord = false + @Published private(set) var currentAudioPowerLevel: CGFloat = 0 + @Published private(set) var recordedAudioPowerHistory = Array( + repeating: BuddyDictationManager.recordedAudioPowerHistoryBaselineLevel, + count: BuddyDictationManager.recordedAudioPowerHistoryLength + ) + @Published private(set) var microphoneButtonRecordingStartedAt: Date? + @Published private(set) var transcriptionProviderDisplayName = "" + @Published var lastErrorMessage: String? + @Published private(set) var currentPermissionProblem: BuddyDictationPermissionProblem? + + var isDictationInProgress: Bool { + isPreparingToRecord || isRecordingFromMicrophoneButton || isRecordingFromKeyboardShortcut || isFinalizingTranscript + } + + var isActivelyRecordingAudio: Bool { + isRecordingFromMicrophoneButton || isRecordingFromKeyboardShortcut + } + + var isMicrophoneButtonActivelyRecordingAudio: Bool { + isRecordingFromMicrophoneButton + } + + var isMicrophoneButtonSessionBusy: Bool { + activeStartSource == .microphoneButton + && (isPreparingToRecord || isRecordingFromMicrophoneButton || isFinalizingTranscript) + } + + var needsInitialPermissionPrompt: Bool { + if transcriptionProvider.requiresSpeechRecognitionPermission { + return AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined + || SFSpeechRecognizer.authorizationStatus() == .notDetermined + } + + return AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined + } + + private let transcriptionProvider: any BuddyTranscriptionProvider + private let audioEngine = AVAudioEngine() + private var activeTranscriptionSession: (any BuddyStreamingTranscriptionSession)? + private var activeStartSource: BuddyDictationStartSource? + private var draftCallbacks: BuddyDictationDraftCallbacks? + private var draftTextBeforeCurrentDictation = "" + private var latestRecognizedText = "" + private var shouldAutomaticallySubmitFinalDraft = false + private var hasFinishedCurrentDictationSession = false + private var finalizeFallbackWorkItem: DispatchWorkItem? + private var pendingStartRequestIdentifier = UUID() + private var contextualKeyterms: [String] = [] + private var lastRecordedAudioPowerSampleDate = Date.distantPast + private var activePermissionRequestTask: Task? + /// Timestamp of the last completed permission request, used to debounce + /// rapid follow-up requests that arrive before macOS updates its cache. + private var lastPermissionRequestCompletedAt: Date? + + override init() { + let transcriptionProvider = BuddyTranscriptionProviderFactory.makeDefaultProvider() + self.transcriptionProvider = transcriptionProvider + self.transcriptionProviderDisplayName = transcriptionProvider.displayName + super.init() + } + + func updateContextualKeyterms(_ contextualKeyterms: [String]) { + self.contextualKeyterms = contextualKeyterms + } + + func startPersistentDictationFromMicrophoneButton( + currentDraftText: String, + updateDraftText: @escaping (String) -> Void, + submitDraftText: @escaping (String) -> Void + ) async { + await startPushToTalk( + startSource: .microphoneButton, + currentDraftText: currentDraftText, + updateDraftText: updateDraftText, + submitDraftText: submitDraftText, + shouldAutomaticallySubmitFinalDraftOnStop: false + ) + } + + func startPushToTalkFromKeyboardShortcut( + currentDraftText: String, + updateDraftText: @escaping (String) -> Void, + submitDraftText: @escaping (String) -> Void + ) async { + await startPushToTalk( + startSource: .keyboardShortcut, + currentDraftText: currentDraftText, + updateDraftText: updateDraftText, + submitDraftText: submitDraftText, + shouldAutomaticallySubmitFinalDraftOnStop: currentDraftText + .trimmingCharacters(in: .whitespacesAndNewlines) + .isEmpty + ) + } + + func stopPersistentDictationFromMicrophoneButton() { + stopPushToTalk(expectedStartSource: .microphoneButton) + } + + func stopPushToTalkFromKeyboardShortcut() { + stopPushToTalk(expectedStartSource: .keyboardShortcut) + } + + func cancelCurrentDictation(preserveDraftText: Bool = true) { + pendingStartRequestIdentifier = UUID() + + guard isDictationInProgress else { return } + + finalizeFallbackWorkItem?.cancel() + finalizeFallbackWorkItem = nil + + if preserveDraftText { + let currentDraftText = composeDraftText(withTranscribedText: latestRecognizedText) + draftCallbacks?.updateDraftText(currentDraftText) + } + + audioEngine.stop() + audioEngine.inputNode.removeTap(onBus: 0) + activeTranscriptionSession?.cancel() + + resetSessionState() + } + + func requestInitialPushToTalkPermissionsIfNeeded() async { + guard needsInitialPermissionPrompt else { return } + guard !isDictationInProgress else { return } + + lastErrorMessage = nil + currentPermissionProblem = nil + isPreparingToRecord = true + + NSApplication.shared.activate(ignoringOtherApps: true) + + do { + try await Task.sleep(for: .milliseconds(200)) + } catch { + // If the task is cancelled while we are waiting for macOS to bring + // the app forward, we can safely continue into the permission check. + } + + let hasPermissions = await requestMicrophoneAndSpeechPermissionsWithoutDuplicatePrompts() + isPreparingToRecord = false + + if hasPermissions { + lastErrorMessage = nil + } + } + + private func startPushToTalk( + startSource: BuddyDictationStartSource, + currentDraftText: String, + updateDraftText: @escaping (String) -> Void, + submitDraftText: @escaping (String) -> Void, + shouldAutomaticallySubmitFinalDraftOnStop: Bool + ) async { + guard !isDictationInProgress else { return } + + print("🎙️ BuddyDictationManager: start requested (\(startSource))") + + if needsInitialPermissionPrompt { + print("🎙️ BuddyDictationManager: requesting initial permissions") + NSApplication.shared.activate(ignoringOtherApps: true) + + do { + try await Task.sleep(for: .milliseconds(200)) + } catch { + // If the task is cancelled while the app is being activated, + // we can safely continue into the permission request. + } + } + + let startRequestIdentifier = UUID() + pendingStartRequestIdentifier = startRequestIdentifier + + lastErrorMessage = nil + currentPermissionProblem = nil + isPreparingToRecord = true + + guard await requestMicrophoneAndSpeechPermissionsWithoutDuplicatePrompts() else { + print("🎙️ BuddyDictationManager: permissions missing or denied") + isPreparingToRecord = false + return + } + guard !Task.isCancelled else { + print("🎙️ BuddyDictationManager: start cancelled (shortcut released during permission check)") + isPreparingToRecord = false + return + } + guard pendingStartRequestIdentifier == startRequestIdentifier else { + print("🎙️ BuddyDictationManager: start request superseded") + isPreparingToRecord = false + return + } + + draftTextBeforeCurrentDictation = currentDraftText + latestRecognizedText = "" + draftCallbacks = BuddyDictationDraftCallbacks( + updateDraftText: updateDraftText, + submitDraftText: submitDraftText + ) + activeStartSource = startSource + shouldAutomaticallySubmitFinalDraft = shouldAutomaticallySubmitFinalDraftOnStop + hasFinishedCurrentDictationSession = false + isFinalizingTranscript = false + isRecordingFromMicrophoneButton = startSource == .microphoneButton + isRecordingFromKeyboardShortcut = startSource == .keyboardShortcut + isKeyboardShortcutSessionActiveOrFinalizing = startSource == .keyboardShortcut + currentAudioPowerLevel = 0 + recordedAudioPowerHistory = Array( + repeating: Self.recordedAudioPowerHistoryBaselineLevel, + count: Self.recordedAudioPowerHistoryLength + ) + microphoneButtonRecordingStartedAt = nil + lastRecordedAudioPowerSampleDate = .distantPast + + guard !Task.isCancelled else { + print("🎙️ BuddyDictationManager: start cancelled (shortcut released before recording began)") + resetSessionState() + return + } + + do { + try await startRecognitionSession() + guard !Task.isCancelled else { + print("🎙️ BuddyDictationManager: start cancelled (shortcut released during session start)") + audioEngine.stop() + audioEngine.inputNode.removeTap(onBus: 0) + activeTranscriptionSession?.cancel() + resetSessionState() + return + } + if startSource == .microphoneButton { + microphoneButtonRecordingStartedAt = Date() + } + isPreparingToRecord = false + print("🎙️ BuddyDictationManager: recognition session started") + } catch { + isPreparingToRecord = false + lastErrorMessage = userFacingErrorMessage( + from: error, + fallback: "couldn't start voice input. try again." + ) + print("❌ BuddyDictationManager: failed to start recognition session (\(transcriptionProvider.displayName)): \(error)") + resetSessionState() + } + } + + private func stopPushToTalk(expectedStartSource: BuddyDictationStartSource) { + pendingStartRequestIdentifier = UUID() + + guard activeStartSource == expectedStartSource else { + isPreparingToRecord = false + return + } + guard !isFinalizingTranscript else { return } + + print("🎙️ BuddyDictationManager: stop requested (\(expectedStartSource))") + + isRecordingFromMicrophoneButton = false + isRecordingFromKeyboardShortcut = false + isFinalizingTranscript = true + + let finalTranscriptFallbackDelaySeconds = activeTranscriptionSession?.finalTranscriptFallbackDelaySeconds + ?? Self.defaultFinalTranscriptFallbackDelaySeconds + + audioEngine.stop() + audioEngine.inputNode.removeTap(onBus: 0) + activeTranscriptionSession?.requestFinalTranscript() + + finalizeFallbackWorkItem?.cancel() + let shouldSubmitFinalDraftWhenFallbackTriggers = shouldAutomaticallySubmitFinalDraft + let fallbackWorkItem = DispatchWorkItem { [weak self] in + Task { @MainActor in + self?.finishCurrentDictationSessionIfNeeded( + shouldSubmitFinalDraft: shouldSubmitFinalDraftWhenFallbackTriggers + ) + } + } + finalizeFallbackWorkItem = fallbackWorkItem + DispatchQueue.main.asyncAfter( + deadline: .now() + finalTranscriptFallbackDelaySeconds, + execute: fallbackWorkItem + ) + } + + private func startRecognitionSession() async throws { + activeTranscriptionSession?.cancel() + activeTranscriptionSession = nil + + print("🎙️ BuddyDictationManager: opening transcription provider \(transcriptionProvider.displayName)") + + let activeTranscriptionSession = try await transcriptionProvider.startStreamingSession( + keyterms: buildTranscriptionKeyterms(), + onTranscriptUpdate: { [weak self] transcriptText in + Task { @MainActor in + self?.latestRecognizedText = transcriptText + } + }, + onFinalTranscriptReady: { [weak self] transcriptText in + Task { @MainActor in + guard let self else { return } + self.latestRecognizedText = transcriptText + + if self.isFinalizingTranscript { + self.finishCurrentDictationSessionIfNeeded( + shouldSubmitFinalDraft: self.shouldAutomaticallySubmitFinalDraft + ) + } + } + }, + onError: { [weak self] error in + Task { @MainActor in + self?.handleRecognitionError(error) + } + } + ) + + self.activeTranscriptionSession = activeTranscriptionSession + print("🎙️ BuddyDictationManager: provider ready, starting audio engine") + + let inputNode = audioEngine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in + self?.activeTranscriptionSession?.appendAudioBuffer(buffer) + self?.updateAudioPowerLevel(from: buffer) + } + + audioEngine.prepare() + try audioEngine.start() + } + + private func handleRecognitionError(_ error: Error) { + if hasFinishedCurrentDictationSession { + return + } + + if isFinalizingTranscript && !latestRecognizedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + finishCurrentDictationSessionIfNeeded( + shouldSubmitFinalDraft: shouldAutomaticallySubmitFinalDraft + ) + } else { + print("❌ Buddy dictation error (\(transcriptionProvider.displayName)): \(error)") + lastErrorMessage = userFacingErrorMessage( + from: error, + fallback: "couldn't transcribe that. try again." + ) + cancelCurrentDictation(preserveDraftText: false) + } + } + + private func finishCurrentDictationSessionIfNeeded(shouldSubmitFinalDraft: Bool) { + guard !hasFinishedCurrentDictationSession else { return } + hasFinishedCurrentDictationSession = true + + finalizeFallbackWorkItem?.cancel() + finalizeFallbackWorkItem = nil + + let finalDraftText = composeDraftText(withTranscribedText: latestRecognizedText) + let finalTranscriptText = latestRecognizedText.trimmingCharacters(in: .whitespacesAndNewlines) + let currentDraftCallbacks = draftCallbacks + + if !shouldSubmitFinalDraft && !finalDraftText.isEmpty { + currentDraftCallbacks?.updateDraftText(finalDraftText) + } + + audioEngine.stop() + audioEngine.inputNode.removeTap(onBus: 0) + activeTranscriptionSession?.cancel() + + resetSessionState() + + guard shouldSubmitFinalDraft else { return } + guard !finalTranscriptText.isEmpty else { return } + + currentDraftCallbacks?.submitDraftText(finalDraftText) + } + + private func composeDraftText(withTranscribedText transcribedText: String) -> String { + let trimmedTranscriptText = transcribedText.trimmingCharacters(in: .whitespacesAndNewlines) + + guard !trimmedTranscriptText.isEmpty else { + return draftTextBeforeCurrentDictation + } + + let trimmedExistingDraftText = draftTextBeforeCurrentDictation + .trimmingCharacters(in: .whitespacesAndNewlines) + + guard !trimmedExistingDraftText.isEmpty else { + return trimmedTranscriptText + } + + if draftTextBeforeCurrentDictation.hasSuffix(" ") || draftTextBeforeCurrentDictation.hasSuffix("\n") { + return draftTextBeforeCurrentDictation + trimmedTranscriptText + } + + return draftTextBeforeCurrentDictation + " " + trimmedTranscriptText + } + + private func resetSessionState() { + pendingStartRequestIdentifier = UUID() + activeTranscriptionSession = nil + draftCallbacks = nil + activeStartSource = nil + draftTextBeforeCurrentDictation = "" + latestRecognizedText = "" + shouldAutomaticallySubmitFinalDraft = false + hasFinishedCurrentDictationSession = false + isPreparingToRecord = false + isRecordingFromMicrophoneButton = false + isRecordingFromKeyboardShortcut = false + isKeyboardShortcutSessionActiveOrFinalizing = false + isFinalizingTranscript = false + currentAudioPowerLevel = 0 + recordedAudioPowerHistory = Array( + repeating: Self.recordedAudioPowerHistoryBaselineLevel, + count: Self.recordedAudioPowerHistoryLength + ) + microphoneButtonRecordingStartedAt = nil + lastRecordedAudioPowerSampleDate = .distantPast + } + + private func buildTranscriptionKeyterms() -> [String] { + let baseKeyterms = [ + "makesomething", + "Learning Buddy", + "Codex", + "Claude", + "Anthropic", + "OpenAI", + "SwiftUI", + "Xcode", + "Vercel", + "Next.js", + "localhost" + ] + + let combinedKeyterms = baseKeyterms + contextualKeyterms + var uniqueNormalizedKeyterms = Set() + var orderedKeyterms: [String] = [] + + for keyterm in combinedKeyterms { + let trimmedKeyterm = keyterm.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedKeyterm.isEmpty else { continue } + + let normalizedKeyterm = trimmedKeyterm.lowercased() + if uniqueNormalizedKeyterms.contains(normalizedKeyterm) { + continue + } + + uniqueNormalizedKeyterms.insert(normalizedKeyterm) + orderedKeyterms.append(trimmedKeyterm) + } + + return orderedKeyterms + } + + private func updateAudioPowerLevel(from audioBuffer: AVAudioPCMBuffer) { + guard let channelData = audioBuffer.floatChannelData else { return } + + let channelSamples = channelData[0] + let frameCount = Int(audioBuffer.frameLength) + guard frameCount > 0 else { return } + + var summedSquares: Float = 0 + for sampleIndex in 0..= Self.recordedAudioPowerHistorySampleIntervalSeconds { + self.lastRecordedAudioPowerSampleDate = now + self.appendRecordedAudioPowerSample( + max(CGFloat(boostedLevel), Self.recordedAudioPowerHistoryBaselineLevel) + ) + } + } + } + + private func appendRecordedAudioPowerSample(_ audioPowerSample: CGFloat) { + var updatedRecordedAudioPowerHistory = recordedAudioPowerHistory + updatedRecordedAudioPowerHistory.append(audioPowerSample) + + if updatedRecordedAudioPowerHistory.count > Self.recordedAudioPowerHistoryLength { + updatedRecordedAudioPowerHistory.removeFirst( + updatedRecordedAudioPowerHistory.count - Self.recordedAudioPowerHistoryLength + ) + } + + recordedAudioPowerHistory = updatedRecordedAudioPowerHistory + } + + private func requestMicrophoneAndSpeechPermissionsIfNeeded() async -> Bool { + let hasMicrophonePermission = await requestMicrophonePermissionIfNeeded() + guard hasMicrophonePermission else { + lastErrorMessage = "microphone permission is required for push to talk." + return false + } + + guard transcriptionProvider.requiresSpeechRecognitionPermission else { + return true + } + + let hasSpeechRecognitionPermission = await requestSpeechRecognitionPermissionIfNeeded() + guard hasSpeechRecognitionPermission else { + lastErrorMessage = "speech recognition permission is required for push to talk." + return false + } + + return true + } + + /// macOS can show the microphone/speech sheet again if we accidentally fan out + /// multiple permission requests before the first one finishes. We keep exactly + /// one in-flight request task so rapid repeat presses all await the same result. + /// + /// After the task completes, we skip re-requesting for a short cooldown period + /// so macOS has time to update its authorization cache. This prevents the + /// permission dialog from popping up again on rapid follow-up presses. + private func requestMicrophoneAndSpeechPermissionsWithoutDuplicatePrompts() async -> Bool { + // If a permission request is already in-flight, reuse it. + if let activePermissionRequestTask { + return await activePermissionRequestTask.value + } + + // If we just finished a permission request very recently, skip re-requesting. + // macOS can briefly report .notDetermined even after the user tapped Allow, + // so we trust the cached result for a short window. + if let lastPermissionRequestCompletedAt, + Date().timeIntervalSince(lastPermissionRequestCompletedAt) < 1.0 { + return AVCaptureDevice.authorizationStatus(for: .audio) != .denied + && AVCaptureDevice.authorizationStatus(for: .audio) != .restricted + } + + let permissionRequestTask = Task { @MainActor in + await self.requestMicrophoneAndSpeechPermissionsIfNeeded() + } + + activePermissionRequestTask = permissionRequestTask + + let hasPermissions = await permissionRequestTask.value + activePermissionRequestTask = nil + lastPermissionRequestCompletedAt = Date() + return hasPermissions + } + + private func requestMicrophonePermissionIfNeeded() async -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + currentPermissionProblem = nil + return true + case .notDetermined: + let isGranted = await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { isGranted in + continuation.resume(returning: isGranted) + } + } + currentPermissionProblem = isGranted ? nil : .microphoneAccessDenied + return isGranted + case .denied, .restricted: + currentPermissionProblem = .microphoneAccessDenied + return false + @unknown default: + currentPermissionProblem = .microphoneAccessDenied + return false + } + } + + private func requestSpeechRecognitionPermissionIfNeeded() async -> Bool { + switch SFSpeechRecognizer.authorizationStatus() { + case .authorized: + currentPermissionProblem = nil + return true + case .notDetermined: + let isGranted = await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { authorizationStatus in + continuation.resume(returning: authorizationStatus == .authorized) + } + } + currentPermissionProblem = isGranted ? nil : .speechRecognitionDenied + return isGranted + case .denied, .restricted: + currentPermissionProblem = .speechRecognitionDenied + return false + @unknown default: + currentPermissionProblem = .speechRecognitionDenied + return false + } + } + + func openRelevantPrivacySettings() { + let settingsURLString: String + + switch currentPermissionProblem { + case .microphoneAccessDenied: + settingsURLString = "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" + case .speechRecognitionDenied: + settingsURLString = "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition" + case nil: + settingsURLString = "x-apple.systempreferences:com.apple.preference.security" + } + + guard let settingsURL = URL(string: settingsURLString) else { return } + NSWorkspace.shared.open(settingsURL) + } + + private func userFacingErrorMessage(from error: Error, fallback: String) -> String { + if let localizedError = error as? LocalizedError, + let errorDescription = localizedError.errorDescription? + .trimmingCharacters(in: .whitespacesAndNewlines), + !errorDescription.isEmpty { + return errorDescription + } + + let errorDescription = error.localizedDescription.trimmingCharacters(in: .whitespacesAndNewlines) + if !errorDescription.isEmpty, + errorDescription != "The operation couldn’t be completed." { + return errorDescription + } + + return fallback + } +} diff --git a/leanring-buddy/BuddyTranscriptionProvider.swift b/leanring-buddy/BuddyTranscriptionProvider.swift new file mode 100644 index 0000000..0a75715 --- /dev/null +++ b/leanring-buddy/BuddyTranscriptionProvider.swift @@ -0,0 +1,100 @@ +// +// BuddyTranscriptionProvider.swift +// leanring-buddy +// +// Shared protocol surface for voice transcription backends. +// + +import AVFoundation +import Foundation + +protocol BuddyStreamingTranscriptionSession: AnyObject { + var finalTranscriptFallbackDelaySeconds: TimeInterval { get } + func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) + func requestFinalTranscript() + func cancel() +} + +protocol BuddyTranscriptionProvider { + var displayName: String { get } + var requiresSpeechRecognitionPermission: Bool { get } + var isConfigured: Bool { get } + var unavailableExplanation: String? { get } + + func startStreamingSession( + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) async throws -> any BuddyStreamingTranscriptionSession +} + +enum BuddyTranscriptionProviderFactory { + private enum PreferredProvider: String { + case assemblyAI = "assemblyai" + case openAI = "openai" + case appleSpeech = "apple" + } + + static func makeDefaultProvider() -> any BuddyTranscriptionProvider { + let provider = resolveProvider() + print("🎙️ Transcription: using \(provider.displayName)") + return provider + } + + private static func resolveProvider() -> any BuddyTranscriptionProvider { + let preferredProviderRawValue = AppBundleConfiguration + .stringValue(forKey: "VoiceTranscriptionProvider")? + .lowercased() + let preferredProvider = preferredProviderRawValue.flatMap(PreferredProvider.init(rawValue:)) + + let assemblyAIProvider = AssemblyAIStreamingTranscriptionProvider() + let openAIProvider = OpenAIAudioTranscriptionProvider() + + if preferredProvider == .appleSpeech { + return AppleSpeechTranscriptionProvider() + } + + if preferredProvider == .assemblyAI { + if assemblyAIProvider.isConfigured { + return assemblyAIProvider + } + + print("⚠️ Transcription: AssemblyAI preferred but not configured, falling back") + + if openAIProvider.isConfigured { + print("⚠️ Transcription: using OpenAI as fallback") + return openAIProvider + } + + print("⚠️ Transcription: using Apple Speech as fallback") + return AppleSpeechTranscriptionProvider() + } + + if preferredProvider == .openAI { + if openAIProvider.isConfigured { + return openAIProvider + } + + print("⚠️ Transcription: OpenAI preferred but not configured, falling back") + + if assemblyAIProvider.isConfigured { + print("⚠️ Transcription: using AssemblyAI as fallback") + return assemblyAIProvider + } + + print("⚠️ Transcription: using Apple Speech as fallback") + return AppleSpeechTranscriptionProvider() + } + + if assemblyAIProvider.isConfigured { + return assemblyAIProvider + } + + if openAIProvider.isConfigured { + return openAIProvider + } + + return AppleSpeechTranscriptionProvider() + } +} diff --git a/leanring-buddy/ClaudeAPI.swift b/leanring-buddy/ClaudeAPI.swift new file mode 100644 index 0000000..0c7070b --- /dev/null +++ b/leanring-buddy/ClaudeAPI.swift @@ -0,0 +1,291 @@ +// +// ClaudeAPI.swift +// Claude API Implementation with streaming support +// + +import Foundation + +/// Claude API helper with streaming for progressive text display. +class ClaudeAPI { + private static let tlsWarmupLock = NSLock() + private static var hasStartedTLSWarmup = false + + private let apiURL: URL + var model: String + private let session: URLSession + + init(proxyURL: String, model: String = "claude-sonnet-4-6") { + self.apiURL = URL(string: proxyURL)! + self.model = model + + // Use .default instead of .ephemeral so TLS session tickets are cached. + // Ephemeral sessions do a full TLS handshake on every request, which causes + // transient -1200 (errSSLPeerHandshakeFail) errors with large image payloads. + // Disable URL/cookie caching to avoid storing responses or credentials on disk. + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 120 + config.timeoutIntervalForResource = 300 + config.waitsForConnectivity = true + config.urlCache = nil + config.httpCookieStorage = nil + self.session = URLSession(configuration: config) + + // Fire a lightweight HEAD request in the background to pre-establish the TLS + // connection. This caches the TLS session ticket so the first real API call + // (which carries a large image payload) doesn't need a cold TLS handshake. + warmUpTLSConnectionIfNeeded() + } + + private func makeAPIRequest() -> URLRequest { + var request = URLRequest(url: apiURL) + request.httpMethod = "POST" + request.timeoutInterval = 120 + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + return request + } + + /// Detects the MIME type of image data by inspecting the first bytes. + /// Screen captures from ScreenCaptureKit are JPEG, but pasted images from the + /// clipboard are PNG. The API rejects requests where the declared media_type + /// doesn't match the actual image format. + private func detectImageMediaType(for imageData: Data) -> String { + // PNG files start with the 8-byte signature: 89 50 4E 47 0D 0A 1A 0A + if imageData.count >= 4 { + let pngSignature: [UInt8] = [0x89, 0x50, 0x4E, 0x47] + let firstFourBytes = [UInt8](imageData.prefix(4)) + if firstFourBytes == pngSignature { + return "image/png" + } + } + // Default to JPEG — screen captures use JPEG compression + return "image/jpeg" + } + + /// Sends a no-op HEAD request to the API host to establish and cache a TLS session. + /// Failures are silently ignored — this is purely an optimization. + private func warmUpTLSConnectionIfNeeded() { + Self.tlsWarmupLock.lock() + let shouldStartTLSWarmup = !Self.hasStartedTLSWarmup + if shouldStartTLSWarmup { + Self.hasStartedTLSWarmup = true + } + Self.tlsWarmupLock.unlock() + + guard shouldStartTLSWarmup else { return } + + guard var warmupURLComponents = URLComponents(url: apiURL, resolvingAgainstBaseURL: false) else { + return + } + + // The TLS session ticket is host-scoped, so warming the root host is enough. + // Hitting the host instead of `/v1/messages` avoids extra endpoint-specific noise. + warmupURLComponents.path = "/" + warmupURLComponents.query = nil + warmupURLComponents.fragment = nil + + guard let warmupURL = warmupURLComponents.url else { + return + } + + var warmupRequest = URLRequest(url: warmupURL) + warmupRequest.httpMethod = "HEAD" + warmupRequest.timeoutInterval = 10 + session.dataTask(with: warmupRequest) { _, _, _ in + // Response doesn't matter — the TLS handshake is the goal + }.resume() + } + + /// Send a vision request to Claude with streaming. + /// Calls `onTextChunk` on the main actor each time new text arrives so the UI updates progressively. + /// Returns the full accumulated text and total duration when the stream completes. + func analyzeImageStreaming( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userPlaceholder: String, assistantResponse: String)] = [], + userPrompt: String, + onTextChunk: @MainActor @Sendable (String) -> Void + ) async throws -> (text: String, duration: TimeInterval) { + let startTime = Date() + + var request = makeAPIRequest() + + // Build messages array + var messages: [[String: Any]] = [] + + for (userPlaceholder, assistantResponse) in conversationHistory { + messages.append(["role": "user", "content": userPlaceholder]) + messages.append(["role": "assistant", "content": assistantResponse]) + } + + // Build current message with all labeled images + prompt + var contentBlocks: [[String: Any]] = [] + for image in images { + contentBlocks.append([ + "type": "image", + "source": [ + "type": "base64", + "media_type": detectImageMediaType(for: image.data), + "data": image.data.base64EncodedString() + ] + ]) + contentBlocks.append([ + "type": "text", + "text": image.label + ]) + } + contentBlocks.append([ + "type": "text", + "text": userPrompt + ]) + messages.append(["role": "user", "content": contentBlocks]) + + let body: [String: Any] = [ + "model": model, + "max_tokens": 1024, + "stream": true, + "system": systemPrompt, + "messages": messages + ] + + let bodyData = try JSONSerialization.data(withJSONObject: body) + request.httpBody = bodyData + let payloadMB = Double(bodyData.count) / 1_048_576.0 + print("🌐 Claude streaming request: \(String(format: "%.1f", payloadMB))MB, \(images.count) image(s)") + + // Use bytes streaming for SSE (Server-Sent Events) + let (byteStream, response) = try await session.bytes(for: request) + + guard let httpResponse = response as? HTTPURLResponse else { + throw NSError( + domain: "ClaudeAPI", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid HTTP response"] + ) + } + + // If non-2xx status, read the full body as error text + guard (200...299).contains(httpResponse.statusCode) else { + var errorBodyChunks: [String] = [] + for try await line in byteStream.lines { + errorBodyChunks.append(line) + } + let errorBody = errorBodyChunks.joined(separator: "\n") + throw NSError( + domain: "ClaudeAPI", + code: httpResponse.statusCode, + userInfo: [NSLocalizedDescriptionKey: "API Error (\(httpResponse.statusCode)): \(errorBody)"] + ) + } + + // Parse SSE stream — each event is "data: {json}\n\n" + var accumulatedResponseText = "" + + for try await line in byteStream.lines { + // SSE lines look like: "data: {...}" + guard line.hasPrefix("data: ") else { continue } + let jsonString = String(line.dropFirst(6)) // Drop "data: " prefix + + // End of stream marker + guard jsonString != "[DONE]" else { break } + + guard let jsonData = jsonString.data(using: .utf8), + let eventPayload = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any], + let eventType = eventPayload["type"] as? String else { + continue + } + + // We care about content_block_delta events that contain text chunks + if eventType == "content_block_delta", + let delta = eventPayload["delta"] as? [String: Any], + let deltaType = delta["type"] as? String, + deltaType == "text_delta", + let textChunk = delta["text"] as? String { + accumulatedResponseText += textChunk + // Send the accumulated text so far to the UI for progressive rendering + let currentAccumulatedText = accumulatedResponseText + await onTextChunk(currentAccumulatedText) + } + } + + let duration = Date().timeIntervalSince(startTime) + return (text: accumulatedResponseText, duration: duration) + } + + /// Non-streaming fallback for validation requests where we don't need progressive display. + func analyzeImage( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userPlaceholder: String, assistantResponse: String)] = [], + userPrompt: String + ) async throws -> (text: String, duration: TimeInterval) { + let startTime = Date() + + var request = makeAPIRequest() + + var messages: [[String: Any]] = [] + for (userPlaceholder, assistantResponse) in conversationHistory { + messages.append(["role": "user", "content": userPlaceholder]) + messages.append(["role": "assistant", "content": assistantResponse]) + } + + // Build current message with all labeled images + prompt + var contentBlocks: [[String: Any]] = [] + for image in images { + contentBlocks.append([ + "type": "image", + "source": [ + "type": "base64", + "media_type": detectImageMediaType(for: image.data), + "data": image.data.base64EncodedString() + ] + ]) + contentBlocks.append([ + "type": "text", + "text": image.label + ]) + } + contentBlocks.append([ + "type": "text", + "text": userPrompt + ]) + messages.append(["role": "user", "content": contentBlocks]) + + let body: [String: Any] = [ + "model": model, + "max_tokens": 256, + "system": systemPrompt, + "messages": messages + ] + + let bodyData = try JSONSerialization.data(withJSONObject: body) + request.httpBody = bodyData + let payloadMB = Double(bodyData.count) / 1_048_576.0 + print("🌐 Claude request: \(String(format: "%.1f", payloadMB))MB, \(images.count) image(s)") + + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let responseString = String(data: data, encoding: .utf8) ?? "Unknown error" + throw NSError( + domain: "ClaudeAPI", + code: (response as? HTTPURLResponse)?.statusCode ?? -1, + userInfo: [NSLocalizedDescriptionKey: "API Error: \(responseString)"] + ) + } + + let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] + guard let content = json?["content"] as? [[String: Any]], + let textBlock = content.first(where: { ($0["type"] as? String) == "text" }), + let text = textBlock["text"] as? String else { + throw NSError( + domain: "ClaudeAPI", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid response format"] + ) + } + + let duration = Date().timeIntervalSince(startTime) + return (text: text, duration: duration) + } +} diff --git a/leanring-buddy/ClickyAnalytics.swift b/leanring-buddy/ClickyAnalytics.swift new file mode 100644 index 0000000..29e2613 --- /dev/null +++ b/leanring-buddy/ClickyAnalytics.swift @@ -0,0 +1,121 @@ +// +// ClickyAnalytics.swift +// leanring-buddy +// +// Centralized PostHog analytics wrapper. All event names and properties +// are defined here so instrumentation is consistent and easy to audit. +// + +import Foundation +import PostHog + +enum ClickyAnalytics { + + // MARK: - Setup + + static func configure() { + let config = PostHogConfig( + apiKey: "phc_xcQPygmhTMzzYh8wNW92CCwoXmnzqyChAixh8zgpqC3C", + host: "https://us.i.posthog.com" + ) + PostHogSDK.shared.setup(config) + } + + // MARK: - App Lifecycle + + /// Fired once on every app launch in applicationDidFinishLaunching. + static func trackAppOpened() { + let version = Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown" + PostHogSDK.shared.capture("app_opened", properties: [ + "app_version": version + ]) + } + + // MARK: - Onboarding + + /// User clicked the Start button to begin onboarding for the first time. + static func trackOnboardingStarted() { + PostHogSDK.shared.capture("onboarding_started") + } + + /// User clicked "Watch Onboarding Again" from the panel footer. + static func trackOnboardingReplayed() { + PostHogSDK.shared.capture("onboarding_replayed") + } + + /// The onboarding video finished playing to the end. + static func trackOnboardingVideoCompleted() { + PostHogSDK.shared.capture("onboarding_video_completed") + } + + /// The 40s onboarding demo interaction where Clicky points at something. + static func trackOnboardingDemoTriggered() { + PostHogSDK.shared.capture("onboarding_demo_triggered") + } + + // MARK: - Permissions + + /// All three permissions (accessibility, screen recording, mic) are granted. + static func trackAllPermissionsGranted() { + PostHogSDK.shared.capture("all_permissions_granted") + } + + /// A single permission was granted. Called when polling detects a change. + static func trackPermissionGranted(permission: String) { + PostHogSDK.shared.capture("permission_granted", properties: [ + "permission": permission + ]) + } + + // MARK: - Voice Interaction + + /// User pressed the push-to-talk shortcut (control+option) to start talking. + static func trackPushToTalkStarted() { + PostHogSDK.shared.capture("push_to_talk_started") + } + + /// User released the shortcut — transcript is being finalized. + static func trackPushToTalkReleased() { + PostHogSDK.shared.capture("push_to_talk_released") + } + + /// Transcription completed and the user's message is being sent to the AI. + static func trackUserMessageSent(transcript: String) { + PostHogSDK.shared.capture("user_message_sent", properties: [ + "transcript": transcript, + "character_count": transcript.count + ]) + } + + /// Claude responded and the response is being spoken via TTS. + static func trackAIResponseReceived(response: String) { + PostHogSDK.shared.capture("ai_response_received", properties: [ + "response": response, + "character_count": response.count + ]) + } + + /// Claude's response included a [POINT:x,y:label] coordinate tag, + /// so the buddy is flying to point at a UI element. + static func trackElementPointed(elementLabel: String?) { + PostHogSDK.shared.capture("element_pointed", properties: [ + "element_label": elementLabel ?? "unknown" + ]) + } + + // MARK: - Errors + + /// An error occurred during the AI response pipeline. + static func trackResponseError(error: String) { + PostHogSDK.shared.capture("response_error", properties: [ + "error": error + ]) + } + + /// An error occurred during TTS playback. + static func trackTTSError(error: String) { + PostHogSDK.shared.capture("tts_error", properties: [ + "error": error + ]) + } +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift new file mode 100644 index 0000000..0234cf1 --- /dev/null +++ b/leanring-buddy/CompanionManager.swift @@ -0,0 +1,1026 @@ +// +// CompanionManager.swift +// leanring-buddy +// +// Central state manager for the companion voice mode. Owns the push-to-talk +// pipeline (dictation manager + global shortcut monitor + overlay) and +// exposes observable voice state for the panel UI. +// + +import AVFoundation +import Combine +import Foundation +import PostHog +import ScreenCaptureKit +import SwiftUI + +enum CompanionVoiceState { + case idle + case listening + case processing + case responding +} + +@MainActor +final class CompanionManager: ObservableObject { + @Published private(set) var voiceState: CompanionVoiceState = .idle + @Published private(set) var lastTranscript: String? + @Published private(set) var currentAudioPowerLevel: CGFloat = 0 + @Published private(set) var hasAccessibilityPermission = false + @Published private(set) var hasScreenRecordingPermission = false + @Published private(set) var hasMicrophonePermission = false + @Published private(set) var hasScreenContentPermission = false + + /// Screen location (global AppKit coords) of a detected UI element the + /// buddy should fly to and point at. Parsed from Claude's response; + /// observed by BlueCursorView to trigger the flight animation. + @Published var detectedElementScreenLocation: CGPoint? + /// The display frame (global AppKit coords) of the screen the detected + /// element is on, so BlueCursorView knows which screen overlay should animate. + @Published var detectedElementDisplayFrame: CGRect? + /// Custom speech bubble text for the pointing animation. When set, + /// BlueCursorView uses this instead of a random pointer phrase. + @Published var detectedElementBubbleText: String? + + // MARK: - Onboarding Video State (shared across all screen overlays) + + @Published var onboardingVideoPlayer: AVPlayer? + @Published var showOnboardingVideo: Bool = false + @Published var onboardingVideoOpacity: Double = 0.0 + private var onboardingVideoEndObserver: NSObjectProtocol? + private var onboardingDemoTimeObserver: Any? + + // MARK: - Onboarding Prompt Bubble + + /// Text streamed character-by-character on the cursor after the onboarding video ends. + @Published var onboardingPromptText: String = "" + @Published var onboardingPromptOpacity: Double = 0.0 + @Published var showOnboardingPrompt: Bool = false + + // MARK: - Onboarding Music + + private var onboardingMusicPlayer: AVAudioPlayer? + private var onboardingMusicFadeTimer: Timer? + + let buddyDictationManager = BuddyDictationManager() + let globalPushToTalkShortcutMonitor = GlobalPushToTalkShortcutMonitor() + let overlayWindowManager = OverlayWindowManager() + // Response text is now displayed inline on the cursor overlay via + // streamingResponseText, so no separate response overlay manager is needed. + + /// Base URL for the Cloudflare Worker proxy. All API requests route + /// through this so keys never ship in the app binary. + private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + + private lazy var claudeAPI: ClaudeAPI = { + return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) + }() + + private lazy var elevenLabsTTSClient: ElevenLabsTTSClient = { + return ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts") + }() + + /// Conversation history so Claude remembers prior exchanges within a session. + /// Each entry is the user's transcript and Claude's response. + private var conversationHistory: [(userTranscript: String, assistantResponse: String)] = [] + + /// The currently running AI response task, if any. Cancelled when the user + /// speaks again so a new response can begin immediately. + private var currentResponseTask: Task? + + private var shortcutTransitionCancellable: AnyCancellable? + private var voiceStateCancellable: AnyCancellable? + private var audioPowerCancellable: AnyCancellable? + private var accessibilityCheckTimer: Timer? + private var pendingKeyboardShortcutStartTask: Task? + /// Scheduled hide for transient cursor mode — cancelled if the user + /// speaks again before the delay elapses. + private var transientHideTask: Task? + + /// True when all three required permissions (accessibility, screen recording, + /// microphone) are granted. Used by the panel to show a single "all good" state. + var allPermissionsGranted: Bool { + hasAccessibilityPermission && hasScreenRecordingPermission && hasMicrophonePermission && hasScreenContentPermission + } + + /// Whether the blue cursor overlay is currently visible on screen. + /// Used by the panel to show accurate status text ("Active" vs "Ready"). + @Published private(set) var isOverlayVisible: Bool = false + + /// The Claude model used for voice responses. Persisted to UserDefaults. + @Published var selectedModel: String = UserDefaults.standard.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + + func setSelectedModel(_ model: String) { + selectedModel = model + UserDefaults.standard.set(model, forKey: "selectedClaudeModel") + claudeAPI.model = model + } + + /// User preference for whether the Clicky cursor should be shown. + /// When toggled off, the overlay is hidden and push-to-talk is disabled. + /// Persisted to UserDefaults so the choice survives app restarts. + @Published var isClickyCursorEnabled: Bool = UserDefaults.standard.object(forKey: "isClickyCursorEnabled") == nil + ? true + : UserDefaults.standard.bool(forKey: "isClickyCursorEnabled") + + func setClickyCursorEnabled(_ enabled: Bool) { + isClickyCursorEnabled = enabled + UserDefaults.standard.set(enabled, forKey: "isClickyCursorEnabled") + transientHideTask?.cancel() + transientHideTask = nil + + if enabled { + overlayWindowManager.hasShownOverlayBefore = true + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } else { + overlayWindowManager.hideOverlay() + isOverlayVisible = false + } + } + + /// Whether the user has completed onboarding at least once. Persisted + /// to UserDefaults so the Start button only appears on first launch. + var hasCompletedOnboarding: Bool { + get { UserDefaults.standard.bool(forKey: "hasCompletedOnboarding") } + set { UserDefaults.standard.set(newValue, forKey: "hasCompletedOnboarding") } + } + + /// Whether the user has submitted their email during onboarding. + @Published var hasSubmittedEmail: Bool = UserDefaults.standard.bool(forKey: "hasSubmittedEmail") + + /// Submits the user's email to FormSpark and identifies them in PostHog. + func submitEmail(_ email: String) { + let trimmedEmail = email.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedEmail.isEmpty else { return } + + hasSubmittedEmail = true + UserDefaults.standard.set(true, forKey: "hasSubmittedEmail") + + // Identify user in PostHog + PostHogSDK.shared.identify(trimmedEmail, userProperties: [ + "email": trimmedEmail + ]) + + // Submit to FormSpark + Task { + var request = URLRequest(url: URL(string: "https://submit-form.com/RWbGJxmIs")!) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try? JSONSerialization.data(withJSONObject: ["email": trimmedEmail]) + _ = try? await URLSession.shared.data(for: request) + } + } + + func start() { + refreshAllPermissions() + print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") + startPermissionPolling() + bindVoiceStateObservation() + bindAudioPowerLevel() + bindShortcutTransitions() + // Eagerly touch the Claude API so its TLS warmup handshake completes + // well before the onboarding demo fires at ~40s into the video. + _ = claudeAPI + + // If the user already completed onboarding AND all permissions are + // still granted, show the cursor overlay immediately. If permissions + // were revoked (e.g. signing change), don't show the cursor — the + // panel will show the permissions UI instead. + if hasCompletedOnboarding && allPermissionsGranted && isClickyCursorEnabled { + overlayWindowManager.hasShownOverlayBefore = true + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + } + + /// Called by BlueCursorView after the buddy finishes its pointing + /// animation and returns to cursor-following mode. + /// Triggers the onboarding sequence — dismisses the panel and restarts + /// the overlay so the welcome animation and intro video play. + func triggerOnboarding() { + // Post notification so the panel manager can dismiss the panel + NotificationCenter.default.post(name: .clickyDismissPanel, object: nil) + + // Mark onboarding as completed so the Start button won't appear + // again on future launches — the cursor will auto-show instead + hasCompletedOnboarding = true + + ClickyAnalytics.trackOnboardingStarted() + + // Play Besaid theme at 60% volume, fade out after 1m 30s + startOnboardingMusic() + + // Show the overlay for the first time — isFirstAppearance triggers + // the welcome animation and onboarding video + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + + /// Replays the onboarding experience from the "Watch Onboarding Again" + /// footer link. Same flow as triggerOnboarding but the cursor overlay + /// is already visible so we just restart the welcome animation and video. + func replayOnboarding() { + NotificationCenter.default.post(name: .clickyDismissPanel, object: nil) + ClickyAnalytics.trackOnboardingReplayed() + startOnboardingMusic() + // Tear down any existing overlays and recreate with isFirstAppearance = true + overlayWindowManager.hasShownOverlayBefore = false + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + + private func stopOnboardingMusic() { + onboardingMusicFadeTimer?.invalidate() + onboardingMusicFadeTimer = nil + onboardingMusicPlayer?.stop() + onboardingMusicPlayer = nil + } + + private func startOnboardingMusic() { + stopOnboardingMusic() + guard let musicURL = Bundle.main.url(forResource: "ff", withExtension: "mp3") else { + print("⚠️ Clicky: ff.mp3 not found in bundle") + return + } + + do { + let player = try AVAudioPlayer(contentsOf: musicURL) + player.volume = 0.3 + player.play() + self.onboardingMusicPlayer = player + + // After 1m 30s, fade the music out over 3s + onboardingMusicFadeTimer = Timer.scheduledTimer(withTimeInterval: 90.0, repeats: false) { [weak self] _ in + self?.fadeOutOnboardingMusic() + } + } catch { + print("⚠️ Clicky: Failed to play onboarding music: \(error)") + } + } + + private func fadeOutOnboardingMusic() { + guard let player = onboardingMusicPlayer else { return } + + let fadeSteps = 30 + let fadeDuration: Double = 3.0 + let stepInterval = fadeDuration / Double(fadeSteps) + let volumeDecrement = player.volume / Float(fadeSteps) + var stepsRemaining = fadeSteps + + onboardingMusicFadeTimer = Timer.scheduledTimer(withTimeInterval: stepInterval, repeats: true) { [weak self] timer in + stepsRemaining -= 1 + player.volume -= volumeDecrement + + if stepsRemaining <= 0 { + timer.invalidate() + player.stop() + self?.onboardingMusicPlayer = nil + self?.onboardingMusicFadeTimer = nil + } + } + } + + func clearDetectedElementLocation() { + detectedElementScreenLocation = nil + detectedElementDisplayFrame = nil + detectedElementBubbleText = nil + } + + func stop() { + globalPushToTalkShortcutMonitor.stop() + buddyDictationManager.cancelCurrentDictation() + overlayWindowManager.hideOverlay() + transientHideTask?.cancel() + + currentResponseTask?.cancel() + currentResponseTask = nil + shortcutTransitionCancellable?.cancel() + voiceStateCancellable?.cancel() + audioPowerCancellable?.cancel() + accessibilityCheckTimer?.invalidate() + accessibilityCheckTimer = nil + } + + func refreshAllPermissions() { + let previouslyHadAccessibility = hasAccessibilityPermission + let previouslyHadScreenRecording = hasScreenRecordingPermission + let previouslyHadMicrophone = hasMicrophonePermission + let previouslyHadAll = allPermissionsGranted + + let currentlyHasAccessibility = WindowPositionManager.hasAccessibilityPermission() + hasAccessibilityPermission = currentlyHasAccessibility + + if currentlyHasAccessibility { + globalPushToTalkShortcutMonitor.start() + } else { + globalPushToTalkShortcutMonitor.stop() + } + + hasScreenRecordingPermission = WindowPositionManager.hasScreenRecordingPermission() + + let micAuthStatus = AVCaptureDevice.authorizationStatus(for: .audio) + hasMicrophonePermission = micAuthStatus == .authorized + + // Debug: log permission state on changes + if previouslyHadAccessibility != hasAccessibilityPermission + || previouslyHadScreenRecording != hasScreenRecordingPermission + || previouslyHadMicrophone != hasMicrophonePermission { + print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") + } + + // Track individual permission grants as they happen + if !previouslyHadAccessibility && hasAccessibilityPermission { + ClickyAnalytics.trackPermissionGranted(permission: "accessibility") + } + if !previouslyHadScreenRecording && hasScreenRecordingPermission { + ClickyAnalytics.trackPermissionGranted(permission: "screen_recording") + } + if !previouslyHadMicrophone && hasMicrophonePermission { + ClickyAnalytics.trackPermissionGranted(permission: "microphone") + } + // Screen content permission is persisted — once the user has approved the + // SCShareableContent picker, we don't need to re-check it. + if !hasScreenContentPermission { + hasScreenContentPermission = UserDefaults.standard.bool(forKey: "hasScreenContentPermission") + } + + if !previouslyHadAll && allPermissionsGranted { + ClickyAnalytics.trackAllPermissionsGranted() + } + } + + /// Triggers the macOS screen content picker by performing a dummy + /// screenshot capture. Once the user approves, we persist the grant + /// so they're never asked again during onboarding. + @Published private(set) var isRequestingScreenContent = false + + func requestScreenContentPermission() { + guard !isRequestingScreenContent else { return } + isRequestingScreenContent = true + Task { + do { + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + guard let display = content.displays.first else { + await MainActor.run { isRequestingScreenContent = false } + return + } + let filter = SCContentFilter(display: display, excludingWindows: []) + let config = SCStreamConfiguration() + config.width = 320 + config.height = 240 + let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + // Verify the capture actually returned real content — a 0x0 or + // fully-empty image means the user denied the prompt. + let didCapture = image.width > 0 && image.height > 0 + print("🔑 Screen content capture result — width: \(image.width), height: \(image.height), didCapture: \(didCapture)") + await MainActor.run { + isRequestingScreenContent = false + guard didCapture else { return } + hasScreenContentPermission = true + UserDefaults.standard.set(true, forKey: "hasScreenContentPermission") + ClickyAnalytics.trackPermissionGranted(permission: "screen_content") + + // If onboarding was already completed, show the cursor overlay now + if hasCompletedOnboarding && allPermissionsGranted && !isOverlayVisible && isClickyCursorEnabled { + overlayWindowManager.hasShownOverlayBefore = true + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + } + } catch { + print("⚠️ Screen content permission request failed: \(error)") + await MainActor.run { isRequestingScreenContent = false } + } + } + } + + // MARK: - Private + + /// Triggers the system microphone prompt if the user has never been asked. + /// Once granted/denied the status sticks and polling picks it up. + private func promptForMicrophoneIfNotDetermined() { + guard AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined else { return } + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + Task { @MainActor [weak self] in + self?.hasMicrophonePermission = granted + } + } + } + + /// Polls all permissions frequently so the UI updates live after the + /// user grants them in System Settings. Screen Recording is the exception — + /// macOS requires an app restart for that one to take effect. + private func startPermissionPolling() { + accessibilityCheckTimer = Timer.scheduledTimer(withTimeInterval: 1.5, repeats: true) { [weak self] _ in + Task { @MainActor [weak self] in + self?.refreshAllPermissions() + } + } + } + + private func bindAudioPowerLevel() { + audioPowerCancellable = buddyDictationManager.$currentAudioPowerLevel + .receive(on: DispatchQueue.main) + .sink { [weak self] powerLevel in + self?.currentAudioPowerLevel = powerLevel + } + } + + private func bindVoiceStateObservation() { + voiceStateCancellable = buddyDictationManager.$isRecordingFromKeyboardShortcut + .combineLatest( + buddyDictationManager.$isFinalizingTranscript, + buddyDictationManager.$isPreparingToRecord + ) + .receive(on: DispatchQueue.main) + .sink { [weak self] isRecording, isFinalizing, isPreparing in + guard let self else { return } + // Don't override .responding — the AI response pipeline + // manages that state directly until streaming finishes. + guard self.voiceState != .responding else { return } + + if isFinalizing { + self.voiceState = .processing + } else if isRecording { + self.voiceState = .listening + } else if isPreparing { + self.voiceState = .processing + } else { + self.voiceState = .idle + // If the user pressed and released the hotkey without + // saying anything, no response task runs — schedule the + // transient hide here so the overlay doesn't get stuck. + // Only do this when no response is in flight, otherwise + // the brief idle gap between recording and processing + // would prematurely hide the overlay. + if self.currentResponseTask == nil { + self.scheduleTransientHideIfNeeded() + } + } + } + } + + private func bindShortcutTransitions() { + shortcutTransitionCancellable = globalPushToTalkShortcutMonitor + .shortcutTransitionPublisher + .receive(on: DispatchQueue.main) + .sink { [weak self] transition in + self?.handleShortcutTransition(transition) + } + } + + private func handleShortcutTransition(_ transition: BuddyPushToTalkShortcut.ShortcutTransition) { + switch transition { + case .pressed: + guard !buddyDictationManager.isDictationInProgress else { return } + // Don't register push-to-talk while the onboarding video is playing + guard !showOnboardingVideo else { return } + + // Cancel any pending transient hide so the overlay stays visible + transientHideTask?.cancel() + transientHideTask = nil + + // If the cursor is hidden, bring it back transiently for this interaction + if !isClickyCursorEnabled && !isOverlayVisible { + overlayWindowManager.hasShownOverlayBefore = true + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + + // Dismiss the menu bar panel so it doesn't cover the screen + NotificationCenter.default.post(name: .clickyDismissPanel, object: nil) + + // Cancel any in-progress response and TTS from a previous utterance + currentResponseTask?.cancel() + elevenLabsTTSClient.stopPlayback() + clearDetectedElementLocation() + + // Dismiss the onboarding prompt if it's showing + if showOnboardingPrompt { + withAnimation(.easeOut(duration: 0.3)) { + onboardingPromptOpacity = 0.0 + } + DispatchQueue.main.asyncAfter(deadline: .now() + 0.35) { + self.showOnboardingPrompt = false + self.onboardingPromptText = "" + } + } + + + ClickyAnalytics.trackPushToTalkStarted() + + pendingKeyboardShortcutStartTask?.cancel() + pendingKeyboardShortcutStartTask = Task { + await buddyDictationManager.startPushToTalkFromKeyboardShortcut( + currentDraftText: "", + updateDraftText: { _ in + // Partial transcripts are hidden (waveform-only UI) + }, + submitDraftText: { [weak self] finalTranscript in + self?.lastTranscript = finalTranscript + print("🗣️ Companion received transcript: \(finalTranscript)") + ClickyAnalytics.trackUserMessageSent(transcript: finalTranscript) + self?.sendTranscriptToClaudeWithScreenshot(transcript: finalTranscript) + } + ) + } + case .released: + // Cancel the pending start task in case the user released the shortcut + // before the async startPushToTalk had a chance to begin recording. + // Without this, a quick press-and-release drops the release event and + // leaves the waveform overlay stuck on screen indefinitely. + ClickyAnalytics.trackPushToTalkReleased() + pendingKeyboardShortcutStartTask?.cancel() + pendingKeyboardShortcutStartTask = nil + buddyDictationManager.stopPushToTalkFromKeyboardShortcut() + case .none: + break + } + } + + // MARK: - Companion Prompt + + private static let companionVoiceResponseSystemPrompt = """ + you're clicky, a friendly always-on companion that lives in the user's menu bar. the user just spoke to you via push-to-talk and you can see their screen(s). your reply will be spoken aloud via text-to-speech, so write the way you'd actually talk. this is an ongoing conversation — you remember everything they've said before. + + rules: + - default to one or two sentences. be direct and dense. BUT if the user asks you to explain more, go deeper, or elaborate, then go all out — give a thorough, detailed explanation with no length limit. + - all lowercase, casual, warm. no emojis. + - write for the ear, not the eye. short sentences. no lists, bullet points, markdown, or formatting — just natural speech. + - don't use abbreviations or symbols that sound weird read aloud. write "for example" not "e.g.", spell out small numbers. + - if the user's question relates to what's on their screen, reference specific things you see. + - if the screenshot doesn't seem relevant to their question, just answer the question directly. + - you can help with anything — coding, writing, general knowledge, brainstorming. + - never say "simply" or "just". + - don't read out code verbatim. describe what the code does or what needs to change conversationally. + - focus on giving a thorough, useful explanation. don't end with simple yes/no questions like "want me to explain more?" or "should i show you?" — those are dead ends that force the user to just say yes. + - instead, when it fits naturally, end by planting a seed — mention something bigger or more ambitious they could try, a related concept that goes deeper, or a next-level technique that builds on what you just explained. make it something worth coming back for, not a question they'd just nod to. it's okay to not end with anything extra if the answer is complete on its own. + - if you receive multiple screen images, the one labeled "primary focus" is where the cursor is — prioritize that one but reference others if relevant. + + element pointing: + you have a small blue triangle cursor that can fly to and point at things on screen. use it whenever pointing would genuinely help the user — if they're asking how to do something, looking for a menu, trying to find a button, or need help navigating an app, point at the relevant element. err on the side of pointing rather than not pointing, because it makes your help way more useful and concrete. + + don't point at things when it would be pointless — like if the user asks a general knowledge question, or the conversation has nothing to do with what's on screen, or you'd just be pointing at something obvious they're already looking at. but if there's a specific UI element, menu, button, or area on screen that's relevant to what you're helping with, point at it. + + when you point, append a coordinate tag at the very end of your response, AFTER your spoken text. the screenshot images are labeled with their pixel dimensions. use those dimensions as the coordinate space. the origin (0,0) is the top-left corner of the image. x increases rightward, y increases downward. + + format: [POINT:x,y:label] where x,y are integer pixel coordinates in the screenshot's coordinate space, and label is a short 1-3 word description of the element (like "search bar" or "save button"). if the element is on the cursor's screen you can omit the screen number. if the element is on a DIFFERENT screen, append :screenN where N is the screen number from the image label (e.g. :screen2). this is important — without the screen number, the cursor will point at the wrong place. + + if pointing wouldn't help, append [POINT:none]. + + examples: + - user asks how to color grade in final cut: "you'll want to open the color inspector — it's right up in the top right area of the toolbar. click that and you'll get all the color wheels and curves. [POINT:1100,42:color inspector]" + - user asks what html is: "html stands for hypertext markup language, it's basically the skeleton of every web page. curious how it connects to the css you're looking at? [POINT:none]" + - user asks how to commit in xcode: "see that source control menu up top? click that and hit commit, or you can use command option c as a shortcut. [POINT:285,11:source control]" + - element is on screen 2 (not where cursor is): "that's over on your other monitor — see the terminal window? [POINT:400,300:terminal:screen2]" + """ + + // MARK: - AI Response Pipeline + + /// Captures a screenshot, sends it along with the transcript to Claude, + /// and plays the response aloud via ElevenLabs TTS. The cursor stays in + /// the spinner/processing state until TTS audio begins playing. + /// Claude's response may include a [POINT:x,y:label] tag which triggers + /// the buddy to fly to that element on screen. + private func sendTranscriptToClaudeWithScreenshot(transcript: String) { + currentResponseTask?.cancel() + elevenLabsTTSClient.stopPlayback() + + currentResponseTask = Task { + // Stay in processing (spinner) state — no streaming text displayed + voiceState = .processing + + do { + // Capture all connected screens so the AI has full context + let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + + guard !Task.isCancelled else { return } + + // Build image labels with the actual screenshot pixel dimensions + // so Claude's coordinate space matches the image it sees. We + // scale from screenshot pixels to display points ourselves. + let labeledImages = screenCaptures.map { capture in + let dimensionInfo = " (image dimensions: \(capture.screenshotWidthInPixels)x\(capture.screenshotHeightInPixels) pixels)" + return (data: capture.imageData, label: capture.label + dimensionInfo) + } + + // Pass conversation history so Claude remembers prior exchanges + let historyForAPI = conversationHistory.map { entry in + (userPlaceholder: entry.userTranscript, assistantResponse: entry.assistantResponse) + } + + let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( + images: labeledImages, + systemPrompt: Self.companionVoiceResponseSystemPrompt, + conversationHistory: historyForAPI, + userPrompt: transcript, + onTextChunk: { _ in + // No streaming text display — spinner stays until TTS plays + } + ) + + guard !Task.isCancelled else { return } + + // Parse the [POINT:...] tag from Claude's response + let parseResult = Self.parsePointingCoordinates(from: fullResponseText) + let spokenText = parseResult.spokenText + + // Handle element pointing if Claude returned coordinates. + // Switch to idle BEFORE setting the location so the triangle + // becomes visible and can fly to the target. Without this, the + // spinner hides the triangle and the flight animation is invisible. + let hasPointCoordinate = parseResult.coordinate != nil + if hasPointCoordinate { + voiceState = .idle + } + + // Pick the screen capture matching Claude's screen number, + // falling back to the cursor screen if not specified. + let targetScreenCapture: CompanionScreenCapture? = { + if let screenNumber = parseResult.screenNumber, + screenNumber >= 1 && screenNumber <= screenCaptures.count { + return screenCaptures[screenNumber - 1] + } + return screenCaptures.first(where: { $0.isCursorScreen }) + }() + + if let pointCoordinate = parseResult.coordinate, + let targetScreenCapture { + // Claude's coordinates are in the screenshot's pixel space + // (top-left origin, e.g. 1280x831). Scale to the display's + // point space (e.g. 1512x982), then convert to AppKit global coords. + let screenshotWidth = CGFloat(targetScreenCapture.screenshotWidthInPixels) + let screenshotHeight = CGFloat(targetScreenCapture.screenshotHeightInPixels) + let displayWidth = CGFloat(targetScreenCapture.displayWidthInPoints) + let displayHeight = CGFloat(targetScreenCapture.displayHeightInPoints) + let displayFrame = targetScreenCapture.displayFrame + + // Clamp to screenshot coordinate space + let clampedX = max(0, min(pointCoordinate.x, screenshotWidth)) + let clampedY = max(0, min(pointCoordinate.y, screenshotHeight)) + + // Scale from screenshot pixels to display points + let displayLocalX = clampedX * (displayWidth / screenshotWidth) + let displayLocalY = clampedY * (displayHeight / screenshotHeight) + + // Convert from top-left origin (screenshot) to bottom-left origin (AppKit) + let appKitY = displayHeight - displayLocalY + + // Convert display-local coords to global screen coords + let globalLocation = CGPoint( + x: displayLocalX + displayFrame.origin.x, + y: appKitY + displayFrame.origin.y + ) + + detectedElementScreenLocation = globalLocation + detectedElementDisplayFrame = displayFrame + ClickyAnalytics.trackElementPointed(elementLabel: parseResult.elementLabel) + print("🎯 Element pointing: (\(Int(pointCoordinate.x)), \(Int(pointCoordinate.y))) → \"\(parseResult.elementLabel ?? "element")\"") + } else { + print("🎯 Element pointing: \(parseResult.elementLabel ?? "no element")") + } + + // Save this exchange to conversation history (with the point tag + // stripped so it doesn't confuse future context) + conversationHistory.append(( + userTranscript: transcript, + assistantResponse: spokenText + )) + + // Keep only the last 10 exchanges to avoid unbounded context growth + if conversationHistory.count > 10 { + conversationHistory.removeFirst(conversationHistory.count - 10) + } + + print("🧠 Conversation history: \(conversationHistory.count) exchanges") + + ClickyAnalytics.trackAIResponseReceived(response: spokenText) + + // Play the response via TTS. Keep the spinner (processing state) + // until the audio actually starts playing, then switch to responding. + if !spokenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + do { + try await elevenLabsTTSClient.speakText(spokenText) + // speakText returns after player.play() — audio is now playing + voiceState = .responding + } catch { + ClickyAnalytics.trackTTSError(error: error.localizedDescription) + print("⚠️ ElevenLabs TTS error: \(error)") + speakCreditsErrorFallback() + } + } + } catch is CancellationError { + // User spoke again — response was interrupted + } catch { + ClickyAnalytics.trackResponseError(error: error.localizedDescription) + print("⚠️ Companion response error: \(error)") + speakCreditsErrorFallback() + } + + if !Task.isCancelled { + voiceState = .idle + scheduleTransientHideIfNeeded() + } + } + } + + /// If the cursor is in transient mode (user toggled "Show Clicky" off), + /// waits for TTS playback and any pointing animation to finish, then + /// fades out the overlay after a 1-second pause. Cancelled automatically + /// if the user starts another push-to-talk interaction. + private func scheduleTransientHideIfNeeded() { + guard !isClickyCursorEnabled && isOverlayVisible else { return } + + transientHideTask?.cancel() + transientHideTask = Task { + // Wait for TTS audio to finish playing + while elevenLabsTTSClient.isPlaying { + try? await Task.sleep(nanoseconds: 200_000_000) + guard !Task.isCancelled else { return } + } + + // Wait for pointing animation to finish (location is cleared + // when the buddy flies back to the cursor) + while detectedElementScreenLocation != nil { + try? await Task.sleep(nanoseconds: 200_000_000) + guard !Task.isCancelled else { return } + } + + // Pause 1s after everything finishes, then fade out + try? await Task.sleep(nanoseconds: 1_000_000_000) + guard !Task.isCancelled else { return } + overlayWindowManager.fadeOutAndHideOverlay() + isOverlayVisible = false + } + } + + /// Speaks a hardcoded error message using macOS system TTS when API + /// credits run out. Uses NSSpeechSynthesizer so it works even when + /// ElevenLabs is down. + private func speakCreditsErrorFallback() { + let utterance = "I'm all out of credits. Please DM Farza and tell him to bring me back to life." + let synthesizer = NSSpeechSynthesizer() + synthesizer.startSpeaking(utterance) + voiceState = .responding + } + + // MARK: - Point Tag Parsing + + /// Result of parsing a [POINT:...] tag from Claude's response. + struct PointingParseResult { + /// The response text with the [POINT:...] tag removed — this is what gets spoken. + let spokenText: String + /// The parsed pixel coordinate, or nil if Claude said "none" or no tag was found. + let coordinate: CGPoint? + /// Short label describing the element (e.g. "run button"), or "none". + let elementLabel: String? + /// Which screen the coordinate refers to (1-based), or nil to default to cursor screen. + let screenNumber: Int? + } + + /// Parses a [POINT:x,y:label:screenN] or [POINT:none] tag from the end of Claude's response. + /// Returns the spoken text (tag removed) and the optional coordinate + label + screen number. + static func parsePointingCoordinates(from responseText: String) -> PointingParseResult { + // Match [POINT:none] or [POINT:123,456:label] or [POINT:123,456:label:screen2] + let pattern = #"\[POINT:(?:none|(\d+)\s*,\s*(\d+)(?::([^\]:\s][^\]:]*?))?(?::screen(\d+))?)\]\s*$"# + + guard let regex = try? NSRegularExpression(pattern: pattern, options: []), + let match = regex.firstMatch(in: responseText, range: NSRange(responseText.startIndex..., in: responseText)) else { + // No tag found at all + return PointingParseResult(spokenText: responseText, coordinate: nil, elementLabel: nil, screenNumber: nil) + } + + // Remove the tag from the spoken text + let tagRange = Range(match.range, in: responseText)! + let spokenText = String(responseText[..= 3, + let xRange = Range(match.range(at: 1), in: responseText), + let yRange = Range(match.range(at: 2), in: responseText), + let x = Double(responseText[xRange]), + let y = Double(responseText[yRange]) else { + return PointingParseResult(spokenText: spokenText, coordinate: nil, elementLabel: "none", screenNumber: nil) + } + + var elementLabel: String? = nil + if match.numberOfRanges >= 4, let labelRange = Range(match.range(at: 3), in: responseText) { + elementLabel = String(responseText[labelRange]).trimmingCharacters(in: .whitespaces) + } + + var screenNumber: Int? = nil + if match.numberOfRanges >= 5, let screenRange = Range(match.range(at: 4), in: responseText) { + screenNumber = Int(responseText[screenRange]) + } + + return PointingParseResult( + spokenText: spokenText, + coordinate: CGPoint(x: x, y: y), + elementLabel: elementLabel, + screenNumber: screenNumber + ) + } + + // MARK: - Onboarding Video + + /// Sets up the onboarding video player, starts playback, and schedules + /// the demo interaction at 40s. Called by BlueCursorView when onboarding starts. + func setupOnboardingVideo() { + guard let videoURL = URL(string: "https://stream.mux.com/e5jB8UuSrtFABVnTHCR7k3sIsmcUHCyhtLu1tzqLlfs.m3u8") else { return } + + let player = AVPlayer(url: videoURL) + player.isMuted = false + player.volume = 0.0 + self.onboardingVideoPlayer = player + self.showOnboardingVideo = true + self.onboardingVideoOpacity = 0.0 + + // Start playback immediately — the video plays while invisible, + // then we fade in both the visual and audio over 1s. + player.play() + + // Wait for SwiftUI to mount the view, then set opacity to 1. + // The .animation modifier on the view handles the actual animation. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { + self.onboardingVideoOpacity = 1.0 + // Fade audio volume from 0 → 1 over 2s to match visual fade + self.fadeInVideoAudio(player: player, targetVolume: 1.0, duration: 2.0) + } + + // At 40 seconds into the video, trigger the onboarding demo where + // Clicky flies to something interesting on screen and comments on it + let demoTriggerTime = CMTime(seconds: 40, preferredTimescale: 600) + onboardingDemoTimeObserver = player.addBoundaryTimeObserver( + forTimes: [NSValue(time: demoTriggerTime)], + queue: .main + ) { [weak self] in + ClickyAnalytics.trackOnboardingDemoTriggered() + self?.performOnboardingDemoInteraction() + } + + // Fade out and clean up when the video finishes + onboardingVideoEndObserver = NotificationCenter.default.addObserver( + forName: AVPlayerItem.didPlayToEndTimeNotification, + object: player.currentItem, + queue: .main + ) { [weak self] _ in + guard let self else { return } + ClickyAnalytics.trackOnboardingVideoCompleted() + self.onboardingVideoOpacity = 0.0 + // Wait for the 2s fade-out animation to complete before tearing down + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.tearDownOnboardingVideo() + // After the video disappears, stream in the prompt to try talking + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + self.startOnboardingPromptStream() + } + } + } + } + + func tearDownOnboardingVideo() { + showOnboardingVideo = false + if let timeObserver = onboardingDemoTimeObserver { + onboardingVideoPlayer?.removeTimeObserver(timeObserver) + onboardingDemoTimeObserver = nil + } + onboardingVideoPlayer?.pause() + onboardingVideoPlayer = nil + if let observer = onboardingVideoEndObserver { + NotificationCenter.default.removeObserver(observer) + onboardingVideoEndObserver = nil + } + } + + private func startOnboardingPromptStream() { + let message = "press control + option and introduce yourself" + onboardingPromptText = "" + showOnboardingPrompt = true + onboardingPromptOpacity = 0.0 + + withAnimation(.easeIn(duration: 0.4)) { + onboardingPromptOpacity = 1.0 + } + + var currentIndex = 0 + Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { timer in + guard currentIndex < message.count else { + timer.invalidate() + // Auto-dismiss after 10 seconds + DispatchQueue.main.asyncAfter(deadline: .now() + 10.0) { + guard self.showOnboardingPrompt else { return } + withAnimation(.easeOut(duration: 0.3)) { + self.onboardingPromptOpacity = 0.0 + } + DispatchQueue.main.asyncAfter(deadline: .now() + 0.35) { + self.showOnboardingPrompt = false + self.onboardingPromptText = "" + } + } + return + } + let index = message.index(message.startIndex, offsetBy: currentIndex) + self.onboardingPromptText.append(message[index]) + currentIndex += 1 + } + } + + /// Gradually raises an AVPlayer's volume from its current level to the + /// target over the specified duration, creating a smooth audio fade-in. + private func fadeInVideoAudio(player: AVPlayer, targetVolume: Float, duration: Double) { + let steps = 20 + let stepInterval = duration / Double(steps) + let volumeIncrement = (targetVolume - player.volume) / Float(steps) + var stepsRemaining = steps + + Timer.scheduledTimer(withTimeInterval: stepInterval, repeats: true) { timer in + stepsRemaining -= 1 + player.volume += volumeIncrement + + if stepsRemaining <= 0 { + timer.invalidate() + player.volume = targetVolume + } + } + } + + // MARK: - Onboarding Demo Interaction + + private static let onboardingDemoSystemPrompt = """ + you're clicky, a small blue cursor buddy living on the user's screen. you're showing off during onboarding — look at their screen and find ONE specific, concrete thing to point at. pick something with a clear name or identity: a specific app icon (say its name), a specific word or phrase of text you can read, a specific filename, a specific button label, a specific tab title, a specific image you can describe. do NOT point at vague things like "a window" or "some text" — be specific about exactly what you see. + + make a short quirky 3-6 word observation about the specific thing you picked — something fun, playful, or curious that shows you actually read/recognized it. no emojis ever. NEVER quote or repeat text you see on screen — just react to it. keep it to 6 words max, no exceptions. + + CRITICAL COORDINATE RULE: you MUST only pick elements near the CENTER of the screen. your x coordinate must be between 20%-80% of the image width. your y coordinate must be between 20%-80% of the image height. do NOT pick anything in the top 20%, bottom 20%, left 20%, or right 20% of the screen. no menu bar items, no dock icons, no sidebar items, no items near any edge. only things clearly in the middle area of the screen. if the only interesting things are near the edges, pick something boring in the center instead. + + respond with ONLY your short comment followed by the coordinate tag. nothing else. all lowercase. + + format: your comment [POINT:x,y:label] + + the screenshot images are labeled with their pixel dimensions. use those dimensions as the coordinate space. origin (0,0) is top-left. x increases rightward, y increases downward. + """ + + /// Captures a screenshot and asks Claude to find something interesting to + /// point at, then triggers the buddy's flight animation. Used during + /// onboarding to demo the pointing feature while the intro video plays. + func performOnboardingDemoInteraction() { + // Don't interrupt an active voice response + guard voiceState == .idle || voiceState == .responding else { return } + + Task { + do { + let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + + // Only send the cursor screen so Claude can't pick something + // on a different monitor that we can't point at. + guard let cursorScreenCapture = screenCaptures.first(where: { $0.isCursorScreen }) else { + print("🎯 Onboarding demo: no cursor screen found") + return + } + + let dimensionInfo = " (image dimensions: \(cursorScreenCapture.screenshotWidthInPixels)x\(cursorScreenCapture.screenshotHeightInPixels) pixels)" + let labeledImages = [(data: cursorScreenCapture.imageData, label: cursorScreenCapture.label + dimensionInfo)] + + let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( + images: labeledImages, + systemPrompt: Self.onboardingDemoSystemPrompt, + userPrompt: "look around my screen and find something interesting to point at", + onTextChunk: { _ in } + ) + + let parseResult = Self.parsePointingCoordinates(from: fullResponseText) + + guard let pointCoordinate = parseResult.coordinate else { + print("🎯 Onboarding demo: no element to point at") + return + } + + let screenshotWidth = CGFloat(cursorScreenCapture.screenshotWidthInPixels) + let screenshotHeight = CGFloat(cursorScreenCapture.screenshotHeightInPixels) + let displayWidth = CGFloat(cursorScreenCapture.displayWidthInPoints) + let displayHeight = CGFloat(cursorScreenCapture.displayHeightInPoints) + let displayFrame = cursorScreenCapture.displayFrame + + let clampedX = max(0, min(pointCoordinate.x, screenshotWidth)) + let clampedY = max(0, min(pointCoordinate.y, screenshotHeight)) + let displayLocalX = clampedX * (displayWidth / screenshotWidth) + let displayLocalY = clampedY * (displayHeight / screenshotHeight) + let appKitY = displayHeight - displayLocalY + let globalLocation = CGPoint( + x: displayLocalX + displayFrame.origin.x, + y: appKitY + displayFrame.origin.y + ) + + // Set custom bubble text so the pointing animation uses Claude's + // comment instead of a random phrase + detectedElementBubbleText = parseResult.spokenText + detectedElementScreenLocation = globalLocation + detectedElementDisplayFrame = displayFrame + print("🎯 Onboarding demo: pointing at \"\(parseResult.elementLabel ?? "element")\" — \"\(parseResult.spokenText)\"") + } catch { + print("⚠️ Onboarding demo error: \(error)") + } + } + } +} diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift new file mode 100644 index 0000000..76789b4 --- /dev/null +++ b/leanring-buddy/CompanionPanelView.swift @@ -0,0 +1,761 @@ +// +// CompanionPanelView.swift +// leanring-buddy +// +// The SwiftUI content hosted inside the menu bar panel. Shows the companion +// voice status, push-to-talk shortcut, and quick settings. Designed to feel +// like Loom's recording panel — dark, rounded, minimal, and special. +// + +import AVFoundation +import SwiftUI + +struct CompanionPanelView: View { + @ObservedObject var companionManager: CompanionManager + @State private var emailInput: String = "" + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + panelHeader + Divider() + .background(DS.Colors.borderSubtle) + .padding(.horizontal, 16) + + permissionsCopySection + .padding(.top, 16) + .padding(.horizontal, 16) + + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 12) + + modelPickerRow + .padding(.horizontal, 16) + } + + if !companionManager.allPermissionsGranted { + Spacer() + .frame(height: 16) + + settingsSection + .padding(.horizontal, 16) + } + + if !companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 16) + + startButton + .padding(.horizontal, 16) + } + + // Show Clicky toggle — hidden for now + // if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + // Spacer() + // .frame(height: 16) + // + // showClickyCursorToggleRow + // .padding(.horizontal, 16) + // } + + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 16) + + dmFarzaButton + .padding(.horizontal, 16) + } + + Spacer() + .frame(height: 12) + + Divider() + .background(DS.Colors.borderSubtle) + .padding(.horizontal, 16) + + footerSection + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + .frame(width: 320) + .background(panelBackground) + } + + // MARK: - Header + + private var panelHeader: some View { + HStack { + HStack(spacing: 8) { + // Animated status dot + Circle() + .fill(statusDotColor) + .frame(width: 8, height: 8) + .shadow(color: statusDotColor.opacity(0.6), radius: 4) + + Text("Clicky") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(DS.Colors.textPrimary) + } + + Spacer() + + Text(statusText) + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + + Button(action: { + NotificationCenter.default.post(name: .clickyDismissPanel, object: nil) + }) { + Image(systemName: "xmark") + .font(.system(size: 10, weight: .semibold)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 20, height: 20) + .background( + Circle() + .fill(Color.white.opacity(0.08)) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + .padding(.horizontal, 16) + .padding(.vertical, 14) + } + + // MARK: - Permissions Copy + + @ViewBuilder + private var permissionsCopySection: some View { + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Text("Hold Control+Option to talk.") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + } else if companionManager.allPermissionsGranted && !companionManager.hasSubmittedEmail { + VStack(alignment: .leading, spacing: 4) { + Text("Drop your email to get started.") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + Text("If I keep building this, I'll keep you in the loop.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + } + .frame(maxWidth: .infinity, alignment: .leading) + } else if companionManager.allPermissionsGranted { + Text("You're all set. Hit Start to meet Clicky.") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + } else if companionManager.hasCompletedOnboarding { + // Permissions were revoked after onboarding — tell user to re-grant + VStack(alignment: .leading, spacing: 6) { + Text("Permissions needed") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(DS.Colors.textSecondary) + + Text("Some permissions were revoked. Grant all four below to keep using Clicky.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + } else { + VStack(alignment: .leading, spacing: 6) { + Text("Hi, I'm Farza. This is Clicky.") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(DS.Colors.textSecondary) + + Text("A side project I made for fun to help me learn stuff as I use my computer.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + Text("Nothing runs in the background. Clicky will only take a screenshot when you press the hot key. So, you can give that permission in peace. If you are still sus, eh, I can't do much there champ.") + .font(.system(size: 11)) + .foregroundColor(Color(red: 0.9, green: 0.4, blue: 0.4)) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + } + + // MARK: - Email + Start Button + + @ViewBuilder + private var startButton: some View { + if !companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + if !companionManager.hasSubmittedEmail { + VStack(spacing: 8) { + TextField("Enter your email", text: $emailInput) + .textFieldStyle(.plain) + .font(.system(size: 13)) + .foregroundColor(DS.Colors.textPrimary) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.medium, style: .continuous) + .fill(Color.white.opacity(0.08)) + ) + .overlay( + RoundedRectangle(cornerRadius: DS.CornerRadius.medium, style: .continuous) + .stroke(DS.Colors.borderSubtle, lineWidth: 0.5) + ) + + Button(action: { + companionManager.submitEmail(emailInput) + }) { + Text("Submit") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .frame(maxWidth: .infinity) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.large, style: .continuous) + .fill(emailInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + ? DS.Colors.accent.opacity(0.4) + : DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + .disabled(emailInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + } else { + Button(action: { + companionManager.triggerOnboarding() + }) { + Text("Start") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .frame(maxWidth: .infinity) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.large, style: .continuous) + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + } + + // MARK: - Permissions + + private var settingsSection: some View { + VStack(spacing: 2) { + Text("PERMISSIONS") + .font(.system(size: 10, weight: .semibold, design: .rounded)) + .foregroundColor(DS.Colors.textTertiary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.bottom, 6) + + microphonePermissionRow + + accessibilityPermissionRow + + screenRecordingPermissionRow + + if companionManager.hasScreenRecordingPermission { + screenContentPermissionRow + } + + } + } + + private var accessibilityPermissionRow: some View { + let isGranted = companionManager.hasAccessibilityPermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "hand.raised") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + Text("Accessibility") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + HStack(spacing: 6) { + Button(action: { + // Triggers the system accessibility prompt (AXIsProcessTrustedWithOptions) + // on first attempt, then opens System Settings on subsequent attempts. + WindowPositionManager.requestAccessibilityPermission() + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + // Reveals the app in Finder so the user can drag it into + // the Accessibility list if it doesn't appear automatically + // (common with unsigned dev builds). + WindowPositionManager.revealAppInFinder() + WindowPositionManager.openAccessibilitySettings() + }) { + Text("Find App") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .stroke(DS.Colors.borderSubtle, lineWidth: 0.8) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + } + .padding(.vertical, 6) + } + + private var screenRecordingPermissionRow: some View { + let isGranted = companionManager.hasScreenRecordingPermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "rectangle.dashed.badge.record") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + VStack(alignment: .leading, spacing: 1) { + Text("Screen Recording") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Text(isGranted + ? "Only takes a screenshot when you use the hotkey" + : "Quit and reopen after granting") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + Button(action: { + // Triggers the native macOS screen recording prompt on first + // attempt (auto-adds app to the list), then opens System Settings + // on subsequent attempts. + WindowPositionManager.requestScreenRecordingPermission() + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.vertical, 6) + } + + private var screenContentPermissionRow: some View { + let isGranted = companionManager.hasScreenContentPermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "eye") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + Text("Screen Content") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + Button(action: { + companionManager.requestScreenContentPermission() + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.vertical, 6) + } + + private var microphonePermissionRow: some View { + let isGranted = companionManager.hasMicrophonePermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "mic") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + Text("Microphone") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + Button(action: { + // Triggers the native macOS microphone permission dialog on + // first attempt. If already denied, opens System Settings. + let status = AVCaptureDevice.authorizationStatus(for: .audio) + if status == .notDetermined { + AVCaptureDevice.requestAccess(for: .audio) { _ in } + } else { + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") { + NSWorkspace.shared.open(url) + } + } + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.vertical, 6) + } + + private func permissionRow( + label: String, + iconName: String, + isGranted: Bool, + settingsURL: String + ) -> some View { + HStack { + HStack(spacing: 8) { + Image(systemName: iconName) + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + Text(label) + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + Button(action: { + if let url = URL(string: settingsURL) { + NSWorkspace.shared.open(url) + } + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.vertical, 6) + } + + + + // MARK: - Show Clicky Cursor Toggle + + private var showClickyCursorToggleRow: some View { + HStack { + HStack(spacing: 8) { + Image(systemName: "cursorarrow") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 16) + + Text("Show Clicky") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + Toggle("", isOn: Binding( + get: { companionManager.isClickyCursorEnabled }, + set: { companionManager.setClickyCursorEnabled($0) } + )) + .toggleStyle(.switch) + .labelsHidden() + .tint(DS.Colors.accent) + .scaleEffect(0.8) + } + .padding(.vertical, 4) + } + + private var speechToTextProviderRow: some View { + HStack { + HStack(spacing: 8) { + Image(systemName: "mic.badge.waveform") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 16) + + Text("Speech to Text") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + Text(companionManager.buddyDictationManager.transcriptionProviderDisplayName) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .padding(.vertical, 4) + } + + // MARK: - Model Picker + + private var modelPickerRow: some View { + HStack { + Text("Model") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Spacer() + + HStack(spacing: 0) { + modelOptionButton(label: "Sonnet", modelID: "claude-sonnet-4-6") + modelOptionButton(label: "Opus", modelID: "claude-opus-4-6") + } + .background( + RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(Color.white.opacity(0.06)) + ) + .overlay( + RoundedRectangle(cornerRadius: 6, style: .continuous) + .stroke(DS.Colors.borderSubtle, lineWidth: 0.5) + ) + } + .padding(.vertical, 4) + } + + private func modelOptionButton(label: String, modelID: String) -> some View { + let isSelected = companionManager.selectedModel == modelID + return Button(action: { + companionManager.setSelectedModel(modelID) + }) { + Text(label) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(isSelected ? DS.Colors.textPrimary : DS.Colors.textTertiary) + .padding(.horizontal, 10) + .padding(.vertical, 5) + .background( + RoundedRectangle(cornerRadius: 5, style: .continuous) + .fill(isSelected ? Color.white.opacity(0.1) : Color.clear) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + + // MARK: - DM Farza Button + + private var dmFarzaButton: some View { + Button(action: { + if let url = URL(string: "https://x.com/farzatv") { + NSWorkspace.shared.open(url) + } + }) { + HStack(spacing: 8) { + Image(systemName: "bubble.left.fill") + .font(.system(size: 12, weight: .medium)) + + VStack(alignment: .leading, spacing: 2) { + Text("Got feedback? DM me") + .font(.system(size: 12, weight: .semibold)) + Text("Bugs, ideas, anything — I read every message.") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.medium, style: .continuous) + .fill(Color.white.opacity(0.06)) + ) + .overlay( + RoundedRectangle(cornerRadius: DS.CornerRadius.medium, style: .continuous) + .stroke(DS.Colors.borderSubtle, lineWidth: 0.5) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + + // MARK: - Footer + + private var footerSection: some View { + HStack { + Button(action: { + NSApp.terminate(nil) + }) { + HStack(spacing: 6) { + Image(systemName: "power") + .font(.system(size: 11, weight: .medium)) + Text("Quit Clicky") + .font(.system(size: 12, weight: .medium)) + } + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + + if companionManager.hasCompletedOnboarding { + Spacer() + + Button(action: { + companionManager.replayOnboarding() + }) { + HStack(spacing: 6) { + Image(systemName: "play.circle") + .font(.system(size: 11, weight: .medium)) + Text("Watch Onboarding Again") + .font(.system(size: 12, weight: .medium)) + } + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + } + + // MARK: - Visual Helpers + + private var panelBackground: some View { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(DS.Colors.background) + .shadow(color: Color.black.opacity(0.5), radius: 20, x: 0, y: 10) + .shadow(color: Color.black.opacity(0.3), radius: 4, x: 0, y: 2) + } + + private var statusDotColor: Color { + if !companionManager.isOverlayVisible { + return DS.Colors.textTertiary + } + switch companionManager.voiceState { + case .idle: + return DS.Colors.success + case .listening: + return DS.Colors.blue400 + case .processing, .responding: + return DS.Colors.blue400 + } + } + + private var statusText: String { + if !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { + return "Setup" + } + if !companionManager.isOverlayVisible { + return "Ready" + } + switch companionManager.voiceState { + case .idle: + return "Active" + case .listening: + return "Listening" + case .processing: + return "Processing" + case .responding: + return "Responding" + } + } + +} diff --git a/leanring-buddy/CompanionResponseOverlay.swift b/leanring-buddy/CompanionResponseOverlay.swift new file mode 100644 index 0000000..a11c624 --- /dev/null +++ b/leanring-buddy/CompanionResponseOverlay.swift @@ -0,0 +1,217 @@ +// +// CompanionResponseOverlay.swift +// leanring-buddy +// +// Cursor-following overlay that displays streaming AI response text. +// Uses a non-activating NSPanel so it floats above all apps without +// stealing focus, and repositions itself near the mouse cursor each frame. +// + +import AppKit +import Combine +import SwiftUI + +// MARK: - View Model + +@MainActor +final class CompanionResponseOverlayViewModel: ObservableObject { + @Published var streamingResponseText: String = "" + @Published var isShowingResponse: Bool = false +} + +// MARK: - Overlay Manager + +@MainActor +final class CompanionResponseOverlayManager { + private let overlayViewModel = CompanionResponseOverlayViewModel() + private var overlayPanel: NSPanel? + private var cursorTrackingTimer: Timer? + private var autoHideWorkItem: DispatchWorkItem? + + /// The horizontal offset from the cursor to the left edge of the overlay panel. + private let cursorOffsetX: CGFloat = 22 + /// The vertical offset from the cursor downward to the top edge of the overlay panel. + private let cursorOffsetY: CGFloat = 6 + /// Maximum width of the overlay panel. + private let overlayMaxWidth: CGFloat = 340 + + func showOverlayAndBeginStreaming() { + autoHideWorkItem?.cancel() + autoHideWorkItem = nil + + overlayViewModel.streamingResponseText = "" + overlayViewModel.isShowingResponse = true + createOverlayPanelIfNeeded() + startCursorTracking() + overlayPanel?.alphaValue = 1 + overlayPanel?.orderFrontRegardless() + } + + func updateStreamingText(_ accumulatedText: String) { + overlayViewModel.streamingResponseText = accumulatedText + resizePanelToFitContent() + } + + func finishStreaming() { + // Keep the response visible for a few seconds after streaming ends, + // then fade out so the user has time to read the last chunk. + let hideWork = DispatchWorkItem { [weak self] in + self?.fadeOutAndHide() + } + autoHideWorkItem = hideWork + DispatchQueue.main.asyncAfter(deadline: .now() + 6, execute: hideWork) + } + + func hideOverlay() { + autoHideWorkItem?.cancel() + autoHideWorkItem = nil + stopCursorTracking() + overlayViewModel.isShowingResponse = false + overlayViewModel.streamingResponseText = "" + overlayPanel?.orderOut(nil) + } + + // MARK: - Private + + private func createOverlayPanelIfNeeded() { + if overlayPanel != nil { return } + + let initialFrame = NSRect(x: 0, y: 0, width: overlayMaxWidth, height: 40) + let responseOverlayPanel = NSPanel( + contentRect: initialFrame, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + + responseOverlayPanel.level = .statusBar + responseOverlayPanel.isOpaque = false + responseOverlayPanel.backgroundColor = .clear + responseOverlayPanel.hasShadow = false + responseOverlayPanel.ignoresMouseEvents = true + responseOverlayPanel.hidesOnDeactivate = false + responseOverlayPanel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary] + responseOverlayPanel.isExcludedFromWindowsMenu = true + + let hostingView = NSHostingView( + rootView: CompanionResponseOverlayView(viewModel: overlayViewModel) + .frame(maxWidth: overlayMaxWidth) + ) + hostingView.frame = initialFrame + responseOverlayPanel.contentView = hostingView + + overlayPanel = responseOverlayPanel + } + + private func startCursorTracking() { + // 60fps cursor tracking so the panel stays glued to the mouse + cursorTrackingTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 60.0, repeats: true) { [weak self] _ in + Task { @MainActor [weak self] in + self?.repositionPanelNearCursor() + } + } + } + + private func stopCursorTracking() { + cursorTrackingTimer?.invalidate() + cursorTrackingTimer = nil + } + + private func repositionPanelNearCursor() { + guard let overlayPanel else { return } + + let mouseLocation = NSEvent.mouseLocation + let panelSize = overlayPanel.frame.size + + // Position the panel to the right of and slightly below the cursor. + // In macOS screen coordinates, Y increases upward, so "below" means + // subtracting from the cursor Y. + var panelOriginX = mouseLocation.x + cursorOffsetX + var panelOriginY = mouseLocation.y - cursorOffsetY - panelSize.height + + // Clamp to the visible frame of the screen containing the cursor + // so the panel never goes off-screen. + if let currentScreen = screenContainingPoint(mouseLocation) { + let visibleFrame = currentScreen.visibleFrame + + // If the panel would go off the right edge, flip it to the left of the cursor + if panelOriginX + panelSize.width > visibleFrame.maxX { + panelOriginX = mouseLocation.x - cursorOffsetX - panelSize.width + } + + // If the panel would go below the bottom edge, push it above the cursor + if panelOriginY < visibleFrame.minY { + panelOriginY = mouseLocation.y + cursorOffsetY + } + + // Final clamp + panelOriginX = max(visibleFrame.minX, min(panelOriginX, visibleFrame.maxX - panelSize.width)) + panelOriginY = max(visibleFrame.minY, min(panelOriginY, visibleFrame.maxY - panelSize.height)) + } + + overlayPanel.setFrameOrigin(CGPoint(x: panelOriginX, y: panelOriginY)) + } + + private func resizePanelToFitContent() { + guard let overlayPanel, let contentView = overlayPanel.contentView else { return } + + let fittingSize = contentView.fittingSize + let newWidth = min(fittingSize.width, overlayMaxWidth) + let newHeight = fittingSize.height + + // Keep the panel origin relative to the cursor (the timer handles that), + // but update the frame size so the content fits. + var frame = overlayPanel.frame + let heightDelta = newHeight - frame.height + frame.size = CGSize(width: newWidth, height: newHeight) + // Adjust origin Y so the panel grows upward (toward the cursor), not downward + frame.origin.y -= heightDelta + overlayPanel.setFrame(frame, display: true) + contentView.frame = NSRect(origin: .zero, size: frame.size) + } + + private func fadeOutAndHide() { + guard let overlayPanel else { return } + + NSAnimationContext.runAnimationGroup({ context in + context.duration = 0.4 + overlayPanel.animator().alphaValue = 0 + }, completionHandler: { [weak self] in + Task { @MainActor in + self?.hideOverlay() + } + }) + } + + private func screenContainingPoint(_ point: CGPoint) -> NSScreen? { + NSScreen.screens.first { $0.frame.contains(point) } + } +} + +// MARK: - SwiftUI View + +private struct CompanionResponseOverlayView: View { + @ObservedObject var viewModel: CompanionResponseOverlayViewModel + + var body: some View { + if viewModel.isShowingResponse { + Text(viewModel.streamingResponseText.isEmpty ? "..." : viewModel.streamingResponseText) + .font(.system(size: 13, weight: .regular)) + .foregroundColor(DS.Colors.textPrimary) + .lineSpacing(3) + .fixedSize(horizontal: false, vertical: true) + .frame(maxWidth: 300, alignment: .leading) + .padding(.horizontal, 14) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(DS.Colors.surface1.opacity(0.95)) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .stroke(DS.Colors.borderSubtle.opacity(0.5), lineWidth: 0.8) + ) + .shadow(color: Color.black.opacity(0.35), radius: 16, x: 0, y: 8) + ) + } + } +} diff --git a/leanring-buddy/CompanionScreenCaptureUtility.swift b/leanring-buddy/CompanionScreenCaptureUtility.swift new file mode 100644 index 0000000..7978417 --- /dev/null +++ b/leanring-buddy/CompanionScreenCaptureUtility.swift @@ -0,0 +1,132 @@ +// +// CompanionScreenCaptureUtility.swift +// leanring-buddy +// +// Standalone screenshot capture for the companion voice flow. +// Decoupled from the legacy ScreenshotManager so the companion mode +// can capture screenshots independently without session state. +// + +import AppKit +import ScreenCaptureKit + +struct CompanionScreenCapture { + let imageData: Data + let label: String + let isCursorScreen: Bool + let displayWidthInPoints: Int + let displayHeightInPoints: Int + let displayFrame: CGRect + let screenshotWidthInPixels: Int + let screenshotHeightInPixels: Int +} + +@MainActor +enum CompanionScreenCaptureUtility { + + /// Captures all connected displays as JPEG data, labeling each with + /// whether the user's cursor is on that screen. This gives the AI + /// full context across multiple monitors. + static func captureAllScreensAsJPEG() async throws -> [CompanionScreenCapture] { + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + + guard !content.displays.isEmpty else { + throw NSError(domain: "CompanionScreenCapture", code: -1, + userInfo: [NSLocalizedDescriptionKey: "No display available for capture"]) + } + + let mouseLocation = NSEvent.mouseLocation + + // Exclude all windows belonging to this app so the AI sees + // only the user's content, not our overlays or panels. + let ownBundleIdentifier = Bundle.main.bundleIdentifier + let ownAppWindows = content.windows.filter { window in + window.owningApplication?.bundleIdentifier == ownBundleIdentifier + } + + // Build a lookup from display ID to NSScreen so we can use AppKit-coordinate + // frames instead of CG-coordinate frames. NSEvent.mouseLocation and NSScreen.frame + // both use AppKit coordinates (bottom-left origin), while SCDisplay.frame uses + // Core Graphics coordinates (top-left origin). On multi-display setups, the Y + // origins differ for secondary displays, which breaks cursor-contains checks + // and downstream coordinate conversions. + var nsScreenByDisplayID: [CGDirectDisplayID: NSScreen] = [:] + for screen in NSScreen.screens { + if let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID { + nsScreenByDisplayID[screenNumber] = screen + } + } + + // Sort displays so the cursor screen is always first + let sortedDisplays = content.displays.sorted { displayA, displayB in + let frameA = nsScreenByDisplayID[displayA.displayID]?.frame ?? displayA.frame + let frameB = nsScreenByDisplayID[displayB.displayID]?.frame ?? displayB.frame + let aContainsCursor = frameA.contains(mouseLocation) + let bContainsCursor = frameB.contains(mouseLocation) + if aContainsCursor != bContainsCursor { return aContainsCursor } + return false + } + + var capturedScreens: [CompanionScreenCapture] = [] + + for (displayIndex, display) in sortedDisplays.enumerated() { + // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so + // displayFrame is in the same coordinate system as NSEvent.mouseLocation + // and the overlay window's screenFrame in BlueCursorView. + let displayFrame = nsScreenByDisplayID[display.displayID]?.frame + ?? CGRect(x: display.frame.origin.x, y: display.frame.origin.y, + width: CGFloat(display.width), height: CGFloat(display.height)) + let isCursorScreen = displayFrame.contains(mouseLocation) + + let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) + + let configuration = SCStreamConfiguration() + let maxDimension = 1280 + let aspectRatio = CGFloat(display.width) / CGFloat(display.height) + if display.width >= display.height { + configuration.width = maxDimension + configuration.height = Int(CGFloat(maxDimension) / aspectRatio) + } else { + configuration.height = maxDimension + configuration.width = Int(CGFloat(maxDimension) * aspectRatio) + } + + let cgImage = try await SCScreenshotManager.captureImage( + contentFilter: filter, + configuration: configuration + ) + + guard let jpegData = NSBitmapImageRep(cgImage: cgImage) + .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + continue + } + + let screenLabel: String + if sortedDisplays.count == 1 { + screenLabel = "user's screen (cursor is here)" + } else if isCursorScreen { + screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)" + } else { + screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — secondary screen" + } + + capturedScreens.append(CompanionScreenCapture( + imageData: jpegData, + label: screenLabel, + isCursorScreen: isCursorScreen, + displayWidthInPoints: Int(displayFrame.width), + displayHeightInPoints: Int(displayFrame.height), + displayFrame: displayFrame, + screenshotWidthInPixels: configuration.width, + screenshotHeightInPixels: configuration.height + )) + } + + guard !capturedScreens.isEmpty else { + throw NSError(domain: "CompanionScreenCapture", code: -2, + userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"]) + } + + return capturedScreens + } +} diff --git a/leanring-buddy/DesignSystem.swift b/leanring-buddy/DesignSystem.swift new file mode 100644 index 0000000..d77e12a --- /dev/null +++ b/leanring-buddy/DesignSystem.swift @@ -0,0 +1,880 @@ +// +// DesignSystem.swift +// leanring-buddy +// +// Centralized design system using a blue accent palette on dark surfaces, +// with a unified button style system. All colors, button styles, and +// interaction states are defined here as the single source of truth. +// + +import SwiftUI +import AppKit + +// MARK: - Design System Namespace + +/// The top-level namespace for all design system tokens. +/// Usage: `DS.Colors.background`, `DS.Colors.accent`, etc. +enum DS { + + // MARK: - Color Tokens + + enum Colors { + + // ── Backgrounds ────────────────────────────────────────────── + // Layered surfaces from deepest to most elevated. + // Higher surfaces are lighter, creating a sense of depth. + + /// The deepest background — used for the main app window fill. + static let background = Color(hex: "#101211") + + /// First elevation layer — used for cards, sidebar, top bar backgrounds. + static let surface1 = Color(hex: "#171918") + + /// Second elevation layer — used for input fields, elevated cards, chat bubbles. + static let surface2 = Color(hex: "#202221") + + /// Third elevation layer — used for hover backgrounds on interactive elements. + static let surface3 = Color(hex: "#272A29") + + /// Fourth elevation layer — used for active/pressed states on interactive elements. + static let surface4 = Color(hex: "#2E3130") + + // ── Borders ────────────────────────────────────────────────── + + /// Subtle border — used for card outlines, dividers, input field borders. + static let borderSubtle = Color(hex: "#373B39") + + /// Strong border — used for focused inputs, hovered card outlines. + static let borderStrong = Color(hex: "#444947") + + // ── Text ───────────────────────────────────────────────────── + + /// Primary text — main body text, titles, headings. + static let textPrimary = Color(hex: "#ECEEED") + + /// Secondary text — descriptions, hints, muted labels. + static let textSecondary = Color(hex: "#ADB5B2") + + /// Tertiary text — very muted, used for section labels, timestamps, disabled text. + static let textTertiary = Color(hex: "#6B736F") + + /// Text used on top of the accent fill (#2563eb blue), like the primary button label. + /// White on #2563eb achieves ~5.1:1 contrast — WCAG AA compliant. + /// White on #1d4ed8 hover achieves ~6.5:1 — also WCAG AA compliant. + static let textOnAccent: Color = .white + + // ── Tailwind Blue Scale ───────────────────────────────────── + // Full Tailwind CSS v4 blue palette for consistent blue usage. + // + // Usage guide: + // 50–100 → Very subtle tinted backgrounds (selected rows, hover fills on dark surfaces) + // 200–300 → Light text/icons on dark backgrounds, disabled states + // 400 → Bright accent text, links, icons, chat user bubbles + // 500 → Mid-tone fills, badges, secondary buttons + // 600 → Primary action fills (buttons, toggles) — main accent + // 700 → Hover/pressed state for primary actions + // 800–900 → Deep backgrounds, dark overlays, header bars + // 950 → Deepest blue — near-black tinted backgrounds + + static let blue50 = Color(hex: "#eff6ff") + static let blue100 = Color(hex: "#dbeafe") + static let blue200 = Color(hex: "#bfdbfe") + static let blue300 = Color(hex: "#93c5fd") + static let blue400 = Color(hex: "#60a5fa") + static let blue500 = Color(hex: "#3b82f6") + static let blue600 = Color(hex: "#2563eb") + static let blue700 = Color(hex: "#1d4ed8") + static let blue800 = Color(hex: "#1e40af") + static let blue900 = Color(hex: "#1e3a8a") + static let blue950 = Color(hex: "#172554") + + // ── Accent (derived from blue scale) ─────────────────────── + // The primary fill is Blue 600; hover darkens to Blue 700. + + /// Accent fill — used for solid button backgrounds. + /// #2563eb → ~5.1:1 contrast with white text (WCAG AA). + static let accent = blue600 + + /// Accent hover — slightly darker blue for hover state. + /// #1d4ed8 → ~6.5:1 contrast with white text (WCAG AA+). + static let accentHover = blue700 + + /// Accent text — bright blue used for accent-colored text and icons + /// on dark backgrounds (links, active nav items, highlighted labels). + static let accentText = blue400 + + /// Very subtle accent tint — used for selected item backgrounds (e.g. current step + /// in the sidebar). Low opacity so it doesn't overpower. + static let accentSubtle = blue500.opacity(0.10) + + // ── Semantic Colors ────────────────────────────────────────── + + /// Destructive/error actions — delete buttons, error messages, close button hover. + static let destructive = Color(hex: "#E5484D") // Radix Red 9 + + /// Destructive hover state. + static let destructiveHover = Color(hex: "#F2555A") // Radix Red 10 + + /// Destructive used for text on dark backgrounds (brighter for readability). + static let destructiveText = Color(hex: "#FF6369") // Radix Red 11 + + /// Success — checkmarks, granted status, completion indicators. + /// Independent green so success states are visually distinct from the blue accent. + static let success = Color(hex: "#34D399") // Tailwind Emerald 400 + + /// Warning — caution messages, manual verification failure explanations. + static let warning = Color(hex: "#FFB224") // Radix Amber 9 + + /// Warning text — brighter variant for text on dark backgrounds. + static let warningText = Color(hex: "#F1A10D") // Radix Amber 11 + + /// Info/feature highlight — used for prompt card headers, code highlights. + /// Lighter than accentText so informational elements are visually distinct + /// from interactive accent-colored elements. + static let info = Color(hex: "#70B8FF") // Radix Blue 9 + + /// Inline code text color — slightly brighter blue for monospace code snippets. + static let codeText = Color(hex: "#9DC2FF") // Radix Blue 11 variant + + // ── Overlay Cursor ─────────────────────────────────────────── + + /// The blue cursor/bubble color used in OverlayWindow. + /// Kept distinct from the accent since it serves a different purpose + /// (screen overlay vs in-app UI). + static let overlayCursorBlue = Color(hex: "#3380FF") + + // ── Floating Button Gradient ───────────────────────────────── + + /// The floating session button gradient colors (unchanged from original — + /// this gradient is intentionally distinct from the rest of the palette + /// to make the floating button stand out as a "jewel" on the desktop). + static let floatingGradientPurple = Color(hex: "#8F46EB") + static let floatingGradientPink = Color(hex: "#E84D9E") + static let floatingGradientOrange = Color(hex: "#FF8C33") + + // ── Help Chat ────────────────────────────────────────────── + + /// User message bubble background in the help chat. + /// Blue 800 — deep blue that's clearly distinct from the dark surface + /// while keeping white text highly readable (~9:1 contrast). + static let helpChatUserBubble = blue800 + + /// Slightly lighter variant for hover/pressed states on user bubbles. + static let helpChatUserBubbleHover = blue700 + + /// Footer/backdrop behind the floating help chat. + /// Slightly lighter than the main window background so the chat zone reads + /// as a distinct docked surface even before the pill input is visible. + static let helpChatBackdrop = Color(hex: "#212121") + + // ── Disabled State ─────────────────────────────────────────── + // Following Material Design 3's disabled pattern: + // Container: onSurface at 12% opacity + // Content: onSurface at 38% opacity + + /// Disabled button/container background. + static var disabledBackground: Color { + textPrimary.opacity(0.12) + } + + /// Disabled text/icon color. + static var disabledText: Color { + textPrimary.opacity(0.38) + } + } + + // MARK: - Spacing (for reference, not enforced) + + enum Spacing { + static let xs: CGFloat = 4 + static let sm: CGFloat = 8 + static let md: CGFloat = 12 + static let lg: CGFloat = 16 + static let xl: CGFloat = 20 + static let xxl: CGFloat = 24 + static let xxxl: CGFloat = 32 + } + + // MARK: - Corner Radii + + enum CornerRadius { + /// Small elements like tags, badges. + static let small: CGFloat = 6 + /// Buttons, input fields, small cards. + static let medium: CGFloat = 8 + /// Cards, dialogs, chat bubbles. + static let large: CGFloat = 10 + /// Large panels, permission cards. + static let extraLarge: CGFloat = 12 + /// Pill-shaped buttons (the continue button). + static let pill: CGFloat = .infinity + } + + // MARK: - Animation Durations + + enum Animation { + /// Quick state changes — hover in/out, press feedback. + static let fast: Double = 0.15 + /// Standard transitions — content reveal, button state changes. + static let normal: Double = 0.25 + /// Slower, more dramatic — fade-ins, celebration screen elements. + static let slow: Double = 0.4 + } + + // MARK: - State Layer Opacities + // Based on Material Design 3's state layer system. + // A "state layer" overlays the button's content color at these opacities. + + enum StateLayer { + /// Hover: subtle highlight to indicate interactivity. + static let hover: Double = 0.08 + /// Focus: keyboard navigation indicator (slightly stronger than hover). + static let focus: Double = 0.12 + /// Pressed: active press feedback (same strength as focus). + static let pressed: Double = 0.12 + /// Dragged: strongest overlay (rarely used). + static let dragged: Double = 0.16 + } +} + +// MARK: - Button Styles + +/// Primary button — the main call-to-action per screen. +/// Accent-colored background with white text. One per view maximum. +/// Used for: "start"/"resume", "let's go", "continue", "verify completion". +struct DSPrimaryButtonStyle: ButtonStyle { + var isFullWidth: Bool = true + + @State private var isHovered = false + + // Separate state for the scale expansion so it animates on a slower, + // more gradual timeline (0.6s) than the background color snap (0.15s). + @State private var isHoverScaleExpanded = false + + // Whether the hover glow shadow is active. Builds up gradually (0.6s) + // on hover entry, fades out faster (0.3s) on exit. + @State private var isHoverGlowActive = false + + // Continuously toggles while hovered to drive a gentle breathing pulse + // in the glow shadow. Creates a living, organic feel — like the button + // is softly glowing, not just statically lit. + @State private var isGlowBreathingIn = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: 16, weight: .medium)) + .foregroundColor(DS.Colors.textOnAccent) + .frame(maxWidth: isFullWidth ? .infinity : nil) + .padding(.vertical, 14) + .padding(.horizontal, isFullWidth ? 0 : 20) + .background( + Capsule() + .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) + ) + // Hover glow — builds up gradually, then gently breathes while hovered. + // The breathing oscillates opacity and radius on a slow 2.5s loop, + // creating a candle-flame-like "alive" quality rather than a static highlight. + .shadow( + color: DS.Colors.accent.opacity( + isHoverGlowActive ? (isGlowBreathingIn ? 0.32 : 0.18) : 0 + ), + radius: isHoverGlowActive ? (isGlowBreathingIn ? 16 : 10) : 0 + ) + // Hover: gradually expand to 1.03. Press: snap down to 0.97. + .scaleEffect(configuration.isPressed ? 0.97 : (isHoverScaleExpanded ? 1.03 : 1.0)) + .animation(.easeOut(duration: 0.1), value: configuration.isPressed) + .onHover { hovering in + // Background color — fast snap so the button feels responsive + withAnimation(.easeOut(duration: 0.15)) { + isHovered = hovering + } + + // Scale — slow, gradual expansion (like the button is swelling) + withAnimation(.easeInOut(duration: hovering ? 0.6 : 0.3)) { + isHoverScaleExpanded = hovering + } + + // Glow — builds up gradually on entry, fades faster on exit + withAnimation(.easeInOut(duration: hovering ? 0.6 : 0.3)) { + isHoverGlowActive = hovering + } + + // Breathing glow loop — gentle pulse while hovered. + // The 2.5s cycle keeps it feeling organic, not mechanical. + if hovering { + withAnimation( + .easeInOut(duration: 2.5) + .repeatForever(autoreverses: true) + ) { + isGlowBreathingIn = true + } + } else { + // Override the repeating animation with a finite one to stop cleanly + withAnimation(.easeOut(duration: 0.3)) { + isGlowBreathingIn = false + } + } + + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } + + private func buttonBackgroundColor(isPressed: Bool) -> Color { + if isPressed { + // Pressed: brighten slightly beyond hover + return DS.Colors.accentHover.blendedWithWhite(fraction: DS.StateLayer.pressed) + } else if isHovered { + return DS.Colors.accentHover + } else { + return DS.Colors.accent + } + } +} + +/// Secondary button — supporting actions, less visual weight than primary. +/// Surface-colored background with primary text. Used for: action buttons +/// (download, open link), embedded element buttons. +struct DSSecondaryButtonStyle: ButtonStyle { + var isFullWidth: Bool = true + + @State private var isHovered = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: 16, weight: .medium)) + .foregroundColor(DS.Colors.textPrimary) + .frame(maxWidth: isFullWidth ? .infinity : nil) + .padding(.vertical, 12) + .padding(.horizontal, isFullWidth ? 0 : 16) + .background( + Capsule() + .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) + ) + .scaleEffect(configuration.isPressed ? 0.97 : 1.0) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .onHover { hovering in + isHovered = hovering + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } + + private func buttonBackgroundColor(isPressed: Bool) -> Color { + if isPressed { + return DS.Colors.surface4 + } else if isHovered { + return DS.Colors.surface3 + } else { + return DS.Colors.surface2 + } + } +} + +/// Tertiary/ghost button — low-emphasis actions with subtle hover background. +/// Transparent at rest, shows surface fill on hover. Used for: navigation +/// links, sidebar items, medium-low emphasis actions. +struct DSTertiaryButtonStyle: ButtonStyle { + @State private var isHovered = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: 16, weight: .medium)) + .foregroundColor( + configuration.isPressed + ? DS.Colors.accentHover + : isHovered + ? DS.Colors.accentText + : DS.Colors.textSecondary + ) + .padding(.vertical, 8) + .padding(.horizontal, 12) + .background( + Capsule() + .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) + ) + .scaleEffect(configuration.isPressed ? 0.97 : 1.0) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .onHover { hovering in + isHovered = hovering + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } + + private func buttonBackgroundColor(isPressed: Bool) -> Color { + if isPressed { + return DS.Colors.surface3 + } else if isHovered { + return DS.Colors.surface2 + } else { + return Color.clear + } + } +} + +/// Text button — the lowest-emphasis button style. No background on any +/// state, not even hover. Only the text color changes. Used for: "restart", +/// "skip", "cancel", and other truly minimal inline actions where a +/// background would add too much visual weight. +struct DSTextButtonStyle: ButtonStyle { + var fontSize: CGFloat = 14 + + @State private var isHovered = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: fontSize, weight: .medium)) + .foregroundColor( + configuration.isPressed + ? DS.Colors.textPrimary + : isHovered + ? DS.Colors.textPrimary + : DS.Colors.textTertiary + ) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .onHover { hovering in + isHovered = hovering + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } +} + +/// Outlined button — medium emphasis, used where a border helps define +/// the button's bounds. Used for: display selector, copy prompt. +struct DSOutlinedButtonStyle: ButtonStyle { + var isFullWidth: Bool = true + + @State private var isHovered = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: 16, weight: .medium)) + .foregroundColor(DS.Colors.textPrimary) + .frame(maxWidth: isFullWidth ? .infinity : nil) + .padding(.vertical, 12) + .padding(.horizontal, isFullWidth ? 0 : 16) + .background( + Capsule() + .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) + ) + .overlay( + Capsule() + .stroke( + borderColor(isPressed: configuration.isPressed), + lineWidth: 1 + ) + ) + .scaleEffect(configuration.isPressed ? 0.97 : 1.0) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .onHover { hovering in + isHovered = hovering + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } + + private func buttonBackgroundColor(isPressed: Bool) -> Color { + if isPressed { + return DS.Colors.surface3 + } else if isHovered { + return DS.Colors.surface2 + } else { + return DS.Colors.surface1 + } + } + + private func borderColor(isPressed: Bool) -> Color { + if isPressed || isHovered { + return DS.Colors.borderStrong + } else { + return DS.Colors.borderSubtle + } + } +} + +/// Destructive button — for dangerous/irreversible actions (close session, delete). +/// Red-tinted background that intensifies on hover and press. +struct DSDestructiveButtonStyle: ButtonStyle { + @State private var isHovered = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: 16, weight: .medium)) + .foregroundColor( + isHovered || configuration.isPressed + ? .white + : DS.Colors.destructiveText + ) + .padding(.vertical, 10) + .padding(.horizontal, 16) + .background( + Capsule() + .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) + ) + .overlay( + Capsule() + .stroke( + borderColor(isPressed: configuration.isPressed), + lineWidth: 1 + ) + ) + .scaleEffect(configuration.isPressed ? 0.97 : 1.0) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .onHover { hovering in + isHovered = hovering + if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } + } + } + + private func buttonBackgroundColor(isPressed: Bool) -> Color { + if isPressed { + return DS.Colors.destructive.opacity(0.40) + } else if isHovered { + return DS.Colors.destructive.opacity(0.30) + } else { + return DS.Colors.destructive.opacity(0.10) + } + } + + private func borderColor(isPressed: Bool) -> Color { + if isPressed || isHovered { + return DS.Colors.destructive.opacity(0.40) + } else { + return DS.Colors.destructive.opacity(0.15) + } + } +} + +/// Icon-only button — compact circular button for utility actions. +/// Used for: close button (x), send message, small toolbar actions. +struct DSIconButtonStyle: ButtonStyle { + var size: CGFloat = 28 + var isDestructiveOnHover: Bool = false + var tooltipText: String? = nil + + /// Controls horizontal alignment of the tooltip relative to the button. + /// Use `.leading` for buttons near the left edge of the window (tooltip extends right), + /// `.trailing` for buttons near the right edge (tooltip extends left), + /// and `.center` for buttons in the middle. + var tooltipAlignment: Alignment = .center + + @State private var isHovered = false + @State private var isTooltipVisible = false + @State private var tooltipShowWorkItem: DispatchWorkItem? = nil + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(.system(size: size * 0.43, weight: .semibold)) + .foregroundColor(iconColor(isPressed: configuration.isPressed)) + .frame(width: size, height: size) + .background( + Circle() + .fill(circleBackgroundColor(isPressed: configuration.isPressed)) + ) + .overlay( + Circle() + .stroke(circleBorderColor(isPressed: configuration.isPressed), lineWidth: 1) + ) + .scaleEffect(configuration.isPressed ? 0.93 : 1.0) + .animation(.easeOut(duration: DS.Animation.fast), value: configuration.isPressed) + .animation(.easeOut(duration: DS.Animation.fast), value: isHovered) + .contentShape(Circle()) + // Cursor change via AppKit cursor rects — more reliable than NSCursor.push/pop + // because cursor rects are managed at the window level and don't conflict + // with SwiftUI's internal cursor handling. + .overlay(PointerCursorView()) + .onHover { hovering in + isHovered = hovering + // Show the tooltip after a delay (like native tooltips), hide immediately + tooltipShowWorkItem?.cancel() + if hovering { + let workItem = DispatchWorkItem { + withAnimation(.easeOut(duration: 0.15)) { + isTooltipVisible = true + } + } + tooltipShowWorkItem = workItem + DispatchQueue.main.asyncAfter(deadline: .now() + 0.6, execute: workItem) + } else { + withAnimation(.easeOut(duration: 0.1)) { + isTooltipVisible = false + } + } + } + // Custom styled tooltip — positioned above the button with enough gap + // to not overlap the button. Horizontally aligned based on tooltipAlignment + // so tooltips near window edges don't clip outside the visible area. + // Uses .allowsHitTesting(false) so the tooltip doesn't interfere + // with the button's hover state. + .overlay( + Group { + if isTooltipVisible, let text = tooltipText, !text.isEmpty { + Text(text) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .padding(.vertical, 5) + .background( + RoundedRectangle(cornerRadius: 6) + .fill(DS.Colors.surface3.opacity(0.85)) + ) + .overlay( + ZStack { + RoundedRectangle(cornerRadius: 6) + .stroke(Color.white.opacity(0.20), lineWidth: 0.8) + + RoundedRectangle(cornerRadius: 6) + .trim(from: 0, to: 0.5) + .stroke( + LinearGradient( + colors: [ + Color.white.opacity(0.10), + Color.white.opacity(0.02) + ], + startPoint: .top, + endPoint: .bottom + ), + lineWidth: 0.8 + ) + } + ) + .shadow(color: Color.black.opacity(0.42), radius: 14, x: 0, y: 8) + .shadow(color: Color.black.opacity(0.26), radius: 4, x: 0, y: 2) + .fixedSize() + .offset(y: -(size / 2 + 20)) + .allowsHitTesting(false) + .transition(.opacity) + } + }, + alignment: tooltipAlignment + ) + } + + private func iconColor(isPressed: Bool) -> Color { + if isDestructiveOnHover && (isHovered || isPressed) { + return .white + } + if isPressed { + return DS.Colors.textPrimary + } else if isHovered { + return DS.Colors.textPrimary + } else { + return DS.Colors.textSecondary + } + } + + private func circleBackgroundColor(isPressed: Bool) -> Color { + if isDestructiveOnHover { + if isPressed { + return DS.Colors.destructive.opacity(0.40) + } else if isHovered { + return DS.Colors.destructive.opacity(0.30) + } else { + return DS.Colors.surface2 + } + } + if isPressed { + return DS.Colors.surface4 + } else if isHovered { + return DS.Colors.surface3 + } else { + return DS.Colors.surface2 + } + } + + private func circleBorderColor(isPressed: Bool) -> Color { + if isDestructiveOnHover && (isHovered || isPressed) { + return DS.Colors.destructive.opacity(0.30) + } + if isPressed || isHovered { + return DS.Colors.borderStrong + } else { + return DS.Colors.borderSubtle.opacity(0.5) + } + } +} + +// MARK: - Convenience View Extensions + +extension View { + /// Applies the primary button style (accent-colored CTA). + func dsPrimaryButtonStyle(isFullWidth: Bool = true) -> some View { + self.buttonStyle(DSPrimaryButtonStyle(isFullWidth: isFullWidth)) + } + + /// Applies the secondary button style (surface-colored supporting action). + func dsSecondaryButtonStyle(isFullWidth: Bool = true) -> some View { + self.buttonStyle(DSSecondaryButtonStyle(isFullWidth: isFullWidth)) + } + + /// Applies the tertiary/ghost button style (subtle hover background). + func dsTertiaryButtonStyle() -> some View { + self.buttonStyle(DSTertiaryButtonStyle()) + } + + /// Applies the text-only button style (no background ever, just color change). + func dsTextButtonStyle(fontSize: CGFloat = 14) -> some View { + self.buttonStyle(DSTextButtonStyle(fontSize: fontSize)) + } + + /// Applies the outlined button style (bordered, medium emphasis). + func dsOutlinedButtonStyle(isFullWidth: Bool = true) -> some View { + self.buttonStyle(DSOutlinedButtonStyle(isFullWidth: isFullWidth)) + } + + /// Applies the destructive button style (red-tinted danger action). + func dsDestructiveButtonStyle() -> some View { + self.buttonStyle(DSDestructiveButtonStyle()) + } + + /// Applies the icon-only button style (compact circle). + /// `tooltipAlignment` controls where the tooltip sits horizontally relative to the button: + /// `.leading` for left-edge buttons, `.trailing` for right-edge buttons, `.center` for middle. + func dsIconButtonStyle(size: CGFloat = 28, isDestructiveOnHover: Bool = false, tooltip: String? = nil, tooltipAlignment: Alignment = .center) -> some View { + self.buttonStyle(DSIconButtonStyle(size: size, isDestructiveOnHover: isDestructiveOnHover, tooltipText: tooltip, tooltipAlignment: tooltipAlignment)) + } + + /// Attaches the shared pointing-hand cursor treatment used across interactive controls. + /// Disabled controls can opt out so they keep the default arrow cursor. + func pointerCursor(isEnabled: Bool = true) -> some View { + self.overlay { + if isEnabled { + PointerCursorView() + } + } + } +} + +// MARK: - Buddy Composer Visual Style + +enum BuddyComposerVisualStyle { + static let waveformLeadingColor = Color(hex: "#F3FBFF") + static let waveformTrailingColor = Color(hex: "#8FD2FF") + static let waveformGlowColor = Color(hex: "#AEE3FF") +} + +// MARK: - Pointer Cursor (AppKit Bridge) + +/// Uses AppKit's cursor rect system to reliably show a pointing hand cursor. +/// More reliable than NSCursor.push()/pop() inside SwiftUI's .onHover because +/// cursor rects are managed at the window level and don't conflict with +/// SwiftUI's internal cursor handling. +private class PointerCursorNSView: NSView { + override func resetCursorRects() { + super.resetCursorRects() + addCursorRect(bounds, cursor: .pointingHand) + } + + override func hitTest(_ point: NSPoint) -> NSView? { + return nil + } +} + +private struct PointerCursorView: NSViewRepresentable { + func makeNSView(context: Context) -> NSView { + return PointerCursorNSView() + } + + func updateNSView(_ nsView: NSView, context: Context) { + // Invalidate cursor rects when the view updates (e.g., resizes) + // so AppKit recalculates the cursor area. + nsView.window?.invalidateCursorRects(for: nsView) + } +} + +// MARK: - I-Beam Cursor (AppKit Bridge) + +/// Uses AppKit's cursor rect system to reliably show an I-beam (text selection) cursor. +/// Same approach as PointerCursorView — cursor rects are managed at the window level +/// and don't conflict with SwiftUI's internal cursor handling. +/// Unlike NSCursor.push()/pop() in .onHover, this avoids cursor stack imbalance +/// when the mouse moves quickly between views. +private class IBeamCursorNSView: NSView { + override func resetCursorRects() { + super.resetCursorRects() + addCursorRect(bounds, cursor: .iBeam) + } + + /// Pass through all mouse events so the TextField underneath still receives + /// focus, clicks, and text selection. Cursor rects are registered with the + /// window (via resetCursorRects) and work independently of hit testing. + override func hitTest(_ point: NSPoint) -> NSView? { + return nil + } +} + +struct IBeamCursorView: NSViewRepresentable { + func makeNSView(context: Context) -> NSView { + return IBeamCursorNSView() + } + + func updateNSView(_ nsView: NSView, context: Context) { + // Invalidate cursor rects when the view updates (e.g., resizes) + // so AppKit recalculates the cursor area. + nsView.window?.invalidateCursorRects(for: nsView) + } +} + +// MARK: - Native Tooltip + +/// Uses AppKit's `NSView.toolTip` to show a tooltip on hover. +/// SwiftUI's `.help()` conflicts with `.onHover` tracking areas, so +/// this bridges directly to AppKit's tooltip system which works independently. +private struct NativeTooltipView: NSViewRepresentable { + let tooltip: String + + func makeNSView(context: Context) -> NSView { + let view = NSView() + view.toolTip = tooltip + return view + } + + func updateNSView(_ nsView: NSView, context: Context) { + nsView.toolTip = tooltip + } +} + +extension View { + /// Attaches a native macOS tooltip that works even alongside `.onHover`. + func nativeTooltip(_ text: String?) -> some View { + if let text = text, !text.isEmpty { + return AnyView(self.overlay(NativeTooltipView(tooltip: text))) + } else { + return AnyView(self) + } + } +} + +// MARK: - Color Utilities + +extension Color { + /// Create a Color from a hex string like "#FF5733" or "FF5733". + init(hex: String) { + let hexSanitized = hex.trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: "#", with: "") + + var rgbValue: UInt64 = 0 + Scanner(string: hexSanitized).scanHexInt64(&rgbValue) + + let red = Double((rgbValue & 0xFF0000) >> 16) / 255.0 + let green = Double((rgbValue & 0x00FF00) >> 8) / 255.0 + let blue = Double(rgbValue & 0x0000FF) / 255.0 + + self.init(red: red, green: green, blue: blue) + } + + /// Returns a lighter version of this color by blending toward white. + /// `fraction` is 0.0 (no change) to 1.0 (pure white). + func blendedWithWhite(fraction: Double) -> Color { + // Convert to NSColor to access RGB components for blending + guard let nsColor = NSColor(self).usingColorSpace(.sRGB) else { return self } + + let red = nsColor.redComponent + (1.0 - nsColor.redComponent) * fraction + let green = nsColor.greenComponent + (1.0 - nsColor.greenComponent) * fraction + let blue = nsColor.blueComponent + (1.0 - nsColor.blueComponent) * fraction + + return Color(red: red, green: green, blue: blue) + } +} diff --git a/leanring-buddy/ElementLocationDetector.swift b/leanring-buddy/ElementLocationDetector.swift new file mode 100644 index 0000000..47072b1 --- /dev/null +++ b/leanring-buddy/ElementLocationDetector.swift @@ -0,0 +1,335 @@ +// +// ElementLocationDetector.swift +// leanring-buddy +// +// Uses Claude's Computer Use API to identify the screen location of UI elements +// in screenshots. When a user asks about a visible element (e.g., "click the +// blue button"), this detects the element's coordinates so the buddy can +// animate to it and point at it. +// + +import AppKit +import Foundation + +/// Detects the screen location of UI elements in screenshots using Claude's Computer Use API. +/// The Computer Use tool definition activates Claude's specialized pixel-counting training, +/// which is significantly more accurate than regular vision API coordinate extraction. +/// +/// **Aspect ratio matching**: Instead of always resizing to 1024x768 (4:3), we pick the +/// Anthropic-recommended resolution closest to the display's actual aspect ratio. Most +/// Macs are 16:10 → 1280x800. This avoids distorting the image Claude sees, which +/// significantly improves X-axis coordinate accuracy. +class ElementLocationDetector { + private let apiKey: String + private let apiURL: URL + private let model: String + private let session: URLSession + + /// Anthropic-recommended resolutions for Computer Use, paired with their aspect ratios. + /// We pick the one closest to the actual display aspect ratio to avoid distortion. + /// Higher resolutions get downsampled by the API and degrade precision, so these + /// are intentionally small. + private static let supportedComputerUseResolutions: [(width: Int, height: Int, aspectRatio: Double)] = [ + (1024, 768, 1024.0 / 768.0), // 4:3 = 1.333 (legacy displays) + (1280, 800, 1280.0 / 800.0), // 16:10 = 1.600 (MacBook Air, MacBook Pro, most Macs) + (1366, 768, 1366.0 / 768.0) // ~16:9 = 1.779 (external monitors, ultrawide fallback) + ] + + init(apiKey: String, model: String = "claude-sonnet-4-6") { + self.apiKey = apiKey + self.apiURL = URL(string: "https://api.anthropic.com/v1/messages")! + self.model = model + + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 15 + config.timeoutIntervalForResource = 20 + config.waitsForConnectivity = false + config.urlCache = nil + config.httpCookieStorage = nil + self.session = URLSession(configuration: config) + } + + /// Detects the screen location of a UI element the user is asking about. + /// + /// - Parameters: + /// - screenshotData: JPEG or PNG screenshot data from ScreenCaptureKit + /// - userQuestion: The user's voice transcript (e.g., "How do I add a project?") + /// - displayWidthInPoints: The captured display's width in screen points + /// - displayHeightInPoints: The captured display's height in screen points + /// + /// - Returns: A `CGPoint` in display-local macOS coordinates (bottom-left origin) if an + /// element was identified, or `nil` if no element was found or detection failed. + func detectElementLocation( + screenshotData: Data, + userQuestion: String, + displayWidthInPoints: Int, + displayHeightInPoints: Int + ) async -> CGPoint? { + // Pick the Computer Use resolution that best matches this display's aspect ratio. + // This avoids stretching the screenshot (e.g., squishing a 16:10 Mac display + // into 4:3), which would distort the image Claude sees and degrade X-axis accuracy. + let computerUseResolution = bestComputerUseResolution( + forDisplayWidth: displayWidthInPoints, + displayHeight: displayHeightInPoints + ) + + print("🎯 ElementLocationDetector: display is \(displayWidthInPoints)x\(displayHeightInPoints) " + + "(ratio \(String(format: "%.3f", Double(displayWidthInPoints) / Double(displayHeightInPoints)))), " + + "using Computer Use resolution \(computerUseResolution.width)x\(computerUseResolution.height)") + + // Resize the screenshot to the chosen Computer Use resolution + guard let resizedScreenshotData = resizeScreenshotForComputerUse( + originalImageData: screenshotData, + targetWidth: computerUseResolution.width, + targetHeight: computerUseResolution.height + ) else { + print("⚠️ ElementLocationDetector: failed to resize screenshot") + return nil + } + + // Make the Computer Use API call with the matching resolution declared + guard let computerUseCoordinate = await callComputerUseAPI( + resizedScreenshotData: resizedScreenshotData, + userQuestion: userQuestion, + declaredDisplayWidth: computerUseResolution.width, + declaredDisplayHeight: computerUseResolution.height + ) else { + return nil + } + + // Clamp coordinates to the valid range — Claude occasionally returns + // values slightly outside the declared display dimensions, which would + // map to off-screen positions after scaling. + let clampedX = max(0, min(computerUseCoordinate.x, CGFloat(computerUseResolution.width))) + let clampedY = max(0, min(computerUseCoordinate.y, CGFloat(computerUseResolution.height))) + + // Scale coordinates from the Computer Use resolution back to actual display point dimensions + let scaledX = (clampedX / CGFloat(computerUseResolution.width)) * CGFloat(displayWidthInPoints) + let scaledYTopLeftOrigin = (clampedY / CGFloat(computerUseResolution.height)) * CGFloat(displayHeightInPoints) + + // Convert from top-left origin (Computer Use / CoreGraphics) to bottom-left origin (AppKit) + let scaledYBottomLeftOrigin = CGFloat(displayHeightInPoints) - scaledYTopLeftOrigin + + print("🎯 ElementLocationDetector: mapped (\(Int(clampedX)), \(Int(clampedY))) in " + + "\(computerUseResolution.width)x\(computerUseResolution.height) → " + + "(\(Int(scaledX)), \(Int(scaledYBottomLeftOrigin))) in " + + "\(displayWidthInPoints)x\(displayHeightInPoints) display-local AppKit coords") + + return CGPoint(x: scaledX, y: scaledYBottomLeftOrigin) + } + + // MARK: - Private Helpers + + /// Picks the Anthropic-recommended Computer Use resolution whose aspect ratio + /// is closest to the actual display, minimizing image distortion. + private func bestComputerUseResolution( + forDisplayWidth displayWidth: Int, + displayHeight: Int + ) -> (width: Int, height: Int) { + let displayAspectRatio = Double(displayWidth) / Double(max(1, displayHeight)) + + var bestWidth = 1280 + var bestHeight = 800 + var smallestAspectRatioDifference = Double.greatestFiniteMagnitude + + for resolution in Self.supportedComputerUseResolutions { + let difference = abs(displayAspectRatio - resolution.aspectRatio) + if difference < smallestAspectRatioDifference { + smallestAspectRatioDifference = difference + bestWidth = resolution.width + bestHeight = resolution.height + } + } + + return (width: bestWidth, height: bestHeight) + } + + /// Calls the Claude Computer Use API with a resized screenshot and user question. + /// Returns the raw coordinate from Claude's response in the declared resolution space, or nil. + private func callComputerUseAPI( + resizedScreenshotData: Data, + userQuestion: String, + declaredDisplayWidth: Int, + declaredDisplayHeight: Int + ) async -> CGPoint? { + var request = URLRequest(url: apiURL) + request.httpMethod = "POST" + request.timeoutInterval = 15 + request.setValue(apiKey, forHTTPHeaderField: "x-api-key") + request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + // The beta header activates Computer Use capabilities and the specialized + // pixel-counting training that makes coordinate detection accurate. + request.setValue("computer-use-2025-11-24", forHTTPHeaderField: "anthropic-beta") + + // Detect image media type (PNG vs JPEG) + let mediaType = detectImageMediaType(for: resizedScreenshotData) + let base64Screenshot = resizedScreenshotData.base64EncodedString() + + let userPrompt = """ + The user asked this question while looking at their screen: "\(userQuestion)" + + Look at the screenshot. If there is a specific UI element (button, link, menu item, text field, icon, etc.) that the user should interact with or is asking about, click on that element. + + If the question is purely conceptual (e.g., "what does HTML mean?") and there's no specific element to point to, just respond with text saying "no specific element". + """ + + let body: [String: Any] = [ + "model": model, + "max_tokens": 256, + "tools": [ + [ + "type": "computer_20251124", + "name": "computer", + "display_width_px": declaredDisplayWidth, + "display_height_px": declaredDisplayHeight + ] + ], + "messages": [ + [ + "role": "user", + "content": [ + [ + "type": "image", + "source": [ + "type": "base64", + "media_type": mediaType, + "data": base64Screenshot + ] + ], + [ + "type": "text", + "text": userPrompt + ] + ] + ] + ] + ] + + do { + let bodyData = try JSONSerialization.data(withJSONObject: body) + request.httpBody = bodyData + + let payloadMB = Double(bodyData.count) / 1_048_576.0 + print("🎯 ElementLocationDetector: sending \(String(format: "%.1f", payloadMB))MB request " + + "(declared \(declaredDisplayWidth)x\(declaredDisplayHeight))") + + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + let errorBody = String(data: data, encoding: .utf8) ?? "unknown" + print("⚠️ ElementLocationDetector: API error \(statusCode): \(errorBody.prefix(200))") + return nil + } + + return parseCoordinateFromResponse(data: data) + + } catch { + print("⚠️ ElementLocationDetector: request failed: \(error.localizedDescription)") + return nil + } + } + + /// Parses the Computer Use API response to extract click coordinates. + /// Claude returns a `tool_use` content block with `{"action": "left_click", "coordinate": [x, y]}`. + /// If Claude returns text instead (no element found), returns nil. + private func parseCoordinateFromResponse(data: Data) -> CGPoint? { + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let contentBlocks = json["content"] as? [[String: Any]] else { + print("⚠️ ElementLocationDetector: could not parse response JSON") + return nil + } + + // Look for a tool_use content block (Claude's Computer Use response format) + for block in contentBlocks { + guard let blockType = block["type"] as? String, + blockType == "tool_use", + let input = block["input"] as? [String: Any], + let coordinate = input["coordinate"] as? [NSNumber], + coordinate.count == 2 else { + continue + } + + let x = CGFloat(coordinate[0].doubleValue) + let y = CGFloat(coordinate[1].doubleValue) + print("🎯 ElementLocationDetector: raw coordinate (\(Int(x)), \(Int(y)))") + return CGPoint(x: x, y: y) + } + + // No tool_use block found — Claude responded with text (no element to point at) + print("🎯 ElementLocationDetector: no specific element detected (conceptual question)") + return nil + } + + /// Resizes screenshot data to the specified Computer Use resolution. + /// The target resolution should match the display's aspect ratio to avoid + /// distortion that degrades coordinate accuracy. + /// + /// **Critical Retina fix**: Uses `NSBitmapImageRep` directly instead of + /// `NSImage.lockFocus()`. On Retina displays (2x backing scale), lockFocus + /// creates a bitmap at 2× the declared size (e.g., 2560×1600 for a 1280×800 + /// NSImage). This means the JPEG sent to Claude would be 2× larger than the + /// resolution declared in the Computer Use tool definition, causing Claude's + /// pixel-counting to return coordinates in the wrong scale. + private func resizeScreenshotForComputerUse( + originalImageData: Data, + targetWidth: Int, + targetHeight: Int + ) -> Data? { + guard let originalImage = NSImage(data: originalImageData) else { return nil } + + // Create a bitmap representation with exact pixel dimensions. + // This bypasses NSImage's Retina-aware coordinate system which would + // otherwise double the actual pixel count on 2x displays. + guard let bitmapRep = NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: targetWidth, + pixelsHigh: targetHeight, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) else { + return nil + } + + // Set the point size to match pixel dimensions (1:1, no Retina scaling). + bitmapRep.size = NSSize(width: targetWidth, height: targetHeight) + + // Draw the original image into the exact-pixel-dimension bitmap + NSGraphicsContext.saveGraphicsState() + let graphicsContext = NSGraphicsContext(bitmapImageRep: bitmapRep) + NSGraphicsContext.current = graphicsContext + graphicsContext?.imageInterpolation = .high + originalImage.draw( + in: NSRect(x: 0, y: 0, width: targetWidth, height: targetHeight), + from: NSRect(origin: .zero, size: originalImage.size), + operation: .copy, + fraction: 1.0 + ) + NSGraphicsContext.restoreGraphicsState() + + guard let jpegData = bitmapRep.representation(using: .jpeg, properties: [.compressionFactor: 0.85]) else { + return nil + } + + return jpegData + } + + /// Detects MIME type by inspecting the first bytes of image data. + private func detectImageMediaType(for imageData: Data) -> String { + if imageData.count >= 4 { + let pngSignature: [UInt8] = [0x89, 0x50, 0x4E, 0x47] + let firstFourBytes = [UInt8](imageData.prefix(4)) + if firstFourBytes == pngSignature { + return "image/png" + } + } + return "image/jpeg" + } +} diff --git a/leanring-buddy/ElevenLabsTTSClient.swift b/leanring-buddy/ElevenLabsTTSClient.swift new file mode 100644 index 0000000..35545c9 --- /dev/null +++ b/leanring-buddy/ElevenLabsTTSClient.swift @@ -0,0 +1,81 @@ +// +// ElevenLabsTTSClient.swift +// leanring-buddy +// +// Streams text-to-speech audio from ElevenLabs and plays it back +// through the system audio output. Uses the streaming endpoint so +// playback begins before the full audio has been generated. +// + +import AVFoundation +import Foundation + +@MainActor +final class ElevenLabsTTSClient { + private let proxyURL: URL + private let session: URLSession + + /// The audio player for the current TTS playback. Kept alive so the + /// audio finishes playing even if the caller doesn't hold a reference. + private var audioPlayer: AVAudioPlayer? + + init(proxyURL: String) { + self.proxyURL = URL(string: proxyURL)! + + let configuration = URLSessionConfiguration.default + configuration.timeoutIntervalForRequest = 30 + configuration.timeoutIntervalForResource = 60 + self.session = URLSession(configuration: configuration) + } + + /// Sends `text` to ElevenLabs TTS and plays the resulting audio. + /// Throws on network or decoding errors. Cancellation-safe. + func speakText(_ text: String) async throws { + var request = URLRequest(url: proxyURL) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("audio/mpeg", forHTTPHeaderField: "Accept") + + let body: [String: Any] = [ + "text": text, + "model_id": "eleven_flash_v2_5", + "voice_settings": [ + "stability": 0.5, + "similarity_boost": 0.75 + ] + ] + + request.httpBody = try JSONSerialization.data(withJSONObject: body) + + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse else { + throw NSError(domain: "ElevenLabsTTS", code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid response"]) + } + + guard (200...299).contains(httpResponse.statusCode) else { + let errorBody = String(data: data, encoding: .utf8) ?? "Unknown error" + throw NSError(domain: "ElevenLabsTTS", code: httpResponse.statusCode, + userInfo: [NSLocalizedDescriptionKey: "TTS API error (\(httpResponse.statusCode)): \(errorBody)"]) + } + + try Task.checkCancellation() + + let player = try AVAudioPlayer(data: data) + self.audioPlayer = player + player.play() + print("🔊 ElevenLabs TTS: playing \(data.count / 1024)KB audio") + } + + /// Whether TTS audio is currently playing back. + var isPlaying: Bool { + audioPlayer?.isPlaying ?? false + } + + /// Stops any in-progress playback immediately. + func stopPlayback() { + audioPlayer?.stop() + audioPlayer = nil + } +} diff --git a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift new file mode 100644 index 0000000..8020269 --- /dev/null +++ b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift @@ -0,0 +1,132 @@ +// +// GlobalPushToTalkShortcutMonitor.swift +// leanring-buddy +// +// Captures push-to-talk keyboard shortcuts while makesomething is running in the +// background. Uses a listen-only CGEvent tap so modifier-only shortcuts like +// ctrl + option behave more like a real system-wide voice tool. +// + +import AppKit +import Combine +import CoreGraphics +import Foundation + +final class GlobalPushToTalkShortcutMonitor: ObservableObject { + let shortcutTransitionPublisher = PassthroughSubject() + + private var globalEventTap: CFMachPort? + private var globalEventTapRunLoopSource: CFRunLoopSource? + /// Mutated exclusively from the CGEvent tap callback, which runs on + /// `CFRunLoopGetMain()` and therefore always executes on the main thread. + /// Published so the overlay can hide immediately on key release without + /// waiting for the async dictation state pipeline to catch up. + @Published private(set) var isShortcutCurrentlyPressed = false + + deinit { + stop() + } + + func start() { + // If the event tap is already running, don't restart it. + // Restarting resets isShortcutCurrentlyPressed, which would kill + // the waveform overlay mid-press when the permission poller calls + // refreshAllPermissions → start() every few seconds. + guard globalEventTap == nil else { return } + + let monitoredEventTypes: [CGEventType] = [.flagsChanged, .keyDown, .keyUp] + let eventMask = monitoredEventTypes.reduce(CGEventMask(0)) { currentMask, eventType in + currentMask | (CGEventMask(1) << eventType.rawValue) + } + + let eventTapCallback: CGEventTapCallBack = { _, eventType, event, userInfo in + guard let userInfo else { + return Unmanaged.passUnretained(event) + } + + let globalPushToTalkShortcutMonitor = Unmanaged + .fromOpaque(userInfo) + .takeUnretainedValue() + + return globalPushToTalkShortcutMonitor.handleGlobalEventTap( + eventType: eventType, + event: event + ) + } + + guard let globalEventTap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: eventMask, + callback: eventTapCallback, + userInfo: Unmanaged.passUnretained(self).toOpaque() + ) else { + print("⚠️ Global push-to-talk: couldn't create CGEvent tap") + return + } + + guard let globalEventTapRunLoopSource = CFMachPortCreateRunLoopSource( + kCFAllocatorDefault, + globalEventTap, + 0 + ) else { + CFMachPortInvalidate(globalEventTap) + print("⚠️ Global push-to-talk: couldn't create event tap run loop source") + return + } + + self.globalEventTap = globalEventTap + self.globalEventTapRunLoopSource = globalEventTapRunLoopSource + + CFRunLoopAddSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) + CGEvent.tapEnable(tap: globalEventTap, enable: true) + } + + func stop() { + isShortcutCurrentlyPressed = false + + if let globalEventTapRunLoopSource { + CFRunLoopRemoveSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) + self.globalEventTapRunLoopSource = nil + } + + if let globalEventTap { + CFMachPortInvalidate(globalEventTap) + self.globalEventTap = nil + } + } + + private func handleGlobalEventTap( + eventType: CGEventType, + event: CGEvent + ) -> Unmanaged? { + if eventType == .tapDisabledByTimeout || eventType == .tapDisabledByUserInput { + if let globalEventTap { + CGEvent.tapEnable(tap: globalEventTap, enable: true) + } + return Unmanaged.passUnretained(event) + } + + let eventKeyCode = UInt16(event.getIntegerValueField(.keyboardEventKeycode)) + let shortcutTransition = BuddyPushToTalkShortcut.shortcutTransition( + for: eventType, + keyCode: eventKeyCode, + modifierFlagsRawValue: event.flags.rawValue, + wasShortcutPreviouslyPressed: isShortcutCurrentlyPressed + ) + + switch shortcutTransition { + case .none: + break + case .pressed: + isShortcutCurrentlyPressed = true + shortcutTransitionPublisher.send(.pressed) + case .released: + isShortcutCurrentlyPressed = false + shortcutTransitionPublisher.send(.released) + } + + return Unmanaged.passUnretained(event) + } +} diff --git a/leanring-buddy/Info.plist b/leanring-buddy/Info.plist new file mode 100644 index 0000000..e3d2b45 --- /dev/null +++ b/leanring-buddy/Info.plist @@ -0,0 +1,20 @@ + + + + + LSUIElement + + SUFeedURL + https://raw.githubusercontent.com/julianjear/makesomething-mac-app/main/appcast.xml + SUPublicEDKey + /l3d2rw5ZZFRU3AadP/w2Zf8FHfhA6bKv16BQOV5OSk= + VoiceTranscriptionProvider + assemblyai + NSMicrophoneUsageDescription + Clicky uses your microphone so you can talk to it + NSScreenCaptureUsageDescription + Clicky needs screen recording access to see your screen and help you. + NSSpeechRecognitionUsageDescription + Clicky uses speech recognition to transcribe your voice when you talk to it + + diff --git a/leanring-buddy/MenuBarPanelManager.swift b/leanring-buddy/MenuBarPanelManager.swift new file mode 100644 index 0000000..e5eb98d --- /dev/null +++ b/leanring-buddy/MenuBarPanelManager.swift @@ -0,0 +1,243 @@ +// +// MenuBarPanelManager.swift +// leanring-buddy +// +// Manages the NSStatusItem (menu bar icon) and a custom borderless NSPanel +// that drops down below it when clicked. The panel hosts a SwiftUI view +// (CompanionPanelView) via NSHostingView. Uses the same NSPanel pattern as +// FloatingSessionButton and GlobalPushToTalkOverlay for consistency. +// +// The panel is non-activating so it does not steal focus from the user's +// current app, and auto-dismisses when the user clicks outside. +// + +import AppKit +import SwiftUI + +extension Notification.Name { + static let clickyDismissPanel = Notification.Name("clickyDismissPanel") +} + +/// Custom NSPanel subclass that can become the key window even with +/// .nonactivatingPanel style, allowing text fields to receive focus. +private class KeyablePanel: NSPanel { + override var canBecomeKey: Bool { true } +} + +@MainActor +final class MenuBarPanelManager: NSObject { + private var statusItem: NSStatusItem? + private var panel: NSPanel? + private var clickOutsideMonitor: Any? + private var dismissPanelObserver: NSObjectProtocol? + + private let companionManager: CompanionManager + private let panelWidth: CGFloat = 320 + private let panelHeight: CGFloat = 380 + + init(companionManager: CompanionManager) { + self.companionManager = companionManager + super.init() + createStatusItem() + + dismissPanelObserver = NotificationCenter.default.addObserver( + forName: .clickyDismissPanel, + object: nil, + queue: .main + ) { [weak self] _ in + self?.hidePanel() + } + } + + deinit { + if let monitor = clickOutsideMonitor { + NSEvent.removeMonitor(monitor) + } + if let observer = dismissPanelObserver { + NotificationCenter.default.removeObserver(observer) + } + } + + // MARK: - Status Item + + private func createStatusItem() { + statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.squareLength) + + guard let button = statusItem?.button else { return } + + button.image = makeClickyMenuBarIcon() + button.image?.isTemplate = true + button.action = #selector(statusItemClicked) + button.target = self + } + + /// Draws the clicky triangle as a menu bar icon. Uses the same shape + /// and rotation as the in-app cursor so the menu bar icon matches. + private func makeClickyMenuBarIcon() -> NSImage { + let iconSize: CGFloat = 18 + let image = NSImage(size: NSSize(width: iconSize, height: iconSize)) + image.lockFocus() + + let triangleSize = iconSize * 0.7 + let cx = iconSize * 0.50 + let cy = iconSize * 0.50 + let height = triangleSize * sqrt(3.0) / 2.0 + + let top = CGPoint(x: cx, y: cy + height / 1.5) + let bottomLeft = CGPoint(x: cx - triangleSize / 2, y: cy - height / 3) + let bottomRight = CGPoint(x: cx + triangleSize / 2, y: cy - height / 3) + + let angle = 35.0 * .pi / 180.0 + func rotate(_ point: CGPoint) -> CGPoint { + let dx = point.x - cx, dy = point.y - cy + let cosA = CGFloat(cos(angle)), sinA = CGFloat(sin(angle)) + return CGPoint(x: cx + cosA * dx - sinA * dy, y: cy + sinA * dx + cosA * dy) + } + + let path = NSBezierPath() + path.move(to: rotate(top)) + path.line(to: rotate(bottomLeft)) + path.line(to: rotate(bottomRight)) + path.close() + + NSColor.black.setFill() + path.fill() + + image.unlockFocus() + return image + } + + /// Opens the panel automatically on app launch so the user sees + /// permissions and the start button right away. + func showPanelOnLaunch() { + // Small delay so the status item has time to appear in the menu bar + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + self.showPanel() + } + } + + @objc private func statusItemClicked() { + if let panel, panel.isVisible { + hidePanel() + } else { + showPanel() + } + } + + // MARK: - Panel Lifecycle + + private func showPanel() { + if panel == nil { + createPanel() + } + + positionPanelBelowStatusItem() + + panel?.makeKeyAndOrderFront(nil) + panel?.orderFrontRegardless() + installClickOutsideMonitor() + } + + private func hidePanel() { + panel?.orderOut(nil) + removeClickOutsideMonitor() + } + + private func createPanel() { + let companionPanelView = CompanionPanelView(companionManager: companionManager) + .frame(width: panelWidth) + + let hostingView = NSHostingView(rootView: companionPanelView) + hostingView.frame = NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight) + hostingView.wantsLayer = true + hostingView.layer?.backgroundColor = .clear + + let menuBarPanel = KeyablePanel( + contentRect: NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + + menuBarPanel.isFloatingPanel = true + menuBarPanel.level = .floating + menuBarPanel.isOpaque = false + menuBarPanel.backgroundColor = .clear + menuBarPanel.hasShadow = false + menuBarPanel.hidesOnDeactivate = false + menuBarPanel.isExcludedFromWindowsMenu = true + menuBarPanel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + menuBarPanel.isMovableByWindowBackground = false + menuBarPanel.titleVisibility = .hidden + menuBarPanel.titlebarAppearsTransparent = true + + menuBarPanel.contentView = hostingView + panel = menuBarPanel + } + + private func positionPanelBelowStatusItem() { + guard let panel else { return } + guard let buttonWindow = statusItem?.button?.window else { return } + + let statusItemFrame = buttonWindow.frame + let gapBelowMenuBar: CGFloat = 4 + + // Calculate the panel's content height from the hosting view's fitting size + // so the panel snugly wraps the SwiftUI content instead of using a fixed height. + let fittingSize = panel.contentView?.fittingSize ?? CGSize(width: panelWidth, height: panelHeight) + let actualPanelHeight = fittingSize.height + + // Horizontally center the panel beneath the status item icon + let panelOriginX = statusItemFrame.midX - (panelWidth / 2) + let panelOriginY = statusItemFrame.minY - actualPanelHeight - gapBelowMenuBar + + panel.setFrame( + NSRect(x: panelOriginX, y: panelOriginY, width: panelWidth, height: actualPanelHeight), + display: true + ) + } + + // MARK: - Click Outside Dismissal + + /// Installs a global event monitor that hides the panel when the user clicks + /// anywhere outside it — the same transient dismissal behavior as NSPopover. + /// Uses a short delay so that system permission dialogs (triggered by Grant + /// buttons in the panel) don't immediately dismiss the panel when they appear. + private func installClickOutsideMonitor() { + removeClickOutsideMonitor() + + clickOutsideMonitor = NSEvent.addGlobalMonitorForEvents( + matching: [.leftMouseDown, .rightMouseDown] + ) { [weak self] event in + guard let self, let panel = self.panel else { return } + + // Check if the click is inside the status item button — if so, the + // statusItemClicked handler will toggle the panel, so don't also hide. + let clickLocation = NSEvent.mouseLocation + if panel.frame.contains(clickLocation) { + return + } + + // Delay dismissal slightly to avoid closing the panel when + // a system permission dialog appears (e.g. microphone access). + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + guard panel.isVisible else { return } + + // If permissions aren't all granted yet, a system dialog + // may have focus — don't dismiss during onboarding. + if !self.companionManager.allPermissionsGranted && !NSApp.isActive { + return + } + + self.hidePanel() + } + } + } + + private func removeClickOutsideMonitor() { + if let monitor = clickOutsideMonitor { + NSEvent.removeMonitor(monitor) + clickOutsideMonitor = nil + } + } +} diff --git a/leanring-buddy/OpenAIAPI.swift b/leanring-buddy/OpenAIAPI.swift new file mode 100644 index 0000000..d0c3f2a --- /dev/null +++ b/leanring-buddy/OpenAIAPI.swift @@ -0,0 +1,142 @@ +// +// OpenAIAPI.swift +// OpenAI API Implementation +// + +import Foundation + +/// OpenAI API helper for vision analysis +class OpenAIAPI { + private let apiKey: String + private let apiURL: URL + private let model: String + private let session: URLSession + + init(apiKey: String, model: String = "gpt-5.2-2025-12-11") { + self.apiKey = apiKey + self.apiURL = URL(string: "https://api.openai.com/v1/chat/completions")! + self.model = model + + // Use .default instead of .ephemeral so TLS session tickets are cached. + // Ephemeral sessions do a full TLS handshake on every request, which causes + // transient -1200 (errSSLPeerHandshakeFail) errors with large image payloads. + // Disable URL/cookie caching to avoid storing responses or credentials on disk. + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 120 + config.timeoutIntervalForResource = 300 + config.waitsForConnectivity = true + config.urlCache = nil + config.httpCookieStorage = nil + self.session = URLSession(configuration: config) + + // Fire a lightweight HEAD request in the background to pre-establish the TLS + // connection. This caches the TLS session ticket so the first real API call + // (which carries a large image payload) doesn't need a cold TLS handshake. + warmUpTLSConnection() + } + + /// Sends a no-op HEAD request to the API host to establish and cache a TLS session. + /// Failures are silently ignored — this is purely an optimization. + private func warmUpTLSConnection() { + var warmupRequest = URLRequest(url: apiURL) + warmupRequest.httpMethod = "HEAD" + warmupRequest.timeoutInterval = 10 + session.dataTask(with: warmupRequest) { _, _, _ in + // Response doesn't matter — the TLS handshake is the goal + }.resume() + } + + /// Send a vision request to OpenAI with one or more labeled images. + func analyzeImage( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userPlaceholder: String, assistantResponse: String)] = [], + userPrompt: String + ) async throws -> (text: String, duration: TimeInterval) { + let startTime = Date() + + // Build request + var request = URLRequest(url: apiURL) + request.httpMethod = "POST" + request.timeoutInterval = 120 + request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + // Build messages array + var messages: [[String: Any]] = [] + + // Add system message first + messages.append([ + "role": "system", + "content": systemPrompt + ]) + + // Add conversation history + for (userPlaceholder, assistantResponse) in conversationHistory { + messages.append(["role": "user", "content": userPlaceholder]) + messages.append(["role": "assistant", "content": assistantResponse]) + } + + // Build current message with all labeled images + prompt + var contentBlocks: [[String: Any]] = [] + for image in images { + contentBlocks.append([ + "type": "text", + "text": image.label + ]) + contentBlocks.append([ + "type": "image_url", + "image_url": [ + "url": "data:image/jpeg;base64,\(image.data.base64EncodedString())" + ] + ]) + } + contentBlocks.append([ + "type": "text", + "text": userPrompt + ]) + messages.append(["role": "user", "content": contentBlocks]) + + // Build request body + let body: [String: Any] = [ + "model": model, + // `max_tokens` is deprecated/incompatible for some newer OpenAI models. + "max_completion_tokens": 600, + "messages": messages + ] + + let bodyData = try JSONSerialization.data(withJSONObject: body) + request.httpBody = bodyData + let payloadMB = Double(bodyData.count) / 1_048_576.0 + print("🌐 OpenAI request: \(String(format: "%.1f", payloadMB))MB, \(images.count) image(s)") + + // Send request + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let responseString = String(data: data, encoding: .utf8) ?? "Unknown error" + throw NSError( + domain: "OpenAIAPI", + code: (response as? HTTPURLResponse)?.statusCode ?? -1, + userInfo: [NSLocalizedDescriptionKey: "API Error: \(responseString)"] + ) + } + + // Parse response + let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] + guard let choices = json?["choices"] as? [[String: Any]], + let firstChoice = choices.first, + let message = firstChoice["message"] as? [String: Any], + let text = message["content"] as? String else { + throw NSError( + domain: "OpenAIAPI", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid response format"] + ) + } + + let duration = Date().timeIntervalSince(startTime) + return (text: text, duration: duration) + } +} diff --git a/leanring-buddy/OpenAIAudioTranscriptionProvider.swift b/leanring-buddy/OpenAIAudioTranscriptionProvider.swift new file mode 100644 index 0000000..7509209 --- /dev/null +++ b/leanring-buddy/OpenAIAudioTranscriptionProvider.swift @@ -0,0 +1,317 @@ +// +// OpenAIAudioTranscriptionProvider.swift +// leanring-buddy +// +// AI transcription provider backed by OpenAI's audio transcription API. +// + +import AVFoundation +import Foundation + +struct OpenAIAudioTranscriptionProviderError: LocalizedError { + let message: String + + var errorDescription: String? { + message + } +} + +final class OpenAIAudioTranscriptionProvider: BuddyTranscriptionProvider { + private let apiKey = AppBundleConfiguration.stringValue(forKey: "OpenAIAPIKey") + private let modelName = AppBundleConfiguration.stringValue(forKey: "OpenAITranscriptionModel") + ?? "gpt-4o-transcribe" + + let displayName = "OpenAI" + let requiresSpeechRecognitionPermission = false + + var isConfigured: Bool { + apiKey != nil + } + + var unavailableExplanation: String? { + guard !isConfigured else { return nil } + return "OpenAI transcription is not configured. Add OpenAIAPIKey to Info.plist." + } + + func startStreamingSession( + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) async throws -> any BuddyStreamingTranscriptionSession { + guard let apiKey else { + throw OpenAIAudioTranscriptionProviderError( + message: unavailableExplanation ?? "OpenAI transcription is not configured." + ) + } + + return OpenAIAudioTranscriptionSession( + apiKey: apiKey, + modelName: modelName, + keyterms: keyterms, + onTranscriptUpdate: onTranscriptUpdate, + onFinalTranscriptReady: onFinalTranscriptReady, + onError: onError + ) + } +} + +private final class OpenAIAudioTranscriptionSession: BuddyStreamingTranscriptionSession { + let finalTranscriptFallbackDelaySeconds: TimeInterval = 8.0 + + private struct TranscriptionResponse: Decodable { + let text: String + } + + private static let transcriptionURL = URL(string: "https://api.openai.com/v1/audio/transcriptions")! + private static let targetSampleRate = 16_000 + + private let apiKey: String + private let modelName: String + private let keyterms: [String] + private let onTranscriptUpdate: (String) -> Void + private let onFinalTranscriptReady: (String) -> Void + private let onError: (Error) -> Void + + private let stateQueue = DispatchQueue(label: "com.learningbuddy.openai.transcription") + private let audioPCM16Converter = BuddyPCM16AudioConverter( + targetSampleRate: Double(targetSampleRate) + ) + private let urlSession: URLSession + + private var bufferedPCM16AudioData = Data() + private var hasRequestedFinalTranscript = false + private var hasDeliveredFinalTranscript = false + private var isCancelled = false + private var transcriptionUploadTask: Task? + + init( + apiKey: String, + modelName: String, + keyterms: [String], + onTranscriptUpdate: @escaping (String) -> Void, + onFinalTranscriptReady: @escaping (String) -> Void, + onError: @escaping (Error) -> Void + ) { + self.apiKey = apiKey + self.modelName = modelName + self.keyterms = keyterms + self.onTranscriptUpdate = onTranscriptUpdate + self.onFinalTranscriptReady = onFinalTranscriptReady + self.onError = onError + + let urlSessionConfiguration = URLSessionConfiguration.default + urlSessionConfiguration.timeoutIntervalForRequest = 45 + urlSessionConfiguration.timeoutIntervalForResource = 90 + urlSessionConfiguration.waitsForConnectivity = true + self.urlSession = URLSession(configuration: urlSessionConfiguration) + } + + func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { + guard let audioPCM16Data = audioPCM16Converter.convertToPCM16Data(from: audioBuffer), + !audioPCM16Data.isEmpty else { + return + } + + stateQueue.async { + guard !self.hasRequestedFinalTranscript, !self.isCancelled else { return } + self.bufferedPCM16AudioData.append(audioPCM16Data) + } + } + + func requestFinalTranscript() { + stateQueue.async { + guard !self.hasRequestedFinalTranscript, !self.isCancelled else { return } + self.hasRequestedFinalTranscript = true + + let bufferedPCM16AudioData = self.bufferedPCM16AudioData + self.transcriptionUploadTask = Task { [weak self] in + await self?.transcribeBufferedAudio(bufferedPCM16AudioData) + } + } + } + + func cancel() { + stateQueue.async { + self.isCancelled = true + self.bufferedPCM16AudioData.removeAll(keepingCapacity: false) + } + + transcriptionUploadTask?.cancel() + urlSession.invalidateAndCancel() + } + + private func transcribeBufferedAudio(_ bufferedPCM16AudioData: Data) async { + guard !Task.isCancelled else { return } + + let trimmedAudioDataIsEmpty = stateQueue.sync { + isCancelled || bufferedPCM16AudioData.isEmpty + } + + if trimmedAudioDataIsEmpty { + deliverFinalTranscript("") + return + } + + let wavAudioData = BuddyWAVFileBuilder.buildWAVData( + fromPCM16MonoAudio: bufferedPCM16AudioData, + sampleRate: Self.targetSampleRate + ) + + do { + let transcriptText = try await requestTranscription(for: wavAudioData) + guard !stateQueue.sync(execute: { isCancelled }) else { return } + + if !transcriptText.isEmpty { + onTranscriptUpdate(transcriptText) + } + + deliverFinalTranscript(transcriptText) + } catch { + guard !stateQueue.sync(execute: { isCancelled }) else { return } + print("[OpenAI Transcription] ❌ Upload failed (audio size: \(wavAudioData.count) bytes): \(error.localizedDescription)") + onError(error) + } + } + + private func requestTranscription(for wavAudioData: Data) async throws -> String { + let multipartBoundary = "Boundary-\(UUID().uuidString)" + var request = URLRequest(url: Self.transcriptionURL) + request.httpMethod = "POST" + request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.setValue("multipart/form-data; boundary=\(multipartBoundary)", forHTTPHeaderField: "Content-Type") + + let requestBodyData = makeMultipartRequestBody( + boundary: multipartBoundary, + wavAudioData: wavAudioData + ) + request.httpBody = requestBodyData + + let (responseData, response) = try await urlSession.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse else { + throw OpenAIAudioTranscriptionProviderError( + message: "OpenAI transcription returned an invalid response." + ) + } + + guard (200...299).contains(httpResponse.statusCode) else { + let responseText = String(data: responseData, encoding: .utf8) ?? "Unknown error" + throw OpenAIAudioTranscriptionProviderError( + message: "OpenAI transcription failed: \(responseText)" + ) + } + + if let transcriptionResponse = try? JSONDecoder().decode( + TranscriptionResponse.self, + from: responseData + ) { + return transcriptionResponse.text.trimmingCharacters(in: .whitespacesAndNewlines) + } + + let responseText = String(data: responseData, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + + if !responseText.isEmpty { + return responseText + } + + throw OpenAIAudioTranscriptionProviderError( + message: "OpenAI transcription returned an empty transcript." + ) + } + + private func makeMultipartRequestBody( + boundary: String, + wavAudioData: Data + ) -> Data { + var requestBodyData = Data() + + requestBodyData.appendMultipartFormField( + named: "model", + value: modelName, + usingBoundary: boundary + ) + requestBodyData.appendMultipartFormField( + named: "language", + value: "en", + usingBoundary: boundary + ) + requestBodyData.appendMultipartFormField( + named: "response_format", + value: "json", + usingBoundary: boundary + ) + + if let contextualPrompt = transcriptionPromptText() { + requestBodyData.appendMultipartFormField( + named: "prompt", + value: contextualPrompt, + usingBoundary: boundary + ) + } + + requestBodyData.appendMultipartFileField( + named: "file", + filename: "voice-input.wav", + mimeType: "audio/wav", + fileData: wavAudioData, + usingBoundary: boundary + ) + requestBodyData.appendString("--\(boundary)--\r\n") + + return requestBodyData + } + + private func transcriptionPromptText() -> String? { + let normalizedKeyterms = keyterms + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + + guard !normalizedKeyterms.isEmpty else { return nil } + + return """ + This is a short push-to-talk transcript for a coding and product app. Expect product names, technical terms, and app-specific vocabulary such as: \(normalizedKeyterms.joined(separator: ", ")). + """ + } + + private func deliverFinalTranscript(_ transcriptText: String) { + guard !hasDeliveredFinalTranscript else { return } + hasDeliveredFinalTranscript = true + onFinalTranscriptReady(transcriptText) + } + + deinit { + cancel() + } +} + +private extension Data { + mutating func appendString(_ string: String) { + append(string.data(using: .utf8)!) + } + + mutating func appendMultipartFormField( + named fieldName: String, + value: String, + usingBoundary boundary: String + ) { + appendString("--\(boundary)\r\n") + appendString("Content-Disposition: form-data; name=\"\(fieldName)\"\r\n\r\n") + appendString("\(value)\r\n") + } + + mutating func appendMultipartFileField( + named fieldName: String, + filename: String, + mimeType: String, + fileData: Data, + usingBoundary boundary: String + ) { + appendString("--\(boundary)\r\n") + appendString("Content-Disposition: form-data; name=\"\(fieldName)\"; filename=\"\(filename)\"\r\n") + appendString("Content-Type: \(mimeType)\r\n\r\n") + append(fileData) + appendString("\r\n") + } +} diff --git a/leanring-buddy/OverlayWindow.swift b/leanring-buddy/OverlayWindow.swift new file mode 100644 index 0000000..884ebcb --- /dev/null +++ b/leanring-buddy/OverlayWindow.swift @@ -0,0 +1,881 @@ +// +// OverlayWindow.swift +// leanring-buddy +// +// System-wide transparent overlay window for blue glowing cursor. +// One OverlayWindow is created per screen so the cursor buddy +// seamlessly follows the cursor across multiple monitors. +// + +import AppKit +import AVFoundation +import SwiftUI + +class OverlayWindow: NSWindow { + init(screen: NSScreen) { + // Create window covering entire screen + super.init( + contentRect: screen.frame, + styleMask: .borderless, + backing: .buffered, + defer: false + ) + + // Make window transparent and non-interactive + self.isOpaque = false + self.backgroundColor = .clear + self.level = .screenSaver // Always on top, above submenus and popups + self.ignoresMouseEvents = true // Click-through + self.collectionBehavior = [.canJoinAllSpaces, .stationary, .fullScreenAuxiliary] + self.isReleasedWhenClosed = false + self.hasShadow = false + + // Important: Allow the window to appear even when app is not active + self.hidesOnDeactivate = false + + // Cover the entire screen + self.setFrame(screen.frame, display: true) + + // Make sure it's on the right screen + if let screenForWindow = NSScreen.screens.first(where: { $0.frame == screen.frame }) { + self.setFrameOrigin(screenForWindow.frame.origin) + } + } + + // Prevent window from becoming key (no focus stealing) + override var canBecomeKey: Bool { + return false + } + + override var canBecomeMain: Bool { + return false + } +} + +// Cursor-like triangle shape (equilateral) +struct Triangle: Shape { + func path(in rect: CGRect) -> Path { + var path = Path() + let size = min(rect.width, rect.height) + let height = size * sqrt(3.0) / 2.0 + + // Top vertex + path.move(to: CGPoint(x: rect.midX, y: rect.midY - height / 1.5)) + // Bottom left vertex + path.addLine(to: CGPoint(x: rect.midX - size / 2, y: rect.midY + height / 3)) + // Bottom right vertex + path.addLine(to: CGPoint(x: rect.midX + size / 2, y: rect.midY + height / 3)) + path.closeSubpath() + return path + } +} + +// PreferenceKey for tracking bubble size +struct SizePreferenceKey: PreferenceKey { + static var defaultValue: CGSize = .zero + static func reduce(value: inout CGSize, nextValue: () -> CGSize) { + value = nextValue() + } +} + +struct NavigationBubbleSizePreferenceKey: PreferenceKey { + static var defaultValue: CGSize = .zero + static func reduce(value: inout CGSize, nextValue: () -> CGSize) { + value = nextValue() + } +} + +/// The buddy's behavioral mode. Controls whether it follows the cursor, +/// is flying toward a detected UI element, or is pointing at an element. +enum BuddyNavigationMode { + /// Default — buddy follows the mouse cursor with spring animation + case followingCursor + /// Buddy is animating toward a detected UI element location + case navigatingToTarget + /// Buddy has arrived at the target and is pointing at it with a speech bubble + case pointingAtTarget +} + +// SwiftUI view for the blue glowing cursor pointer. +// Each screen gets its own BlueCursorView. The view checks whether +// the cursor is currently on THIS screen and only shows the buddy +// triangle when it is. During voice interaction, the triangle is +// replaced by a waveform (listening), spinner (processing), or +// streaming text bubble (responding). +struct BlueCursorView: View { + let screenFrame: CGRect + let isFirstAppearance: Bool + @ObservedObject var companionManager: CompanionManager + + @State private var cursorPosition: CGPoint + @State private var isCursorOnThisScreen: Bool + + init(screenFrame: CGRect, isFirstAppearance: Bool, companionManager: CompanionManager) { + self.screenFrame = screenFrame + self.isFirstAppearance = isFirstAppearance + self.companionManager = companionManager + + // Seed the cursor position from the current mouse location so the + // buddy doesn't flash at (0,0) before onAppear fires. + let mouseLocation = NSEvent.mouseLocation + let localX = mouseLocation.x - screenFrame.origin.x + let localY = screenFrame.height - (mouseLocation.y - screenFrame.origin.y) + _cursorPosition = State(initialValue: CGPoint(x: localX + 35, y: localY + 25)) + _isCursorOnThisScreen = State(initialValue: screenFrame.contains(mouseLocation)) + } + @State private var timer: Timer? + @State private var welcomeText: String = "" + @State private var showWelcome: Bool = true + @State private var bubbleSize: CGSize = .zero + @State private var bubbleOpacity: Double = 1.0 + @State private var cursorOpacity: Double = 0.0 + + // MARK: - Buddy Navigation State + + /// The buddy's current behavioral mode (following cursor, navigating, or pointing). + @State private var buddyNavigationMode: BuddyNavigationMode = .followingCursor + + /// The rotation angle of the triangle in degrees. Default is -35° (cursor-like). + /// Changes to face the direction of travel when navigating to a target. + @State private var triangleRotationDegrees: Double = -35.0 + + /// Speech bubble text shown when pointing at a detected element. + @State private var navigationBubbleText: String = "" + @State private var navigationBubbleOpacity: Double = 0.0 + @State private var navigationBubbleSize: CGSize = .zero + + /// The cursor position at the moment navigation started, used to detect + /// if the user moves the cursor enough to cancel the navigation. + @State private var cursorPositionWhenNavigationStarted: CGPoint = .zero + + /// Timer driving the frame-by-frame bezier arc flight animation. + /// Invalidated when the flight completes, is canceled, or the view disappears. + @State private var navigationAnimationTimer: Timer? + + /// Scale factor applied to the buddy triangle during flight. Grows to ~1.3x + /// at the midpoint of the arc and shrinks back to 1.0x on landing, creating + /// an energetic "swooping" feel. + @State private var buddyFlightScale: CGFloat = 1.0 + + /// Scale factor for the navigation speech bubble's pop-in entrance. + /// Starts at 0.5 and springs to 1.0 when the first character appears. + @State private var navigationBubbleScale: CGFloat = 1.0 + + /// True when the buddy is flying BACK to the cursor after pointing. + /// Only during the return flight can cursor movement cancel the animation. + @State private var isReturningToCursor: Bool = false + + // MARK: - Onboarding Video Layout + + private let onboardingVideoPlayerWidth: CGFloat = 330 + private let onboardingVideoPlayerHeight: CGFloat = 186 + + private let fullWelcomeMessage = "hey! i'm clicky" + + private let navigationPointerPhrases = [ + "right here!", + "this one!", + "over here!", + "click this!", + "here it is!", + "found it!" + ] + + var body: some View { + ZStack { + // Nearly transparent background (helps with compositing) + Color.black.opacity(0.001) + + // Welcome speech bubble (first launch only) + if isCursorOnThisScreen && showWelcome && !welcomeText.isEmpty { + Text(welcomeText) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(.white) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(DS.Colors.overlayCursorBlue) + .shadow(color: DS.Colors.overlayCursorBlue.opacity(0.5), radius: 6, x: 0, y: 0) + ) + .fixedSize() + .overlay( + GeometryReader { geo in + Color.clear + .preference(key: SizePreferenceKey.self, value: geo.size) + } + ) + .opacity(bubbleOpacity) + .position(x: cursorPosition.x + 10 + (bubbleSize.width / 2), y: cursorPosition.y + 18) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeOut(duration: 0.5), value: bubbleOpacity) + .onPreferenceChange(SizePreferenceKey.self) { newSize in + bubbleSize = newSize + } + } + + // Onboarding video — always in the view tree so opacity animation works + // reliably. When no player exists or opacity is 0, nothing is visible. + // allowsHitTesting(false) prevents it from intercepting clicks. + OnboardingVideoPlayerView(player: companionManager.onboardingVideoPlayer) + .frame(width: onboardingVideoPlayerWidth, height: onboardingVideoPlayerHeight) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .shadow(color: Color.black.opacity(0.4 * companionManager.onboardingVideoOpacity), radius: 12, x: 0, y: 6) + .opacity(isCursorOnThisScreen ? companionManager.onboardingVideoOpacity : 0) + .position( + x: cursorPosition.x + 10 + (onboardingVideoPlayerWidth / 2), + y: cursorPosition.y + 18 + (onboardingVideoPlayerHeight / 2) + ) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeInOut(duration: 2.0), value: companionManager.onboardingVideoOpacity) + .allowsHitTesting(false) + + // Onboarding prompt — "press control + option and say hi" streamed after video ends + if isCursorOnThisScreen && companionManager.showOnboardingPrompt && !companionManager.onboardingPromptText.isEmpty { + Text(companionManager.onboardingPromptText) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(.white) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(DS.Colors.overlayCursorBlue) + .shadow(color: DS.Colors.overlayCursorBlue.opacity(0.5), radius: 6, x: 0, y: 0) + ) + .fixedSize() + .overlay( + GeometryReader { geo in + Color.clear + .preference(key: SizePreferenceKey.self, value: geo.size) + } + ) + .opacity(companionManager.onboardingPromptOpacity) + .position(x: cursorPosition.x + 10 + (bubbleSize.width / 2), y: cursorPosition.y + 18) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeOut(duration: 0.4), value: companionManager.onboardingPromptOpacity) + .onPreferenceChange(SizePreferenceKey.self) { newSize in + bubbleSize = newSize + } + } + + // Navigation pointer bubble — shown when buddy arrives at a detected element. + // Pops in with a scale-bounce (0.5x → 1.0x spring) and a bright initial + // glow that settles, creating a "materializing" effect. + if buddyNavigationMode == .pointingAtTarget && !navigationBubbleText.isEmpty { + Text(navigationBubbleText) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(.white) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(DS.Colors.overlayCursorBlue) + .shadow( + color: DS.Colors.overlayCursorBlue.opacity(0.5 + (1.0 - navigationBubbleScale) * 1.0), + radius: 6 + (1.0 - navigationBubbleScale) * 16, + x: 0, y: 0 + ) + ) + .fixedSize() + .overlay( + GeometryReader { geo in + Color.clear + .preference(key: NavigationBubbleSizePreferenceKey.self, value: geo.size) + } + ) + .scaleEffect(navigationBubbleScale) + .opacity(navigationBubbleOpacity) + .position(x: cursorPosition.x + 10 + (navigationBubbleSize.width / 2), y: cursorPosition.y + 18) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.spring(response: 0.4, dampingFraction: 0.6), value: navigationBubbleScale) + .animation(.easeOut(duration: 0.5), value: navigationBubbleOpacity) + .onPreferenceChange(NavigationBubbleSizePreferenceKey.self) { newSize in + navigationBubbleSize = newSize + } + } + + // Blue triangle cursor — shown when idle or while TTS is playing (responding). + // All three states (triangle, waveform, spinner) stay in the view tree + // permanently and cross-fade via opacity so SwiftUI doesn't remove/re-insert + // them (which caused a visible cursor "pop"). + // + // During cursor following: fast spring animation for snappy tracking. + // During navigation: NO implicit animation — the frame-by-frame bezier + // timer controls position directly at 60fps for a smooth arc flight. + Triangle() + .fill(DS.Colors.overlayCursorBlue) + .frame(width: 16, height: 16) + .rotationEffect(.degrees(triangleRotationDegrees)) + .shadow(color: DS.Colors.overlayCursorBlue, radius: 8 + (buddyFlightScale - 1.0) * 20, x: 0, y: 0) + .scaleEffect(buddyFlightScale) + .opacity(buddyIsVisibleOnThisScreen && (companionManager.voiceState == .idle || companionManager.voiceState == .responding) ? cursorOpacity : 0) + .position(cursorPosition) + .animation( + buddyNavigationMode == .followingCursor + ? .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0) + : nil, + value: cursorPosition + ) + .animation(.easeIn(duration: 0.25), value: companionManager.voiceState) + .animation( + buddyNavigationMode == .navigatingToTarget ? nil : .easeInOut(duration: 0.3), + value: triangleRotationDegrees + ) + + // Blue waveform — replaces the triangle while listening + BlueCursorWaveformView(audioPowerLevel: companionManager.currentAudioPowerLevel) + .opacity(buddyIsVisibleOnThisScreen && companionManager.voiceState == .listening ? cursorOpacity : 0) + .position(cursorPosition) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeIn(duration: 0.15), value: companionManager.voiceState) + + // Blue spinner — shown while the AI is processing (transcription + Claude + waiting for TTS) + BlueCursorSpinnerView() + .opacity(buddyIsVisibleOnThisScreen && companionManager.voiceState == .processing ? cursorOpacity : 0) + .position(cursorPosition) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeIn(duration: 0.15), value: companionManager.voiceState) + + } + .frame(width: screenFrame.width, height: screenFrame.height) + .ignoresSafeArea() + .onAppear { + // Set initial cursor position immediately before starting animation + let mouseLocation = NSEvent.mouseLocation + isCursorOnThisScreen = screenFrame.contains(mouseLocation) + + let swiftUIPosition = convertScreenPointToSwiftUICoordinates(mouseLocation) + self.cursorPosition = CGPoint(x: swiftUIPosition.x + 35, y: swiftUIPosition.y + 25) + + startTrackingCursor() + + // Only show welcome message on first appearance (app start) + // and only if the cursor starts on this screen + if isFirstAppearance && isCursorOnThisScreen { + withAnimation(.easeIn(duration: 2.0)) { + self.cursorOpacity = 1.0 + } + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.bubbleOpacity = 0.0 + startWelcomeAnimation() + } + } else { + self.cursorOpacity = 1.0 + } + } + .onDisappear { + timer?.invalidate() + navigationAnimationTimer?.invalidate() + companionManager.tearDownOnboardingVideo() + } + .onChange(of: companionManager.detectedElementScreenLocation) { newLocation in + // When a UI element location is detected, navigate the buddy to + // that position so it points at the element. + guard let screenLocation = newLocation, + let displayFrame = companionManager.detectedElementDisplayFrame else { + return + } + + // Only navigate if the target is on THIS screen + guard screenFrame.contains(CGPoint(x: displayFrame.midX, y: displayFrame.midY)) + || displayFrame == screenFrame else { + return + } + + startNavigatingToElement(screenLocation: screenLocation) + } + } + + /// Whether the buddy triangle should be visible on this screen. + /// True when cursor is on this screen during normal following, or + /// when navigating/pointing at a target on this screen. When another + /// screen is navigating (detectedElementScreenLocation is set but this + /// screen isn't the one animating), hide the cursor so only one buddy + /// is ever visible at a time. + private var buddyIsVisibleOnThisScreen: Bool { + switch buddyNavigationMode { + case .followingCursor: + // If another screen's BlueCursorView is navigating to an element, + // hide the cursor on this screen to prevent a duplicate buddy + if companionManager.detectedElementScreenLocation != nil { + return false + } + return isCursorOnThisScreen + case .navigatingToTarget, .pointingAtTarget: + return true + } + } + + // MARK: - Cursor Tracking + + private func startTrackingCursor() { + timer = Timer.scheduledTimer(withTimeInterval: 0.016, repeats: true) { _ in + let mouseLocation = NSEvent.mouseLocation + self.isCursorOnThisScreen = self.screenFrame.contains(mouseLocation) + + // During forward flight or pointing, the buddy is NOT interrupted by + // mouse movement — it completes its full animation and return flight. + // Only during the RETURN flight do we allow cursor movement to cancel + // (so the buddy snaps to following if the user moves while it's flying back). + if self.buddyNavigationMode == .navigatingToTarget && self.isReturningToCursor { + let currentMouseInSwiftUI = self.convertScreenPointToSwiftUICoordinates(mouseLocation) + let distanceFromNavigationStart = hypot( + currentMouseInSwiftUI.x - self.cursorPositionWhenNavigationStarted.x, + currentMouseInSwiftUI.y - self.cursorPositionWhenNavigationStarted.y + ) + if distanceFromNavigationStart > 100 { + cancelNavigationAndResumeFollowing() + } + return + } + + // During forward navigation or pointing, just skip cursor tracking + if self.buddyNavigationMode != .followingCursor { + return + } + + // Normal cursor following + let swiftUIPosition = self.convertScreenPointToSwiftUICoordinates(mouseLocation) + let buddyX = swiftUIPosition.x + 35 + let buddyY = swiftUIPosition.y + 25 + self.cursorPosition = CGPoint(x: buddyX, y: buddyY) + } + } + + /// Converts a macOS screen point (AppKit, bottom-left origin) to SwiftUI + /// coordinates (top-left origin) relative to this screen's overlay window. + private func convertScreenPointToSwiftUICoordinates(_ screenPoint: CGPoint) -> CGPoint { + let x = screenPoint.x - screenFrame.origin.x + let y = (screenFrame.origin.y + screenFrame.height) - screenPoint.y + return CGPoint(x: x, y: y) + } + + // MARK: - Element Navigation + + /// Starts animating the buddy toward a detected UI element location. + private func startNavigatingToElement(screenLocation: CGPoint) { + // Don't interrupt welcome animation + guard !showWelcome || welcomeText.isEmpty else { return } + + // Convert the AppKit screen location to SwiftUI coordinates for this screen + let targetInSwiftUI = convertScreenPointToSwiftUICoordinates(screenLocation) + + // Offset the target so the buddy sits beside the element rather than + // directly on top of it — 8px to the right, 12px below. + let offsetTarget = CGPoint( + x: targetInSwiftUI.x + 8, + y: targetInSwiftUI.y + 12 + ) + + // Clamp target to screen bounds with padding + let clampedTarget = CGPoint( + x: max(20, min(offsetTarget.x, screenFrame.width - 20)), + y: max(20, min(offsetTarget.y, screenFrame.height - 20)) + ) + + // Record the current cursor position so we can detect if the user + // moves the mouse enough to cancel the return flight + let mouseLocation = NSEvent.mouseLocation + cursorPositionWhenNavigationStarted = convertScreenPointToSwiftUICoordinates(mouseLocation) + + // Enter navigation mode — stop cursor following + buddyNavigationMode = .navigatingToTarget + isReturningToCursor = false + + animateBezierFlightArc(to: clampedTarget) { + guard self.buddyNavigationMode == .navigatingToTarget else { return } + self.startPointingAtElement() + } + } + + /// Animates the buddy along a quadratic bezier arc from its current position + /// to the specified destination. The triangle rotates to face its direction + /// of travel (tangent to the curve) each frame, scales up at the midpoint + /// for a "swooping" feel, and the glow intensifies during flight. + private func animateBezierFlightArc( + to destination: CGPoint, + onComplete: @escaping () -> Void + ) { + navigationAnimationTimer?.invalidate() + + let startPosition = cursorPosition + let endPosition = destination + + let deltaX = endPosition.x - startPosition.x + let deltaY = endPosition.y - startPosition.y + let distance = hypot(deltaX, deltaY) + + // Flight duration scales with distance: short hops are quick, long + // flights are more dramatic. Clamped to 0.6s–1.4s. + let flightDurationSeconds = min(max(distance / 800.0, 0.6), 1.4) + let frameInterval: Double = 1.0 / 60.0 + let totalFrames = Int(flightDurationSeconds / frameInterval) + var currentFrame = 0 + + // Control point for the quadratic bezier arc. Offset the midpoint + // upward (negative Y in SwiftUI) so the buddy flies in a parabolic arc. + let midPoint = CGPoint( + x: (startPosition.x + endPosition.x) / 2.0, + y: (startPosition.y + endPosition.y) / 2.0 + ) + let arcHeight = min(distance * 0.2, 80.0) + let controlPoint = CGPoint(x: midPoint.x, y: midPoint.y - arcHeight) + + navigationAnimationTimer = Timer.scheduledTimer(withTimeInterval: frameInterval, repeats: true) { _ in + currentFrame += 1 + + if currentFrame > totalFrames { + self.navigationAnimationTimer?.invalidate() + self.navigationAnimationTimer = nil + self.cursorPosition = endPosition + self.buddyFlightScale = 1.0 + onComplete() + return + } + + // Linear progress 0→1 over the flight duration + let linearProgress = Double(currentFrame) / Double(totalFrames) + + // Smoothstep easeInOut: 3t² - 2t³ (Hermite interpolation) + let t = linearProgress * linearProgress * (3.0 - 2.0 * linearProgress) + + // Quadratic bezier: B(t) = (1-t)²·P0 + 2(1-t)t·P1 + t²·P2 + let oneMinusT = 1.0 - t + let bezierX = oneMinusT * oneMinusT * startPosition.x + + 2.0 * oneMinusT * t * controlPoint.x + + t * t * endPosition.x + let bezierY = oneMinusT * oneMinusT * startPosition.y + + 2.0 * oneMinusT * t * controlPoint.y + + t * t * endPosition.y + + self.cursorPosition = CGPoint(x: bezierX, y: bezierY) + + // Rotation: face the direction of travel by computing the tangent + // to the bezier curve. B'(t) = 2(1-t)(P1-P0) + 2t(P2-P1) + let tangentX = 2.0 * oneMinusT * (controlPoint.x - startPosition.x) + + 2.0 * t * (endPosition.x - controlPoint.x) + let tangentY = 2.0 * oneMinusT * (controlPoint.y - startPosition.y) + + 2.0 * t * (endPosition.y - controlPoint.y) + // +90° offset because the triangle's "tip" points up at 0° rotation, + // and atan2 returns 0° for rightward movement + self.triangleRotationDegrees = atan2(tangentY, tangentX) * (180.0 / .pi) + 90.0 + + // Scale pulse: sin curve peaks at midpoint of the flight. + // Buddy grows to ~1.3x at the apex, then shrinks back to 1.0x on landing. + let scalePulse = sin(linearProgress * .pi) + self.buddyFlightScale = 1.0 + scalePulse * 0.3 + } + } + + /// Transitions to pointing mode — shows a speech bubble with a bouncy + /// scale-in entrance and variable-speed character streaming. + private func startPointingAtElement() { + buddyNavigationMode = .pointingAtTarget + + // Rotate back to default pointer angle now that we've arrived + triangleRotationDegrees = -35.0 + + // Reset navigation bubble state — start small for the scale-bounce entrance + navigationBubbleText = "" + navigationBubbleOpacity = 1.0 + navigationBubbleSize = .zero + navigationBubbleScale = 0.5 + + // Use custom bubble text from the companion manager (e.g. onboarding demo) + // if available, otherwise fall back to a random pointer phrase + let pointerPhrase = companionManager.detectedElementBubbleText + ?? navigationPointerPhrases.randomElement() + ?? "right here!" + + streamNavigationBubbleCharacter(phrase: pointerPhrase, characterIndex: 0) { + // All characters streamed — hold for 3 seconds, then fly back + DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { + guard self.buddyNavigationMode == .pointingAtTarget else { return } + self.navigationBubbleOpacity = 0.0 + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { + guard self.buddyNavigationMode == .pointingAtTarget else { return } + self.startFlyingBackToCursor() + } + } + } + } + + /// Streams the navigation bubble text one character at a time with variable + /// delays (30–60ms) for a natural "speaking" rhythm. + private func streamNavigationBubbleCharacter( + phrase: String, + characterIndex: Int, + onComplete: @escaping () -> Void + ) { + guard buddyNavigationMode == .pointingAtTarget else { return } + guard characterIndex < phrase.count else { + onComplete() + return + } + + let charIndex = phrase.index(phrase.startIndex, offsetBy: characterIndex) + navigationBubbleText.append(phrase[charIndex]) + + // On the first character, trigger the scale-bounce entrance + if characterIndex == 0 { + navigationBubbleScale = 1.0 + } + + let characterDelay = Double.random(in: 0.03...0.06) + DispatchQueue.main.asyncAfter(deadline: .now() + characterDelay) { + self.streamNavigationBubbleCharacter( + phrase: phrase, + characterIndex: characterIndex + 1, + onComplete: onComplete + ) + } + } + + /// Flies the buddy back to the current cursor position after pointing is done. + private func startFlyingBackToCursor() { + let mouseLocation = NSEvent.mouseLocation + let cursorInSwiftUI = convertScreenPointToSwiftUICoordinates(mouseLocation) + let cursorWithTrackingOffset = CGPoint(x: cursorInSwiftUI.x + 35, y: cursorInSwiftUI.y + 25) + + cursorPositionWhenNavigationStarted = cursorInSwiftUI + + buddyNavigationMode = .navigatingToTarget + isReturningToCursor = true + + animateBezierFlightArc(to: cursorWithTrackingOffset) { + self.finishNavigationAndResumeFollowing() + } + } + + /// Cancels an in-progress navigation because the user moved the cursor. + private func cancelNavigationAndResumeFollowing() { + navigationAnimationTimer?.invalidate() + navigationAnimationTimer = nil + navigationBubbleText = "" + navigationBubbleOpacity = 0.0 + navigationBubbleScale = 1.0 + buddyFlightScale = 1.0 + finishNavigationAndResumeFollowing() + } + + /// Returns the buddy to normal cursor-following mode after navigation completes. + private func finishNavigationAndResumeFollowing() { + navigationAnimationTimer?.invalidate() + navigationAnimationTimer = nil + buddyNavigationMode = .followingCursor + isReturningToCursor = false + triangleRotationDegrees = -35.0 + buddyFlightScale = 1.0 + navigationBubbleText = "" + navigationBubbleOpacity = 0.0 + navigationBubbleScale = 1.0 + companionManager.clearDetectedElementLocation() + } + + // MARK: - Welcome Animation + + private func startWelcomeAnimation() { + withAnimation(.easeIn(duration: 0.4)) { + self.bubbleOpacity = 1.0 + } + + var currentIndex = 0 + Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { timer in + guard currentIndex < self.fullWelcomeMessage.count else { + timer.invalidate() + // Hold the text for 2 seconds, then fade it out + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.bubbleOpacity = 0.0 + } + DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { + self.showWelcome = false + // Start the onboarding video right after the welcome text disappears + self.companionManager.setupOnboardingVideo() + } + return + } + + let index = self.fullWelcomeMessage.index(self.fullWelcomeMessage.startIndex, offsetBy: currentIndex) + self.welcomeText.append(self.fullWelcomeMessage[index]) + currentIndex += 1 + } + } +} + +// MARK: - Blue Cursor Waveform + +/// A small blue waveform that replaces the triangle cursor while +/// the user is holding the push-to-talk shortcut and speaking. +private struct BlueCursorWaveformView: View { + let audioPowerLevel: CGFloat + + private let barCount = 5 + private let listeningBarProfile: [CGFloat] = [0.4, 0.7, 1.0, 0.7, 0.4] + + var body: some View { + TimelineView(.animation(minimumInterval: 1.0 / 36.0)) { timelineContext in + HStack(alignment: .center, spacing: 2) { + ForEach(0.. CGFloat { + let animationPhase = CGFloat(timelineDate.timeIntervalSinceReferenceDate * 3.6) + CGFloat(barIndex) * 0.35 + let normalizedAudioPowerLevel = max(audioPowerLevel - 0.008, 0) + let easedAudioPowerLevel = pow(min(normalizedAudioPowerLevel * 2.85, 1), 0.76) + let reactiveHeight = easedAudioPowerLevel * 10 * listeningBarProfile[barIndex] + let idlePulse = (sin(animationPhase) + 1) / 2 * 1.5 + return 3 + reactiveHeight + idlePulse + } +} + +// MARK: - Blue Cursor Spinner + +/// A small blue spinning indicator that replaces the triangle cursor +/// while the AI is processing a voice input. +private struct BlueCursorSpinnerView: View { + @State private var isSpinning = false + + var body: some View { + Circle() + .trim(from: 0.15, to: 0.85) + .stroke( + AngularGradient( + colors: [ + DS.Colors.overlayCursorBlue.opacity(0.0), + DS.Colors.overlayCursorBlue + ], + center: .center + ), + style: StrokeStyle(lineWidth: 2.5, lineCap: .round) + ) + .frame(width: 14, height: 14) + .rotationEffect(.degrees(isSpinning ? 360 : 0)) + .shadow(color: DS.Colors.overlayCursorBlue.opacity(0.6), radius: 6, x: 0, y: 0) + .onAppear { + withAnimation(.linear(duration: 0.8).repeatForever(autoreverses: false)) { + isSpinning = true + } + } + } +} + +// Manager for overlay windows — creates one per screen so the cursor +// buddy seamlessly follows the cursor across multiple monitors. +@MainActor +class OverlayWindowManager { + private var overlayWindows: [OverlayWindow] = [] + var hasShownOverlayBefore = false + + func showOverlay(onScreens screens: [NSScreen], companionManager: CompanionManager) { + // Hide any existing overlays + hideOverlay() + + // Track if this is the first time showing overlay (welcome message) + let isFirstAppearance = !hasShownOverlayBefore + hasShownOverlayBefore = true + + // Create one overlay window per screen + for screen in screens { + let window = OverlayWindow(screen: screen) + + let contentView = BlueCursorView( + screenFrame: screen.frame, + isFirstAppearance: isFirstAppearance, + companionManager: companionManager + ) + + let hostingView = NSHostingView(rootView: contentView) + hostingView.frame = screen.frame + window.contentView = hostingView + + overlayWindows.append(window) + window.orderFrontRegardless() + } + } + + func hideOverlay() { + for window in overlayWindows { + window.orderOut(nil) + window.contentView = nil + } + overlayWindows.removeAll() + } + + /// Fades out overlay windows over `duration` seconds, then removes them. + func fadeOutAndHideOverlay(duration: TimeInterval = 0.4) { + let windowsToFade = overlayWindows + overlayWindows.removeAll() + + NSAnimationContext.runAnimationGroup({ context in + context.duration = duration + context.timingFunction = CAMediaTimingFunction(name: .easeIn) + for window in windowsToFade { + window.animator().alphaValue = 0 + } + }, completionHandler: { + for window in windowsToFade { + window.orderOut(nil) + window.contentView = nil + } + }) + } + + func isShowingOverlay() -> Bool { + return !overlayWindows.isEmpty + } +} + +// MARK: - Onboarding Video Player + +/// NSViewRepresentable wrapping an AVPlayerLayer so HLS video plays +/// inside SwiftUI. Uses a custom NSView subclass to keep the player +/// layer sized to the view's bounds automatically. +private struct OnboardingVideoPlayerView: NSViewRepresentable { + let player: AVPlayer? + + func makeNSView(context: Context) -> AVPlayerNSView { + let view = AVPlayerNSView() + view.player = player + return view + } + + func updateNSView(_ nsView: AVPlayerNSView, context: Context) { + nsView.player = player + } +} + +private class AVPlayerNSView: NSView { + var player: AVPlayer? { + didSet { playerLayer.player = player } + } + + private let playerLayer = AVPlayerLayer() + + override init(frame: NSRect) { + super.init(frame: frame) + wantsLayer = true + playerLayer.videoGravity = .resizeAspectFill + layer?.addSublayer(playerLayer) + } + + required init?(coder: NSCoder) { fatalError() } + + override func layout() { + super.layout() + playerLayer.frame = bounds + } +} diff --git a/leanring-buddy/WindowPositionManager.swift b/leanring-buddy/WindowPositionManager.swift new file mode 100644 index 0000000..8f6edf6 --- /dev/null +++ b/leanring-buddy/WindowPositionManager.swift @@ -0,0 +1,262 @@ +// +// WindowPositionManager.swift +// leanring-buddy +// +// Manages positioning the app window on the right edge of the screen +// and shrinking overlapping windows from other apps via the Accessibility API. +// + +import AppKit +import ApplicationServices +import ScreenCaptureKit + +enum PermissionRequestPresentationDestination: Equatable { + case alreadyGranted + case systemPrompt + case systemSettings +} + +@MainActor +class WindowPositionManager { + private static var hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = false + private static var hasAttemptedScreenRecordingSystemPromptDuringCurrentLaunch = false + private static let hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey = "com.learningbuddy.hasPreviouslyConfirmedScreenRecordingPermission" + + /// Returns true when the Mac currently has more than one connected display. + /// Uses AppKit's screen list, which is available without ScreenCaptureKit's + /// shareable-content permission prompt. + static func currentMacHasMultipleDisplays() -> Bool { + NSScreen.screens.count > 1 + } + + // MARK: - Accessibility Permission + + /// Returns true if the app has Accessibility permission. + static func hasAccessibilityPermission() -> Bool { + AXIsProcessTrusted() + } + + /// Presents exactly one permission path per tap: the system prompt on the first + /// attempt, then System Settings on later attempts after macOS has already shown + /// its one-time alert. + @discardableResult + static func requestAccessibilityPermission() -> PermissionRequestPresentationDestination { + let presentationDestination = permissionRequestPresentationDestination( + hasPermissionNow: hasAccessibilityPermission(), + hasAttemptedSystemPrompt: hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch + ) + + switch presentationDestination { + case .alreadyGranted: + return .alreadyGranted + case .systemPrompt: + hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = true + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): true] as CFDictionary + _ = AXIsProcessTrustedWithOptions(options) + case .systemSettings: + openAccessibilitySettings() + } + + return presentationDestination + } + + /// Opens System Settings to the Accessibility pane. + static func openAccessibilitySettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") else { return } + NSWorkspace.shared.open(url) + } + + /// Reveals the running app bundle in Finder so the user can drag it into + /// the Accessibility list if it doesn't appear automatically. + static func revealAppInFinder() { + guard let appURL = Bundle.main.bundleURL as URL? else { return } + NSWorkspace.shared.activateFileViewerSelecting([appURL]) + } + + // MARK: - Screen Recording Permission + + /// Returns true if Screen Recording permission is granted. + static func hasScreenRecordingPermission() -> Bool { + let hasScreenRecordingPermissionNow = CGPreflightScreenCaptureAccess() + if hasScreenRecordingPermissionNow { + UserDefaults.standard.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + } + return hasScreenRecordingPermissionNow + } + + /// Returns true when the app should proceed with session launch without showing + /// the permission gate again. This intentionally falls back to the last known + /// granted state because CGPreflightScreenCaptureAccess() can sometimes return a + /// false negative even though the user has already approved the app. + static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() -> Bool { + shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( + hasScreenRecordingPermissionNow: hasScreenRecordingPermission(), + hasPreviouslyConfirmedScreenRecordingPermission: UserDefaults.standard.bool(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + ) + } + + static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( + hasScreenRecordingPermissionNow: Bool, + hasPreviouslyConfirmedScreenRecordingPermission: Bool + ) -> Bool { + hasScreenRecordingPermissionNow || hasPreviouslyConfirmedScreenRecordingPermission + } + + static func clearPreviouslyConfirmedScreenRecordingPermission() { + UserDefaults.standard.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + } + + /// Prompts the system dialog for Screen Recording permission. + /// Uses the system prompt once, then opens System Settings on later attempts so + /// the user never gets the prompt and the Settings pane at the same time. + @discardableResult + static func requestScreenRecordingPermission() -> PermissionRequestPresentationDestination { + let presentationDestination = permissionRequestPresentationDestination( + hasPermissionNow: hasScreenRecordingPermission(), + hasAttemptedSystemPrompt: hasAttemptedScreenRecordingSystemPromptDuringCurrentLaunch + ) + + switch presentationDestination { + case .alreadyGranted: + return .alreadyGranted + case .systemPrompt: + hasAttemptedScreenRecordingSystemPromptDuringCurrentLaunch = true + _ = CGRequestScreenCaptureAccess() + case .systemSettings: + openScreenRecordingSettings() + } + + return presentationDestination + } + + /// Opens System Settings to the Screen Recording pane. + static func openScreenRecordingSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") else { return } + NSWorkspace.shared.open(url) + } + + static func permissionRequestPresentationDestination( + hasPermissionNow: Bool, + hasAttemptedSystemPrompt: Bool + ) -> PermissionRequestPresentationDestination { + if hasPermissionNow { + return .alreadyGranted + } + + if hasAttemptedSystemPrompt { + return .systemSettings + } + + return .systemPrompt + } + + // MARK: - Window Positioning + + /// Positions the app's main window pinned to the right edge of the screen + /// that contains the given display ID, vertically centered. + static func pinMainWindowToRight(onDisplayID displayID: CGDirectDisplayID?) { + guard let mainWindow = NSApp.windows.first(where: { !($0 is NSPanel) }) else { return } + + // Find the NSScreen matching the selected display, or fall back to the screen + // the window is currently on, or finally the main screen. + let targetScreen: NSScreen + if let displayID, + let matchingScreen = NSScreen.screens.first(where: { $0.displayID == displayID }) { + targetScreen = matchingScreen + } else if let currentScreen = mainWindow.screen { + targetScreen = currentScreen + } else if let mainScreen = NSScreen.main { + targetScreen = mainScreen + } else { + return + } + + let visibleFrame = targetScreen.visibleFrame + let windowSize = mainWindow.frame.size + + let x = visibleFrame.maxX - windowSize.width + let y = visibleFrame.minY + (visibleFrame.height - windowSize.height) / 2.0 + + mainWindow.setFrameOrigin(NSPoint(x: x, y: y)) + } + + // MARK: - Shrink Overlapping Windows + + /// Checks if the frontmost (non-self) app's focused window overlaps our app window + /// on the same monitor and, if so, shrinks it so it no longer overlaps. + /// Only operates if both windows are on the same screen as `targetDisplayID`. + static func shrinkOverlappingFocusedWindow(targetDisplayID: CGDirectDisplayID?) { + guard hasAccessibilityPermission() else { return } + guard let mainWindow = NSApp.windows.first(where: { !($0 is NSPanel) }) else { return } + guard let mainScreen = mainWindow.screen else { return } + + // Only operate if the main window is on the target display + if let targetDisplayID, mainScreen.displayID != targetDisplayID { + return + } + + // Get the frontmost application that isn't us + guard let frontApp = NSWorkspace.shared.frontmostApplication, + frontApp.processIdentifier != ProcessInfo.processInfo.processIdentifier else { + return + } + + let appElement = AXUIElementCreateApplication(frontApp.processIdentifier) + + // Get the focused window of the front app + var focusedWindowValue: AnyObject? + let focusedResult = AXUIElementCopyAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, &focusedWindowValue) + guard focusedResult == .success, let focusedWindow = focusedWindowValue else { return } + + // Get position and size of the focused window + var positionValue: AnyObject? + var sizeValue: AnyObject? + guard AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXPositionAttribute as CFString, &positionValue) == .success, + AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXSizeAttribute as CFString, &sizeValue) == .success else { + return + } + + var otherPosition = CGPoint.zero + var otherSize = CGSize.zero + guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &otherPosition), + AXValueGetValue(sizeValue as! AXValue, .cgSize, &otherSize) else { + return + } + + // The other window's frame in screen coordinates (top-left origin from AX API). + // Convert to check if it's on the same screen as our window. + let otherRight = otherPosition.x + otherSize.width + let ourLeft = mainWindow.frame.origin.x + + // Check that the other window is on the same screen by verifying its origin + // falls within the target screen's bounds. + let screenFrame = mainScreen.frame + let otherCenterX = otherPosition.x + otherSize.width / 2 + // AX uses top-left origin, NSScreen uses bottom-left. Convert AX Y to NSScreen Y. + let otherNSScreenY = screenFrame.maxY - otherPosition.y - otherSize.height + let otherCenterY = otherNSScreenY + otherSize.height / 2 + let otherCenter = NSPoint(x: otherCenterX, y: otherCenterY) + + guard screenFrame.contains(otherCenter) else { return } + + // If the other window's right edge extends past our window's left edge, shrink it. + if otherRight > ourLeft { + let newWidth = ourLeft - otherPosition.x + guard newWidth > 200 else { return } // Don't shrink too small + + var newSize = CGSize(width: newWidth, height: otherSize.height) + guard let newSizeValue = AXValueCreate(.cgSize, &newSize) else { return } + AXUIElementSetAttributeValue(focusedWindow as! AXUIElement, kAXSizeAttribute as CFString, newSizeValue) + } + } +} + +// MARK: - NSScreen Extension + +extension NSScreen { + /// The CGDirectDisplayID for this screen. + var displayID: CGDirectDisplayID { + let key = NSDeviceDescriptionKey("NSScreenNumber") + return deviceDescription[key] as? CGDirectDisplayID ?? 0 + } +} diff --git a/leanring-buddy/codex-add-project.png b/leanring-buddy/codex-add-project.png new file mode 100644 index 0000000..a5222d0 Binary files /dev/null and b/leanring-buddy/codex-add-project.png differ diff --git a/leanring-buddy/enter.mp3 b/leanring-buddy/enter.mp3 new file mode 100644 index 0000000..e8185a5 Binary files /dev/null and b/leanring-buddy/enter.mp3 differ diff --git a/leanring-buddy/eshop.mp3 b/leanring-buddy/eshop.mp3 new file mode 100644 index 0000000..0abd14e Binary files /dev/null and b/leanring-buddy/eshop.mp3 differ diff --git a/leanring-buddy/ff.mp3 b/leanring-buddy/ff.mp3 new file mode 100644 index 0000000..a7cf86d Binary files /dev/null and b/leanring-buddy/ff.mp3 differ diff --git a/leanring-buddy/leanring-buddy.entitlements b/leanring-buddy/leanring-buddy.entitlements new file mode 100644 index 0000000..48d74a4 --- /dev/null +++ b/leanring-buddy/leanring-buddy.entitlements @@ -0,0 +1,18 @@ + + + + + com.apple.security.app-sandbox + + com.apple.security.network.client + + com.apple.security.device.camera + + com.apple.security.device.audio-input + + com.apple.security.temporary-exception.mach-lookup.global-name + + com.apple.screencapturekit.picker + + + diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift new file mode 100644 index 0000000..b004a89 --- /dev/null +++ b/leanring-buddy/leanring_buddyApp.swift @@ -0,0 +1,89 @@ +// +// leanring_buddyApp.swift +// leanring-buddy +// +// Menu bar-only companion app. No dock icon, no main window — just an +// always-available status item in the macOS menu bar. Clicking the icon +// opens a floating panel with companion voice controls. +// + +import ServiceManagement +import SwiftUI +import Sparkle + +@main +struct leanring_buddyApp: App { + @NSApplicationDelegateAdaptor(CompanionAppDelegate.self) var appDelegate + + var body: some Scene { + // The app lives entirely in the menu bar panel managed by the AppDelegate. + // This empty Settings scene satisfies SwiftUI's requirement for at least + // one scene but is never shown (LSUIElement=true removes the app menu). + Settings { + EmptyView() + } + } +} + +/// Manages the companion lifecycle: creates the menu bar panel and starts +/// the companion voice pipeline on launch. +@MainActor +final class CompanionAppDelegate: NSObject, NSApplicationDelegate { + private var menuBarPanelManager: MenuBarPanelManager? + private let companionManager = CompanionManager() + private var sparkleUpdaterController: SPUStandardUpdaterController? + + func applicationDidFinishLaunching(_ notification: Notification) { + print("🎯 Clicky: Starting...") + print("🎯 Clicky: Version \(Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown")") + + UserDefaults.standard.register(defaults: ["NSInitialToolTipDelay": 0]) + + ClickyAnalytics.configure() + ClickyAnalytics.trackAppOpened() + + menuBarPanelManager = MenuBarPanelManager(companionManager: companionManager) + companionManager.start() + // Auto-open the panel if the user still needs to do something: + // either they haven't onboarded yet, or permissions were revoked. + if !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { + menuBarPanelManager?.showPanelOnLaunch() + } + registerAsLoginItemIfNeeded() + // startSparkleUpdater() + } + + func applicationWillTerminate(_ notification: Notification) { + companionManager.stop() + } + + /// Registers the app as a login item so it launches automatically on + /// startup. Uses SMAppService which shows the app in System Settings > + /// General > Login Items, letting the user toggle it off if they want. + private func registerAsLoginItemIfNeeded() { + let loginItemService = SMAppService.mainApp + if loginItemService.status != .enabled { + do { + try loginItemService.register() + print("🎯 Clicky: Registered as login item") + } catch { + print("⚠️ Clicky: Failed to register as login item: \(error)") + } + } + } + + private func startSparkleUpdater() { + let updaterController = SPUStandardUpdaterController( + startingUpdater: false, + updaterDelegate: nil, + userDriverDelegate: nil + ) + self.sparkleUpdaterController = updaterController + + do { + try updaterController.updater.start() + } catch { + print("⚠️ Clicky: Sparkle updater failed to start: \(error)") + } + } +} diff --git a/leanring-buddy/steve.jpg b/leanring-buddy/steve.jpg new file mode 100644 index 0000000..19b2d40 Binary files /dev/null and b/leanring-buddy/steve.jpg differ diff --git a/leanring-buddyTests/leanring_buddyTests.swift b/leanring-buddyTests/leanring_buddyTests.swift new file mode 100644 index 0000000..188fe7a --- /dev/null +++ b/leanring-buddyTests/leanring_buddyTests.swift @@ -0,0 +1,40 @@ +// +// leanring_buddyTests.swift +// leanring-buddyTests +// +// Created by thorfinn on 3/2/26. +// + +import Testing +@testable import leanring_buddy + +struct leanring_buddyTests { + + @Test func firstPermissionRequestUsesSystemPromptOnly() async throws { + let presentationDestination = WindowPositionManager.permissionRequestPresentationDestination( + hasPermissionNow: false, + hasAttemptedSystemPrompt: false + ) + + #expect(presentationDestination == .systemPrompt) + } + + @Test func repeatedPermissionRequestOpensSystemSettings() async throws { + let presentationDestination = WindowPositionManager.permissionRequestPresentationDestination( + hasPermissionNow: false, + hasAttemptedSystemPrompt: true + ) + + #expect(presentationDestination == .systemSettings) + } + + @Test func knownGrantedScreenRecordingPermissionSkipsTheGate() async throws { + let shouldTreatPermissionAsGranted = WindowPositionManager.shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( + hasScreenRecordingPermissionNow: false, + hasPreviouslyConfirmedScreenRecordingPermission: true + ) + + #expect(shouldTreatPermissionAsGranted) + } + +} diff --git a/leanring-buddyUITests/leanring_buddyUITests.swift b/leanring-buddyUITests/leanring_buddyUITests.swift new file mode 100644 index 0000000..844e6b9 --- /dev/null +++ b/leanring-buddyUITests/leanring_buddyUITests.swift @@ -0,0 +1,41 @@ +// +// leanring_buddyUITests.swift +// leanring-buddyUITests +// +// Created by thorfinn on 3/2/26. +// + +import XCTest + +final class leanring_buddyUITests: XCTestCase { + + override func setUpWithError() throws { + // Put setup code here. This method is called before the invocation of each test method in the class. + + // In UI tests it is usually best to stop immediately when a failure occurs. + continueAfterFailure = false + + // In UI tests it’s important to set the initial state - such as interface orientation - required for your tests before they run. The setUp method is a good place to do this. + } + + override func tearDownWithError() throws { + // Put teardown code here. This method is called after the invocation of each test method in the class. + } + + @MainActor + func testExample() throws { + // UI tests must launch the application that they test. + let app = XCUIApplication() + app.launch() + + // Use XCTAssert and related functions to verify your tests produce the correct results. + } + + @MainActor + func testLaunchPerformance() throws { + // This measures how long it takes to launch your application. + measure(metrics: [XCTApplicationLaunchMetric()]) { + XCUIApplication().launch() + } + } +} diff --git a/leanring-buddyUITests/leanring_buddyUITestsLaunchTests.swift b/leanring-buddyUITests/leanring_buddyUITestsLaunchTests.swift new file mode 100644 index 0000000..5fe330b --- /dev/null +++ b/leanring-buddyUITests/leanring_buddyUITestsLaunchTests.swift @@ -0,0 +1,33 @@ +// +// leanring_buddyUITestsLaunchTests.swift +// leanring-buddyUITests +// +// Created by thorfinn on 3/2/26. +// + +import XCTest + +final class leanring_buddyUITestsLaunchTests: XCTestCase { + + override class var runsForEachTargetApplicationUIConfiguration: Bool { + true + } + + override func setUpWithError() throws { + continueAfterFailure = false + } + + @MainActor + func testLaunch() throws { + let app = XCUIApplication() + app.launch() + + // Insert steps here to perform after app launch but before taking a screenshot, + // such as logging into a test account or navigating somewhere in the app + + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = "Launch Screen" + attachment.lifetime = .keepAlways + add(attachment) + } +} diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..9b2f0b3 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,62 @@ +# Release Scripts + +## `release.sh` — Ship a new version of makesomething + +Automates the full release pipeline: build → sign → DMG → notarize → Sparkle appcast → GitHub Release. + +### Quick start + +```bash +# Auto-bumps version and build number from the latest GitHub Release +./scripts/release.sh +``` + +The script checks GitHub for the latest release (e.g. `v1.5`, build 6) and automatically bumps to `v1.6`, build 7. You'll see a confirmation prompt before anything runs. + +### Override version or build + +```bash +# Set a specific marketing version (auto-bumps build) +./scripts/release.sh 2.0 + +# Set both marketing version and build number +./scripts/release.sh 2.0 10 +``` + +### Safety + +- **Duplicate detection**: If the tag already exists on GitHub, the script exits with an error and suggests what to do. +- **Confirmation prompt**: Shows the version, build, and previous release before proceeding. Press `y` to continue. + +### What it does + +1. Fetches the latest release from GitHub to determine version + build +2. Archives the app via `xcodebuild` +3. Exports a signed `.app` with Developer ID +4. Creates a DMG with the drag-to-Applications background +5. Notarizes the DMG with Apple (Gatekeeper compliance) +6. Signs the DMG with the Sparkle EdDSA key +7. Generates `appcast.xml` for Sparkle auto-updates +8. Creates a GitHub Release with the DMG attached +9. Pushes the updated `appcast.xml` to the releases repo + +### One-time setup (prerequisites) + +1. **Xcode** with your Developer ID signing certificate +2. **Homebrew tools**: + ```bash + brew install create-dmg gh + ``` +3. **GitHub CLI auth**: + ```bash + gh auth login + ``` +4. **Apple notarization credentials** (stored in Keychain): + ```bash + xcrun notarytool store-credentials "AC_PASSWORD" \ + --apple-id YOUR_APPLE_ID \ + --team-id YOUR_TEAM_ID + ``` + You'll be prompted for an app-specific password (generate one at [appleid.apple.com](https://appleid.apple.com)). +5. **Sparkle EdDSA key** — already generated and stored in Keychain (done during initial Sparkle setup) +6. **Build the project in Xcode at least once** so SPM downloads Sparkle and the Sparkle CLI tools are available diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..da7f31d --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,276 @@ +#!/bin/bash +set -euo pipefail + +# Add Homebrew to PATH so create-dmg and gh are available in non-interactive shells +export PATH="/opt/homebrew/bin:$PATH" + +# ============================================================================= +# release.sh — Automates the full release pipeline for makesomething +# +# What it does (in order): +# 1. Auto-detects version + build from the latest GitHub Release +# 2. Archives the app via xcodebuild +# 3. Exports a signed + notarized .app +# 4. Wraps it in a DMG with the drag-to-Applications background +# 5. Notarizes the DMG with Apple (so Gatekeeper won't block it) +# 6. Signs the DMG with your Sparkle EdDSA key +# 7. Generates/updates appcast.xml automatically +# 8. Creates a GitHub Release with the DMG attached +# 9. Pushes the updated appcast.xml to the releases repo (makesomething-mac-app) +# +# Usage: +# ./scripts/release.sh Auto-bumps: 1.5 → 1.6, build 6 → 7 +# ./scripts/release.sh 2.0 Sets marketing version to 2.0, auto-bumps build +# ./scripts/release.sh 2.0 10 Sets both marketing version and build number +# +# Prerequisites (one-time setup): +# - Xcode with your Developer ID signing certificate +# - `brew install create-dmg gh` +# - `gh auth login` (GitHub CLI authenticated) +# - Sparkle EdDSA key in your Keychain (already generated) +# - `xcrun notarytool store-credentials "AC_PASSWORD"` (Apple notarization credentials) +# ============================================================================= + +# ── Configuration ──────────────────────────────────────────────────────────── + +SCHEME="leanring-buddy" +APP_NAME="makesomething" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +BUILD_DIR="${PROJECT_DIR}/build" +ARCHIVE_PATH="${BUILD_DIR}/${APP_NAME}.xcarchive" +EXPORT_DIR="${BUILD_DIR}/export" +DMG_OUTPUT_DIR="${BUILD_DIR}/dmg" +RELEASES_DIR="${PROJECT_DIR}/releases" # where generate_appcast reads DMGs from +DMG_BACKGROUND="${PROJECT_DIR}/dmg-background.png" + +GITHUB_REPO="julianjear/makesomething-mac-app" + +# Sparkle tools (auto-discovered from Xcode's SPM cache) +SPARKLE_BIN=$(find ~/Library/Developer/Xcode/DerivedData/leanring-buddy*/SourcePackages/artifacts/sparkle/Sparkle/bin -maxdepth 0 2>/dev/null | head -1) + +if [ -z "$SPARKLE_BIN" ]; then + echo "❌ Sparkle tools not found. Build the project in Xcode first so SPM downloads Sparkle." + exit 1 +fi + +# ── Auto-detect version from latest GitHub Release ────────────────────────── +# Fetches the latest release tag (e.g. "v1.5") and build number from GitHub. +# If no arguments are provided, bumps the minor version by 0.1 and build by 1. +# You can override either or both by passing arguments. + +echo "🔍 Checking latest release on GitHub..." + +LATEST_TAG=$(gh release view --repo "${GITHUB_REPO}" --json tagName --jq '.tagName' 2>/dev/null || echo "") + +if [ -n "$LATEST_TAG" ]; then + # Strip the "v" prefix to get the version number (e.g. "v1.5" → "1.5") + LATEST_VERSION="${LATEST_TAG#v}" + + # Get the build number from the latest release's app bundle inside the DMG. + # We download just the release metadata (not the DMG) and parse the body/notes, + # but the simplest reliable approach is to track it from the GitHub release title + # or from a known incrementing sequence. We use the GitHub API to get asset info + # and derive the build number from the release list count. + LATEST_BUILD=$(gh release list --repo "${GITHUB_REPO}" --json tagName --jq 'length' 2>/dev/null || echo "0") + + echo " Latest release: ${LATEST_TAG} (build ${LATEST_BUILD})" +else + LATEST_VERSION="0.0" + LATEST_BUILD=0 + echo " No previous releases found — starting from scratch" +fi + +# Determine the next marketing version: bump minor by 0.1 +# e.g. "1.5" → "1.6", "2.9" → "3.0" (carries over) +if [ $# -ge 1 ]; then + MARKETING_VERSION="$1" +else + MAJOR=$(echo "$LATEST_VERSION" | cut -d. -f1) + MINOR=$(echo "$LATEST_VERSION" | cut -d. -f2) + NEXT_MINOR=$((MINOR + 1)) + if [ "$NEXT_MINOR" -ge 10 ]; then + MAJOR=$((MAJOR + 1)) + NEXT_MINOR=0 + fi + MARKETING_VERSION="${MAJOR}.${NEXT_MINOR}" +fi + +# Determine the next build number: always increment by 1 +if [ $# -ge 2 ]; then + BUILD_NUMBER="$2" +else + BUILD_NUMBER=$((LATEST_BUILD + 1)) +fi + +DMG_FILENAME="${APP_NAME}.dmg" +TAG="v${MARKETING_VERSION}" + +# ── Safety checks ──────────────────────────────────────────────────────────── + +# Check if this tag already exists on GitHub to prevent accidental duplicates +if gh release view "${TAG}" --repo "${GITHUB_REPO}" &>/dev/null; then + echo "" + echo "❌ Release ${TAG} already exists on GitHub!" + echo " https://github.com/${GITHUB_REPO}/releases/tag/${TAG}" + echo "" + echo " To release a new version, either:" + echo " • Run without arguments to auto-bump: ./scripts/release.sh" + echo " • Specify a higher version: ./scripts/release.sh $(echo "${MARKETING_VERSION} + 0.1" | bc)" + echo " • Delete the existing release first: gh release delete ${TAG} --repo ${GITHUB_REPO} --yes" + exit 1 +fi + +echo "" +echo "🚀 Releasing ${APP_NAME} v${MARKETING_VERSION} (build ${BUILD_NUMBER})" +echo " Previous: ${LATEST_TAG:-none}" +echo "" + +# Confirm with the user before proceeding +read -p " Proceed? (y/N) " -n 1 -r +echo "" +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo " Aborted." + exit 0 +fi +echo "" + +# ── Step 1: Clean build directory ──────────────────────────────────────────── + +echo "🧹 Cleaning build directory and stale DMGs..." +rm -rf "${BUILD_DIR}" +# Remove any leftover temp DMGs from create-dmg (rw.*.dmg) and the previous +# same-named DMG so create-dmg and generate_appcast don't choke on duplicates. +rm -f "${RELEASES_DIR}"/rw.*.dmg "${RELEASES_DIR}/${DMG_FILENAME}" +mkdir -p "${BUILD_DIR}" "${EXPORT_DIR}" "${DMG_OUTPUT_DIR}" "${RELEASES_DIR}" + +# ── Step 2: Archive ────────────────────────────────────────────────────────── + +echo "📦 Archiving..." +xcodebuild archive \ + -scheme "${SCHEME}" \ + -archivePath "${ARCHIVE_PATH}" \ + MARKETING_VERSION="${MARKETING_VERSION}" \ + CURRENT_PROJECT_VERSION="${BUILD_NUMBER}" \ + 2>&1 | tail -5 + +echo "✅ Archive created" + +# ── Step 3: Export (signed + notarized) ────────────────────────────────────── + +# Create an export options plist for Developer ID distribution. +# This tells xcodebuild to sign with your Developer ID certificate +# and submit to Apple for notarization automatically. +EXPORT_OPTIONS="${BUILD_DIR}/ExportOptions.plist" +cat > "${EXPORT_OPTIONS}" << 'PLIST' + + + + + method + developer-id + destination + export + + +PLIST + +echo "📤 Exporting (signing + notarizing — this may take a few minutes)..." +xcodebuild -exportArchive \ + -archivePath "${ARCHIVE_PATH}" \ + -exportPath "${EXPORT_DIR}" \ + -exportOptionsPlist "${EXPORT_OPTIONS}" \ + 2>&1 | tail -5 + +echo "✅ Export complete (signed + notarized)" + +# ── Step 4: Create DMG ────────────────────────────────────────────────────── + +DMG_PATH="${RELEASES_DIR}/${DMG_FILENAME}" + +echo "💿 Creating DMG..." +create-dmg \ + --volname "${APP_NAME}" \ + --window-pos 200 120 \ + --window-size 660 400 \ + --icon-size 100 \ + --icon "${APP_NAME}.app" 160 195 \ + --app-drop-link 500 195 \ + --background "${DMG_BACKGROUND}" \ + "${DMG_PATH}" \ + "${EXPORT_DIR}/${APP_NAME}.app" \ + 2>&1 | tail -3 + +echo "✅ DMG created: ${DMG_PATH}" + +# ── Step 5: Notarize the DMG ───────────────────────────────────────────────── +# The .app inside the DMG is already signed with Developer ID, but the DMG +# itself needs to be submitted to Apple for notarization so Gatekeeper +# allows users to open it without the "Apple could not verify" warning. +# Requires stored credentials: xcrun notarytool store-credentials "AC_PASSWORD" + +echo "🔏 Notarizing DMG with Apple (this may take a few minutes)..." +xcrun notarytool submit "${DMG_PATH}" \ + --keychain-profile "AC_PASSWORD" \ + --wait + +echo "📎 Stapling notarization ticket to DMG..." +xcrun stapler staple "${DMG_PATH}" + +echo "✅ DMG notarized and stapled" + +# ── Step 6: Sign DMG with Sparkle EdDSA key ───────────────────────────────── + +echo "🔐 Signing DMG with Sparkle EdDSA key..." +"${SPARKLE_BIN}/sign_update" "${DMG_PATH}" + +# ── Step 7: Generate / update appcast.xml ──────────────────────────────────── +# generate_appcast reads all DMGs in the releases/ directory, extracts version +# info from the app bundle inside each DMG, signs with your EdDSA key, and +# produces appcast.xml. The --download-url-prefix tells it where users will +# actually download the DMG from (GitHub Releases). + +echo "📡 Generating appcast.xml..." +"${SPARKLE_BIN}/generate_appcast" \ + --download-url-prefix "https://github.com/${GITHUB_REPO}/releases/download/${TAG}/" \ + -o "${PROJECT_DIR}/appcast.xml" \ + "${RELEASES_DIR}" + +echo "✅ appcast.xml updated" + +# ── Step 8: Create GitHub Release ──────────────────────────────────────────── +# Create the release first so the DMG download URL is live before we push the +# appcast that references it. + +echo "🏷️ Creating GitHub Release ${TAG}..." +gh release create "${TAG}" "${DMG_PATH}" \ + --repo "${GITHUB_REPO}" \ + --title "v${MARKETING_VERSION}" \ + --notes "makesomething v${MARKETING_VERSION}" \ + --latest + +# ── Step 9: Push appcast.xml to the releases repo ─────────────────────────── +# The appcast lives in makesomething-mac-app (the releases repo), not in the +# source code repo. We clone it to a temp dir, copy the new appcast, and push. + +echo "📝 Pushing appcast.xml to ${GITHUB_REPO}..." +RELEASES_REPO_DIR=$(mktemp -d) +git clone --depth 1 "https://github.com/${GITHUB_REPO}.git" "${RELEASES_REPO_DIR}" 2>&1 | tail -2 +cp "${PROJECT_DIR}/appcast.xml" "${RELEASES_REPO_DIR}/appcast.xml" +cd "${RELEASES_REPO_DIR}" +git add appcast.xml +git commit -m "Update appcast.xml for v${MARKETING_VERSION}" || echo " (no changes to commit)" +git push || echo " (push failed — you may need to push manually)" +cd "${PROJECT_DIR}" +rm -rf "${RELEASES_REPO_DIR}" + +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo "✅ Release v${MARKETING_VERSION} (build ${BUILD_NUMBER}) complete!" +echo "" +echo " DMG: ${DMG_PATH}" +echo " Appcast: ${PROJECT_DIR}/appcast.xml" +echo " Release: https://github.com/${GITHUB_REPO}/releases/tag/${TAG}" +echo "" +echo " Download URL (always latest):" +echo " https://github.com/${GITHUB_REPO}/releases/latest/download/${DMG_FILENAME}" +echo "═══════════════════════════════════════════════════════════════" diff --git a/worker/package-lock.json b/worker/package-lock.json new file mode 100644 index 0000000..c2383cc --- /dev/null +++ b/worker/package-lock.json @@ -0,0 +1,1617 @@ +{ + "name": "clicky-proxy", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "clicky-proxy", + "devDependencies": { + "wrangler": "^3.0.0" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.3.4.tgz", + "integrity": "sha512-YLPHc8yASwjNkmcDMQMY35yiWjoKAKnhUbPRszBRS0YgH+IXtsMp61j+yTcnCE3oO2DgP0U3iejLC8FTtKDC8Q==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "mime": "^3.0.0" + }, + "engines": { + "node": ">=16.13" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.0.2.tgz", + "integrity": "sha512-nyzYnlZjjV5xT3LizahG1Iu6mnrCaxglJ04rZLpDwlDVDZ7v46lNsfxhV3A/xtfgQuSHmLnc6SVI+KwBpc3Lwg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.14", + "workerd": "^1.20250124.0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20250718.0.tgz", + "integrity": "sha512-FHf4t7zbVN8yyXgQ/r/GqLPaYZSGUVzeR7RnL28Mwj2djyw2ZergvytVc7fdGcczl6PQh+VKGfZCfUqpJlbi9g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20250718.0.tgz", + "integrity": "sha512-fUiyUJYyqqp4NqJ0YgGtp4WJh/II/YZsUnEb6vVy5Oeas8lUOxnN+ZOJ8N/6/5LQCVAtYCChRiIrBbfhTn5Z8Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20250718.0.tgz", + "integrity": "sha512-5+eb3rtJMiEwp08Kryqzzu8d1rUcK+gdE442auo5eniMpT170Dz0QxBrqkg2Z48SFUPYbj+6uknuA5tzdRSUSg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20250718.0.tgz", + "integrity": "sha512-Aa2M/DVBEBQDdATMbn217zCSFKE+ud/teS+fFS+OQqKABLn0azO2qq6ANAHYOIE6Q3Sq4CxDIQr8lGdaJHwUog==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20250718.0.tgz", + "integrity": "sha512-dY16RXKffmugnc67LTbyjdDHZn5NoTF1yHEf2fN4+OaOnoGSp3N1x77QubTDwqZ9zECWxgQfDLjddcH8dWeFhg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild-plugins/node-globals-polyfill": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@esbuild-plugins/node-globals-polyfill/-/node-globals-polyfill-0.2.3.tgz", + "integrity": "sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "esbuild": "*" + } + }, + "node_modules/@esbuild-plugins/node-modules-polyfill": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@esbuild-plugins/node-modules-polyfill/-/node-modules-polyfill-0.2.2.tgz", + "integrity": "sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA==", + "dev": true, + "license": "ISC", + "dependencies": { + "escape-string-regexp": "^4.0.0", + "rollup-plugin-node-polyfills": "^0.2.1" + }, + "peerDependencies": { + "esbuild": "*" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", + "integrity": "sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz", + "integrity": "sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.19.tgz", + "integrity": "sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.19.tgz", + "integrity": "sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz", + "integrity": "sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.19.tgz", + "integrity": "sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.19.tgz", + "integrity": "sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.19.tgz", + "integrity": "sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz", + "integrity": "sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.19.tgz", + "integrity": "sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.19.tgz", + "integrity": "sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.19.tgz", + "integrity": "sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.19.tgz", + "integrity": "sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.19.tgz", + "integrity": "sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.19.tgz", + "integrity": "sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz", + "integrity": "sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.19.tgz", + "integrity": "sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.19.tgz", + "integrity": "sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.19.tgz", + "integrity": "sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz", + "integrity": "sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.19.tgz", + "integrity": "sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz", + "integrity": "sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz", + "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/as-table": { + "version": "1.0.55", + "resolved": "https://registry.npmjs.org/as-table/-/as-table-1.0.55.tgz", + "integrity": "sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "printable-characters": "^1.0.42" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.2.tgz", + "integrity": "sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==", + "dev": true, + "license": "MIT" + }, + "node_modules/defu": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.6.tgz", + "integrity": "sha512-f8mefEW4WIVg4LckePx3mALjQSPQgFlg9U8yaPdlsbdYcHQyj9n2zL2LJEA52smeYxOvmd/nB7TpMtHGMTHcug==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/esbuild": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz", + "integrity": "sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.17.19", + "@esbuild/android-arm64": "0.17.19", + "@esbuild/android-x64": "0.17.19", + "@esbuild/darwin-arm64": "0.17.19", + "@esbuild/darwin-x64": "0.17.19", + "@esbuild/freebsd-arm64": "0.17.19", + "@esbuild/freebsd-x64": "0.17.19", + "@esbuild/linux-arm": "0.17.19", + "@esbuild/linux-arm64": "0.17.19", + "@esbuild/linux-ia32": "0.17.19", + "@esbuild/linux-loong64": "0.17.19", + "@esbuild/linux-mips64el": "0.17.19", + "@esbuild/linux-ppc64": "0.17.19", + "@esbuild/linux-riscv64": "0.17.19", + "@esbuild/linux-s390x": "0.17.19", + "@esbuild/linux-x64": "0.17.19", + "@esbuild/netbsd-x64": "0.17.19", + "@esbuild/openbsd-x64": "0.17.19", + "@esbuild/sunos-x64": "0.17.19", + "@esbuild/win32-arm64": "0.17.19", + "@esbuild/win32-ia32": "0.17.19", + "@esbuild/win32-x64": "0.17.19" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/estree-walker": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz", + "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/exit-hook": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-2.2.1.tgz", + "integrity": "sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/exsolve": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.8.tgz", + "integrity": "sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA==", + "dev": true, + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-source": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/get-source/-/get-source-2.0.12.tgz", + "integrity": "sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==", + "dev": true, + "license": "Unlicense", + "dependencies": { + "data-uri-to-buffer": "^2.0.0", + "source-map": "^0.6.1" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/magic-string": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", + "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "sourcemap-codec": "^1.4.8" + } + }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/miniflare": { + "version": "3.20250718.3", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20250718.3.tgz", + "integrity": "sha512-JuPrDJhwLrNLEJiNLWO7ZzJrv/Vv9kZuwMYCfv0LskQDM6Eonw4OvywO3CH/wCGjgHzha/qyjUh8JQ068TjDgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "acorn": "8.14.0", + "acorn-walk": "8.3.2", + "exit-hook": "2.2.1", + "glob-to-regexp": "0.4.1", + "stoppable": "1.1.0", + "undici": "^5.28.5", + "workerd": "1.20250718.0", + "ws": "8.18.0", + "youch": "3.3.4", + "zod": "3.22.3" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=16.13" + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/printable-characters": { + "version": "1.0.42", + "resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz", + "integrity": "sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==", + "dev": true, + "license": "Unlicense" + }, + "node_modules/rollup-plugin-inject": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rollup-plugin-inject/-/rollup-plugin-inject-3.0.2.tgz", + "integrity": "sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==", + "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject.", + "dev": true, + "license": "MIT", + "dependencies": { + "estree-walker": "^0.6.1", + "magic-string": "^0.25.3", + "rollup-pluginutils": "^2.8.1" + } + }, + "node_modules/rollup-plugin-node-polyfills": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/rollup-plugin-node-polyfills/-/rollup-plugin-node-polyfills-0.2.1.tgz", + "integrity": "sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==", + "dev": true, + "license": "MIT", + "dependencies": { + "rollup-plugin-inject": "^3.0.0" + } + }, + "node_modules/rollup-pluginutils": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz", + "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "estree-walker": "^0.6.1" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", + "deprecated": "Please use @jridgewell/sourcemap-codec instead", + "dev": true, + "license": "MIT" + }, + "node_modules/stacktracey": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/stacktracey/-/stacktracey-2.2.0.tgz", + "integrity": "sha512-ETyQEz+CzXiLjEbyJqpbp+/T79RQD/6wqFucRBIlVNZfYq2Ay7wbretD4cxpbymZlaPWx58aIhPEY1Cr8DlVvg==", + "dev": true, + "license": "Unlicense", + "dependencies": { + "as-table": "^1.0.36", + "get-source": "^2.0.12" + } + }, + "node_modules/stoppable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", + "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4", + "npm": ">=6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/ufo": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", + "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/unenv": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.14.tgz", + "integrity": "sha512-od496pShMen7nOy5VmVJCnq8rptd45vh6Nx/r2iPbrba6pa6p+tS2ywuIHRZ/OBvSbQZB0kWvpO9XBNVFXHD3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "defu": "^6.1.4", + "exsolve": "^1.0.1", + "ohash": "^2.0.10", + "pathe": "^2.0.3", + "ufo": "^1.5.4" + } + }, + "node_modules/workerd": { + "version": "1.20250718.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20250718.0.tgz", + "integrity": "sha512-kqkIJP/eOfDlUyBzU7joBg+tl8aB25gEAGqDap+nFWb+WHhnooxjGHgxPBy3ipw2hnShPFNOQt5lFRxbwALirg==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20250718.0", + "@cloudflare/workerd-darwin-arm64": "1.20250718.0", + "@cloudflare/workerd-linux-64": "1.20250718.0", + "@cloudflare/workerd-linux-arm64": "1.20250718.0", + "@cloudflare/workerd-windows-64": "1.20250718.0" + } + }, + "node_modules/wrangler": { + "version": "3.114.17", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.114.17.tgz", + "integrity": "sha512-tAvf7ly+tB+zwwrmjsCyJ2pJnnc7SZhbnNwXbH+OIdVas3zTSmjcZOjmLKcGGptssAA3RyTKhcF9BvKZzMUycA==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.3.4", + "@cloudflare/unenv-preset": "2.0.2", + "@esbuild-plugins/node-globals-polyfill": "0.2.3", + "@esbuild-plugins/node-modules-polyfill": "0.2.2", + "blake3-wasm": "2.1.5", + "esbuild": "0.17.19", + "miniflare": "3.20250718.3", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.14", + "workerd": "1.20250718.0" + }, + "bin": { + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=16.17.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2", + "sharp": "^0.33.5" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.20250408.0" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/youch/-/youch-3.3.4.tgz", + "integrity": "sha512-UeVBXie8cA35DS6+nBkls68xaBBXCye0CNznrhszZjTbRVnJKQuNsyLKBTTL4ln1o1rh2PKtv35twV7irj5SEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cookie": "^0.7.1", + "mustache": "^4.2.0", + "stacktracey": "^2.1.8" + } + }, + "node_modules/zod": { + "version": "3.22.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.3.tgz", + "integrity": "sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/worker/package.json b/worker/package.json new file mode 100644 index 0000000..445d90a --- /dev/null +++ b/worker/package.json @@ -0,0 +1,11 @@ +{ + "name": "clicky-proxy", + "private": true, + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "wrangler": "^3.0.0" + } +} diff --git a/worker/src/index.ts b/worker/src/index.ts new file mode 100644 index 0000000..2e3e934 --- /dev/null +++ b/worker/src/index.ts @@ -0,0 +1,141 @@ +/** + * Clicky Proxy Worker + * + * Proxies requests to Claude and ElevenLabs APIs so the app never + * ships with raw API keys. Keys are stored as Cloudflare secrets. + * + * Routes: + * POST /chat → Anthropic Messages API (streaming) + * POST /tts → ElevenLabs TTS API + */ + +interface Env { + ANTHROPIC_API_KEY: string; + ELEVENLABS_API_KEY: string; + ELEVENLABS_VOICE_ID: string; + ASSEMBLYAI_API_KEY: string; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + + if (request.method !== "POST") { + return new Response("Method not allowed", { status: 405 }); + } + + try { + if (url.pathname === "/chat") { + return await handleChat(request, env); + } + + if (url.pathname === "/tts") { + return await handleTTS(request, env); + } + + if (url.pathname === "/transcribe-token") { + return await handleTranscribeToken(env); + } + } catch (error) { + console.error(`[${url.pathname}] Unhandled error:`, error); + return new Response( + JSON.stringify({ error: String(error) }), + { status: 500, headers: { "content-type": "application/json" } } + ); + } + + return new Response("Not found", { status: 404 }); + }, +}; + +async function handleChat(request: Request, env: Env): Promise { + const body = await request.text(); + + const response = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "x-api-key": env.ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body, + }); + + if (!response.ok) { + const errorBody = await response.text(); + console.error(`[/chat] Anthropic API error ${response.status}: ${errorBody}`); + return new Response(errorBody, { + status: response.status, + headers: { "content-type": "application/json" }, + }); + } + + return new Response(response.body, { + status: response.status, + headers: { + "content-type": response.headers.get("content-type") || "text/event-stream", + "cache-control": "no-cache", + }, + }); +} + +async function handleTranscribeToken(env: Env): Promise { + const response = await fetch( + "https://streaming.assemblyai.com/v3/token?expires_in_seconds=480", + { + method: "GET", + headers: { + authorization: env.ASSEMBLYAI_API_KEY, + }, + } + ); + + if (!response.ok) { + const errorBody = await response.text(); + console.error(`[/transcribe-token] AssemblyAI token error ${response.status}: ${errorBody}`); + return new Response(errorBody, { + status: response.status, + headers: { "content-type": "application/json" }, + }); + } + + const data = await response.text(); + return new Response(data, { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +async function handleTTS(request: Request, env: Env): Promise { + const body = await request.text(); + const voiceId = env.ELEVENLABS_VOICE_ID; + + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, + { + method: "POST", + headers: { + "xi-api-key": env.ELEVENLABS_API_KEY, + "content-type": "application/json", + accept: "audio/mpeg", + }, + body, + } + ); + + if (!response.ok) { + const errorBody = await response.text(); + console.error(`[/tts] ElevenLabs API error ${response.status}: ${errorBody}`); + return new Response(errorBody, { + status: response.status, + headers: { "content-type": "application/json" }, + }); + } + + return new Response(response.body, { + status: response.status, + headers: { + "content-type": response.headers.get("content-type") || "audio/mpeg", + }, + }); +} diff --git a/worker/wrangler.toml b/worker/wrangler.toml new file mode 100644 index 0000000..b4bdbf3 --- /dev/null +++ b/worker/wrangler.toml @@ -0,0 +1,6 @@ +name = "clicky-proxy" +main = "src/index.ts" +compatibility_date = "2024-01-01" + +[vars] +ELEVENLABS_VOICE_ID = "kPzsL2i3teMYv0FxEYQ6"