wehub-resource-sync
1207 lines
47 KiB
PowerShell
1207 lines
47 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$OperationPath
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
|
|
[Console]::InputEncoding = $utf8NoBom
|
|
[Console]::OutputEncoding = $utf8NoBom
|
|
$OutputEncoding = $utf8NoBom
|
|
|
|
Add-Type -AssemblyName UIAutomationClient
|
|
Add-Type -AssemblyName UIAutomationTypes
|
|
Add-Type -AssemblyName System.Drawing
|
|
Add-Type -AssemblyName System.Windows.Forms
|
|
|
|
Add-Type -TypeDefinition @"
|
|
using System;
|
|
using System.Runtime.InteropServices;
|
|
|
|
public static class OrcaDesktopWin32 {
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
public struct RECT {
|
|
public int Left;
|
|
public int Top;
|
|
public int Right;
|
|
public int Bottom;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
public struct POINT {
|
|
public int X;
|
|
public int Y;
|
|
}
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool ScreenToClient(IntPtr hwnd, ref POINT point);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool PostMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool ShowWindow(IntPtr hwnd, int nCmdShow);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool SetForegroundWindow(IntPtr hwnd);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern IntPtr GetForegroundWindow();
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern bool SetCursorPos(int x, int y);
|
|
|
|
[DllImport("user32.dll")]
|
|
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, int dwData, UIntPtr dwExtraInfo);
|
|
}
|
|
"@
|
|
|
|
$MaxNodes = 1200
|
|
$MaxDepth = 64
|
|
$TextLimit = 500
|
|
$MaxScreenshotPngBytes = 900000
|
|
$MaxScreenshotEdge = 1280
|
|
$MinScreenshotScale = 0.25
|
|
$ScreenshotScaleStep = 0.85
|
|
$BlockedAppFragments = @(
|
|
"1password",
|
|
"bitwarden",
|
|
"dashlane",
|
|
"lastpass",
|
|
"nordpass",
|
|
"proton pass"
|
|
)
|
|
|
|
$WindowsMessages = @{
|
|
Char = 0x0102
|
|
KeyDown = 0x0100
|
|
KeyUp = 0x0101
|
|
MouseMove = 0x0200
|
|
LeftDown = 0x0201
|
|
LeftUp = 0x0202
|
|
RightDown = 0x0204
|
|
RightUp = 0x0205
|
|
MiddleDown = 0x0207
|
|
MiddleUp = 0x0208
|
|
Wheel = 0x020A
|
|
}
|
|
|
|
$MouseEvents = @{
|
|
LeftDown = 0x0002
|
|
LeftUp = 0x0004
|
|
RightDown = 0x0008
|
|
RightUp = 0x0010
|
|
MiddleDown = 0x0020
|
|
MiddleUp = 0x0040
|
|
Wheel = 0x0800
|
|
}
|
|
|
|
function Write-OrcaJson($Payload) {
|
|
$Payload | ConvertTo-Json -Depth 100 -Compress
|
|
}
|
|
|
|
function New-OrcaFrame([double]$X, [double]$Y, [double]$Width, [double]$Height) {
|
|
if ($Width -le 0 -or $Height -le 0) { return $null }
|
|
[pscustomobject]@{ x = $X; y = $Y; width = $Width; height = $Height }
|
|
}
|
|
|
|
function Read-OrcaOperation([string]$Path) {
|
|
Get-Content -Raw -Encoding UTF8 -Path $Path | ConvertFrom-Json
|
|
}
|
|
|
|
function ConvertTo-OrcaLParam([int]$X, [int]$Y) {
|
|
[IntPtr]((($Y -band 0xffff) -shl 16) -bor ($X -band 0xffff))
|
|
}
|
|
|
|
function ConvertTo-OrcaWheelParam([int]$Delta) {
|
|
[IntPtr](($Delta -band 0xffff) -shl 16)
|
|
}
|
|
|
|
function Get-OrcaWindowProcesses {
|
|
@(Get-Process | Where-Object { $_.MainWindowHandle -ne 0 } | Sort-Object ProcessName, Id)
|
|
}
|
|
|
|
function Find-OrcaProcess([string]$Query) {
|
|
$needle = ""
|
|
if ($null -ne $Query) { $needle = $Query.Trim() }
|
|
if ([string]::IsNullOrWhiteSpace($needle)) { throw 'appNotFound("")' }
|
|
if ($needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
$needle = $needle.Substring(4)
|
|
}
|
|
|
|
$parsedProcessId = 0
|
|
$processes = Get-OrcaWindowProcesses
|
|
if ([int]::TryParse($needle, [ref]$parsedProcessId)) {
|
|
$match = $processes | Where-Object { $_.Id -eq $parsedProcessId } | Select-Object -First 1
|
|
if ($null -ne $match) {
|
|
Assert-OrcaProcessAllowed $match
|
|
return $match
|
|
}
|
|
}
|
|
|
|
$processNeedle = $needle
|
|
if ($processNeedle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
$processNeedle = $processNeedle.Substring(0, $processNeedle.Length - 4)
|
|
}
|
|
|
|
$match = $processes | Where-Object {
|
|
$_.ProcessName -ieq $processNeedle -or
|
|
"$($_.ProcessName).exe" -ieq $needle -or
|
|
$_.MainWindowTitle -ieq $needle -or
|
|
$_.MainWindowTitle -ilike "*$needle*"
|
|
} | Select-Object -First 1
|
|
if ($null -ne $match) {
|
|
Assert-OrcaProcessAllowed $match
|
|
return $match
|
|
}
|
|
|
|
throw "appNotFound(`"$Query`")"
|
|
}
|
|
|
|
function Assert-OrcaProcessAllowed($Process) {
|
|
$values = @($Process.ProcessName, $Process.MainWindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() }
|
|
foreach ($fragment in $BlockedAppFragments) {
|
|
foreach ($value in $values) {
|
|
if ($value.Contains($fragment)) {
|
|
throw "appBlocked(`"$($Process.ProcessName)`")"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-OrcaBrowserProcess($Process) {
|
|
$name = ([string]$Process.ProcessName).ToLowerInvariant()
|
|
$browserProcesses = @(
|
|
"arc",
|
|
"brave",
|
|
"chrome",
|
|
"chromium",
|
|
"firefox",
|
|
"librewolf",
|
|
"msedge",
|
|
"opera",
|
|
"vivaldi",
|
|
"zen"
|
|
)
|
|
$browserProcesses -contains $name
|
|
}
|
|
|
|
function Get-OrcaRootElement($Process) {
|
|
if ($Process.MainWindowHandle -eq 0) {
|
|
throw "No top-level UI Automation window is available for $($Process.ProcessName)."
|
|
}
|
|
[Windows.Automation.AutomationElement]::FromHandle([IntPtr]$Process.MainWindowHandle)
|
|
}
|
|
|
|
function Get-OrcaWindowFrame($Process, $RootElement) {
|
|
$rect = New-Object OrcaDesktopWin32+RECT
|
|
if ([OrcaDesktopWin32]::GetWindowRect([IntPtr]$Process.MainWindowHandle, [ref]$rect)) {
|
|
return New-OrcaFrame $rect.Left $rect.Top ($rect.Right - $rect.Left) ($rect.Bottom - $rect.Top)
|
|
}
|
|
|
|
try {
|
|
$bounds = $RootElement.Current.BoundingRectangle
|
|
if (-not $bounds.IsEmpty) {
|
|
return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
|
|
}
|
|
} catch {}
|
|
$null
|
|
}
|
|
|
|
function Get-OrcaWindowId($Process) {
|
|
[int64]$Process.MainWindowHandle
|
|
}
|
|
|
|
function Get-OrcaAppName($Process) {
|
|
if ($Process.ProcessName -eq "ApplicationFrameHost" -and -not [string]::IsNullOrWhiteSpace($Process.MainWindowTitle)) {
|
|
return [string]$Process.MainWindowTitle
|
|
}
|
|
[string]$Process.ProcessName
|
|
}
|
|
|
|
function New-OrcaAppRecord($Process) {
|
|
[pscustomobject]@{
|
|
name = Get-OrcaAppName $Process
|
|
bundleIdentifier = $Process.ProcessName
|
|
bundleId = $Process.ProcessName
|
|
pid = [int]$Process.Id
|
|
}
|
|
}
|
|
|
|
function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) {
|
|
if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
|
|
throw "windowNotFound(`"$WindowIndex`")"
|
|
}
|
|
if ($null -ne $WindowId -and [int64]$WindowId -ne (Get-OrcaWindowId $Process)) {
|
|
throw "windowNotFound(`"$WindowId`")"
|
|
}
|
|
}
|
|
|
|
function Restore-OrcaWindow($Process) {
|
|
if ($Process.MainWindowHandle -eq 0) { return }
|
|
[void][OrcaDesktopWin32]::ShowWindow([IntPtr]$Process.MainWindowHandle, 9)
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow([IntPtr]$Process.MainWindowHandle)
|
|
}
|
|
|
|
function Test-OrcaWindowFocused([IntPtr]$WindowHandle) {
|
|
[OrcaDesktopWin32]::GetForegroundWindow() -eq $WindowHandle
|
|
}
|
|
|
|
function Wait-OrcaWindowFocused([IntPtr]$WindowHandle, [int]$TimeoutMilliseconds) {
|
|
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($stopwatch.ElapsedMilliseconds -lt $TimeoutMilliseconds) {
|
|
if (Test-OrcaWindowFocused $WindowHandle) { return $true }
|
|
Start-Sleep -Milliseconds 50
|
|
}
|
|
Test-OrcaWindowFocused $WindowHandle
|
|
}
|
|
|
|
function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation) {
|
|
if (Test-OrcaWindowFocused $WindowHandle) { return }
|
|
if ([bool]$Operation.restoreWindow) {
|
|
if (Wait-OrcaWindowFocused $WindowHandle 500) { return }
|
|
throw "window_not_focused: keyboard input requires the target window to be focused; restoreWindow was requested but the target window is still not focused; bring it forward manually or check desktop permissions"
|
|
}
|
|
throw "window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window"
|
|
}
|
|
|
|
function Get-OrcaElementFrame($Element, $WindowFrame) {
|
|
try {
|
|
$bounds = $Element.Current.BoundingRectangle
|
|
if ($bounds.IsEmpty) { return $null }
|
|
if ($null -eq $WindowFrame) {
|
|
return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
|
|
}
|
|
New-OrcaFrame ($bounds.X - $WindowFrame.x) ($bounds.Y - $WindowFrame.y) $bounds.Width $bounds.Height
|
|
} catch {
|
|
$null
|
|
}
|
|
}
|
|
|
|
function Get-OrcaProperty($Element, [string]$Name) {
|
|
try { [string]$Element.Current.$Name } catch { "" }
|
|
}
|
|
|
|
function Get-OrcaRuntimeId($Element) {
|
|
try { @($Element.GetRuntimeId()) } catch { @() }
|
|
}
|
|
|
|
function Test-OrcaSensitiveElement($Element) {
|
|
try {
|
|
if ($Element.Current.IsPassword) { return $true }
|
|
} catch {}
|
|
$controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
|
|
$parts = @(
|
|
(Get-OrcaProperty $Element "LocalizedControlType"),
|
|
$controlType,
|
|
(Get-OrcaProperty $Element "Name"),
|
|
(Get-OrcaProperty $Element "AutomationId"),
|
|
(Get-OrcaProperty $Element "ClassName")
|
|
)
|
|
$haystack = (($parts -join " ") -replace "\s+", " ").ToLowerInvariant()
|
|
foreach ($term in @("password", "passcode", "secret", "one-time code", "verification code")) {
|
|
if ($haystack.Contains($term)) { return $true }
|
|
}
|
|
$haystack -match "(^|[^a-z0-9])pin([^a-z0-9]|$)"
|
|
}
|
|
|
|
function Get-OrcaValueText($Element) {
|
|
try {
|
|
if (Test-OrcaSensitiveElement $Element) { return "[redacted]" }
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
|
|
$rawValue = $pattern.Current.Value
|
|
$text = if ($null -eq $rawValue) { "" } else { [string]$rawValue }
|
|
if ($text.Length -gt $TextLimit) { return $text.Substring(0, $TextLimit) + "..." }
|
|
$text
|
|
} catch {
|
|
""
|
|
}
|
|
}
|
|
|
|
function Get-OrcaActions($Element) {
|
|
$actions = New-Object System.Collections.Generic.List[string]
|
|
foreach ($pattern in $Element.GetSupportedPatterns()) {
|
|
$name = [string]$pattern.ProgrammaticName
|
|
if ($name -like "InvokePatternIdentifiers.Pattern") { $actions.Add("Invoke") }
|
|
elseif ($name -like "TogglePatternIdentifiers.Pattern") { $actions.Add("Toggle") }
|
|
elseif ($name -like "SelectionItemPatternIdentifiers.Pattern") { $actions.Add("Select") }
|
|
elseif ($name -like "ScrollPatternIdentifiers.Pattern") { $actions.Add("Scroll") }
|
|
elseif ($name -like "ValuePatternIdentifiers.Pattern") { $actions.Add("SetValue") }
|
|
}
|
|
@($actions | Select-Object -Unique)
|
|
}
|
|
|
|
function Get-OrcaMeaningfulActions($Actions) {
|
|
$noisy = @("Invoke", "ScrollToVisible", "ShowMenu")
|
|
@($Actions | Where-Object { $noisy -notcontains $_ })
|
|
}
|
|
|
|
function Format-OrcaSnapshotText([string]$Text) {
|
|
if ([string]::IsNullOrWhiteSpace($Text)) { return "" }
|
|
(($Text -replace "\s+", " ").Trim())
|
|
}
|
|
|
|
function Format-OrcaValueSegment([string]$RoleKey, [string]$Title, [string]$Value) {
|
|
$clean = Format-OrcaSnapshotText $Value
|
|
if ([string]::IsNullOrWhiteSpace($clean) -or $clean -eq $Title) { return "" }
|
|
if ($RoleKey -eq "heading" -and $clean -match "^\d+$") { return "" }
|
|
if ($RoleKey -in @("text", "edit", "document", "scroll bar", "progress bar")) {
|
|
return " $clean"
|
|
}
|
|
", Value: $clean"
|
|
}
|
|
|
|
function Test-OrcaSuppressChildren([string]$RoleKey, [string]$Title, [string]$Value, [string]$Summary) {
|
|
$hasCompactLabel = -not [string]::IsNullOrWhiteSpace($Title) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Value)) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Summary))
|
|
$hasCompactLabel -and $RoleKey -in @(
|
|
"button",
|
|
"check box",
|
|
"combo box",
|
|
"heading",
|
|
"hyperlink",
|
|
"link",
|
|
"menu item",
|
|
"radio button",
|
|
"tab item"
|
|
)
|
|
}
|
|
|
|
function Get-OrcaTextSnippets($Element, [int]$Limit = 6, [int]$MaxDepth = 3) {
|
|
$values = New-Object System.Collections.Generic.List[string]
|
|
$seen = New-Object System.Collections.Generic.HashSet[string]
|
|
|
|
function Visit-OrcaText($Node, [int]$Depth) {
|
|
if ($values.Count -ge $Limit -or $Depth -gt $MaxDepth) { return }
|
|
$role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
|
|
if ($role -match "text|link|label") {
|
|
foreach ($raw in @((Get-OrcaProperty $Node "Name"), (Get-OrcaValueText $Node))) {
|
|
$value = (($raw -replace "\s+", " ").Trim())
|
|
if (-not [string]::IsNullOrWhiteSpace($value) -and $seen.Add($value)) {
|
|
if ($value.Length -gt 80) { $value = $value.Substring(0, 80) + "..." }
|
|
$values.Add($value)
|
|
if ($values.Count -ge $Limit) { return }
|
|
}
|
|
}
|
|
}
|
|
try {
|
|
$children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
|
|
for ($i = 0; $i -lt $children.Count; $i++) {
|
|
Visit-OrcaText $children.Item($i) ($Depth + 1)
|
|
if ($values.Count -ge $Limit) { return }
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
Visit-OrcaText $Element 0
|
|
@($values.ToArray())
|
|
}
|
|
|
|
function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) {
|
|
$script:sawOrcaText = $false
|
|
$allowed = @("pane", "group", "custom", "unknown", "text", "link", "image")
|
|
|
|
function Visit-OrcaPlainText($Node, [int]$Depth) {
|
|
if ($Depth -gt $MaxDepth) { return $false }
|
|
$role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
|
|
$roleKey = $role.ToLowerInvariant()
|
|
if ($allowed -notcontains $roleKey) { return $false }
|
|
if ($roleKey -match "text|link") { $script:sawOrcaText = $true }
|
|
if (@(Get-OrcaMeaningfulActions @(Get-OrcaActions $Node)).Count -gt 0) { return $false }
|
|
try {
|
|
$children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
|
|
for ($i = 0; $i -lt $children.Count; $i++) {
|
|
if (-not (Visit-OrcaPlainText $children.Item($i) ($Depth + 1))) { return $false }
|
|
}
|
|
} catch {}
|
|
return $true
|
|
}
|
|
|
|
(Visit-OrcaPlainText $Element 0) -and $script:sawOrcaText
|
|
}
|
|
|
|
function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
|
|
$controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
|
|
$nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 }
|
|
[pscustomobject]@{
|
|
index = $Index
|
|
runtimeId = @(Get-OrcaRuntimeId $Element)
|
|
automationId = Get-OrcaProperty $Element "AutomationId"
|
|
name = Get-OrcaProperty $Element "Name"
|
|
controlType = $controlType
|
|
localizedControlType = Get-OrcaProperty $Element "LocalizedControlType"
|
|
className = Get-OrcaProperty $Element "ClassName"
|
|
value = Get-OrcaValueText $Element
|
|
isSelected = Test-OrcaElementSelected $Element
|
|
nativeWindowHandle = $nativeWindowHandle
|
|
frame = Get-OrcaElementFrame $Element $WindowFrame
|
|
actions = @(Get-OrcaActions $Element)
|
|
}
|
|
}
|
|
|
|
function Test-OrcaElementSelected($Element) {
|
|
try {
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.SelectionItemPattern]::Pattern)
|
|
return [bool]$pattern.Current.IsSelected
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = $false) {
|
|
$records = New-Object System.Collections.Generic.List[object]
|
|
$lines = New-Object System.Collections.Generic.List[string]
|
|
$seen = New-Object System.Collections.Generic.HashSet[string]
|
|
$truncation = [pscustomobject]@{
|
|
truncated = $false
|
|
maxNodes = $MaxNodes
|
|
maxDepth = $MaxDepth
|
|
maxDepthReached = $false
|
|
}
|
|
|
|
function Visit-OrcaNode($Node, [int]$Depth) {
|
|
if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) {
|
|
$truncation.truncated = $true
|
|
if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true }
|
|
return
|
|
}
|
|
$identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() }
|
|
if (-not $seen.Add($identity)) { return }
|
|
|
|
$record = New-OrcaElementRecord $Node $records.Count $WindowFrame
|
|
$children = @()
|
|
try {
|
|
$children = @($Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition))
|
|
} catch {}
|
|
$meaningfulActions = @(Get-OrcaMeaningfulActions $record.actions)
|
|
$title = if ([string]::IsNullOrWhiteSpace($record.name)) { $record.automationId } else { $record.name }
|
|
$role = if ([string]::IsNullOrWhiteSpace($record.localizedControlType)) { $record.controlType } else { $record.localizedControlType }
|
|
$roleKey = $role.ToLowerInvariant()
|
|
$snippets = @(Get-OrcaTextSnippets $Node 8 4)
|
|
$genericSummary = $null
|
|
if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) {
|
|
$genericSummary = ($snippets -join " ")
|
|
}
|
|
if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) {
|
|
for ($i = 0; $i -lt $children.Count; $i++) {
|
|
Visit-OrcaNode $children.Item($i) $Depth
|
|
}
|
|
return
|
|
}
|
|
|
|
$records.Add($record)
|
|
|
|
$line = "$($record.index) $role $(Format-OrcaSnapshotText $title)".TrimEnd()
|
|
$line += Format-OrcaValueSegment $roleKey $title $record.value
|
|
if (-not [string]::IsNullOrWhiteSpace($genericSummary) -and $genericSummary -ne $title) {
|
|
$line += ", Text: " + (Format-OrcaSnapshotText $genericSummary)
|
|
} elseif ($roleKey -in @("row", "data item", "list item")) {
|
|
$rowSummary = @((Get-OrcaTextSnippets $Node 6 3)) -join " "
|
|
if (-not [string]::IsNullOrWhiteSpace($rowSummary) -and $rowSummary -ne $title) {
|
|
$line += ", Text: " + (Format-OrcaSnapshotText $rowSummary)
|
|
}
|
|
}
|
|
if ($meaningfulActions.Count -gt 0) {
|
|
$line += ", Secondary Actions: " + ($meaningfulActions -join ", ")
|
|
}
|
|
$lines.Add(("`t" * $Depth) + $line)
|
|
|
|
if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return }
|
|
$childLineStart = $lines.Count
|
|
for ($i = 0; $i -lt $children.Count; $i++) {
|
|
Visit-OrcaNode $children.Item($i) ($Depth + 1)
|
|
}
|
|
if ($CompactBrowserTabs) {
|
|
Compress-OrcaRenderedBrowserTabs $records $lines $childLineStart ($Depth + 1)
|
|
}
|
|
}
|
|
|
|
Visit-OrcaNode $RootElement 0
|
|
[pscustomobject]@{ elements = @($records.ToArray()); lines = @($lines.ToArray()); truncation = $truncation }
|
|
}
|
|
|
|
function Compress-OrcaRenderedBrowserTabs($Records, $Lines, [int]$StartLine, [int]$Depth) {
|
|
$tabLineIndexes = New-Object System.Collections.Generic.List[int]
|
|
for ($lineIndex = $StartLine; $lineIndex -lt $Lines.Count; $lineIndex++) {
|
|
if (Test-OrcaDirectRenderedBrowserTabLine ([string]$Lines[$lineIndex]) $Depth) {
|
|
$tabLineIndexes.Add($lineIndex)
|
|
}
|
|
}
|
|
if ($tabLineIndexes.Count -lt 10) { return }
|
|
|
|
$recordsByIndex = @{}
|
|
foreach ($record in @($Records.ToArray())) {
|
|
$recordsByIndex[[int]$record.index] = $record
|
|
}
|
|
$activeLineIndexes = New-Object System.Collections.Generic.HashSet[int]
|
|
foreach ($lineIndex in $tabLineIndexes) {
|
|
if (Test-OrcaActiveRenderedBrowserTabLine ([string]$Lines[$lineIndex]) $Depth $recordsByIndex) {
|
|
[void]$activeLineIndexes.Add($lineIndex)
|
|
}
|
|
}
|
|
if ($activeLineIndexes.Count -eq 0) { return }
|
|
|
|
$omittedRecordIndexes = New-Object System.Collections.Generic.HashSet[int]
|
|
$omittedCount = 0
|
|
$insertionIndex = $tabLineIndexes[0]
|
|
for ($i = $tabLineIndexes.Count - 1; $i -ge 0; $i--) {
|
|
$lineIndex = $tabLineIndexes[$i]
|
|
if ($activeLineIndexes.Contains($lineIndex)) { continue }
|
|
$recordIndex = Get-OrcaRenderedElementIndex ([string]$Lines[$lineIndex]) $Depth
|
|
if ($null -ne $recordIndex) {
|
|
[void]$omittedRecordIndexes.Add([int]$recordIndex)
|
|
}
|
|
$Lines.RemoveAt($lineIndex)
|
|
$omittedCount++
|
|
}
|
|
if ($omittedCount -le 0) { return }
|
|
for ($recordIndex = $Records.Count - 1; $recordIndex -ge 0; $recordIndex--) {
|
|
if ($omittedRecordIndexes.Contains([int]$Records[$recordIndex].index)) {
|
|
$Records.RemoveAt($recordIndex)
|
|
}
|
|
}
|
|
$Lines.Insert($insertionIndex, (("`t" * $Depth) + "... $omittedCount inactive browser tabs omitted"))
|
|
}
|
|
|
|
function Test-OrcaDirectRenderedBrowserTabLine([string]$Line, [int]$Depth) {
|
|
$indent = "`t" * $Depth
|
|
if (-not $Line.StartsWith($indent)) { return $false }
|
|
$text = $Line.Substring($indent.Length)
|
|
if ($text.StartsWith("`t")) { return $false }
|
|
$text -match "^\d+ (page tab|tab item|tab)($|[ \(,])"
|
|
}
|
|
|
|
function Test-OrcaActiveRenderedBrowserTabLine([string]$Line, [int]$Depth, $RecordsByIndex) {
|
|
if ($Line.Contains("(selected")) { return $true }
|
|
$recordIndex = Get-OrcaRenderedElementIndex $Line $Depth
|
|
if ($null -eq $recordIndex -or -not $RecordsByIndex.ContainsKey([int]$recordIndex)) { return $false }
|
|
$record = $RecordsByIndex[[int]$recordIndex]
|
|
[bool]$record.isSelected -or (Format-OrcaSnapshotText $record.value) -eq "1"
|
|
}
|
|
|
|
function Get-OrcaRenderedElementIndex([string]$Line, [int]$Depth) {
|
|
$text = $Line.Substring(("`t" * $Depth).Length)
|
|
if ($text -match "^(\d+)") { return [int]$Matches[1] }
|
|
$null
|
|
}
|
|
|
|
function ConvertTo-OrcaPngBytes([System.Drawing.Image]$Image) {
|
|
$stream = $null
|
|
try {
|
|
$stream = New-Object System.IO.MemoryStream
|
|
$Image.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
return ,$stream.ToArray()
|
|
} finally {
|
|
if ($null -ne $stream) { $stream.Dispose() }
|
|
}
|
|
}
|
|
|
|
function New-OrcaScreenshotPayload([byte[]]$Bytes, [int]$Width, [int]$Height, [double]$Scale) {
|
|
[pscustomobject]@{
|
|
base64 = [Convert]::ToBase64String($Bytes)
|
|
width = $Width
|
|
height = $Height
|
|
scale = $Scale
|
|
}
|
|
}
|
|
|
|
function Resize-OrcaBitmap([System.Drawing.Bitmap]$Source, [int]$Width, [int]$Height) {
|
|
$resized = $null
|
|
$graphics = $null
|
|
try {
|
|
$resized = New-Object System.Drawing.Bitmap $Width, $Height
|
|
$graphics = [System.Drawing.Graphics]::FromImage($resized)
|
|
$graphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::Bilinear
|
|
$graphics.DrawImage($Source, 0, 0, $Width, $Height)
|
|
$result = $resized
|
|
$resized = $null
|
|
return $result
|
|
} finally {
|
|
if ($null -ne $graphics) { $graphics.Dispose() }
|
|
if ($null -ne $resized) { $resized.Dispose() }
|
|
}
|
|
}
|
|
|
|
function Get-OrcaBoundedScreenshotPayload([System.Drawing.Bitmap]$Bitmap) {
|
|
$originalWidth = [int][Math]::Max(1, $Bitmap.Width)
|
|
$originalHeight = [int][Math]::Max(1, $Bitmap.Height)
|
|
$pngBytes = ConvertTo-OrcaPngBytes $Bitmap
|
|
if ($pngBytes.Length -le $MaxScreenshotPngBytes) {
|
|
return New-OrcaScreenshotPayload $pngBytes $originalWidth $originalHeight 1.0
|
|
}
|
|
|
|
# Why: screenshots cross process boundaries as PNG base64 in JSON; cap noisy
|
|
# large-window payloads to match the macOS provider's memory bounds.
|
|
$scale = [Math]::Min(1.0, $MaxScreenshotEdge / [double][Math]::Max($originalWidth, $originalHeight))
|
|
while ($scale -ge $MinScreenshotScale) {
|
|
$width = [int][Math]::Max(1, [Math]::Round($originalWidth * $scale))
|
|
$height = [int][Math]::Max(1, [Math]::Round($originalHeight * $scale))
|
|
if ($width -eq $originalWidth -and $height -eq $originalHeight) {
|
|
$scale *= $ScreenshotScaleStep
|
|
continue
|
|
}
|
|
|
|
$resized = $null
|
|
try {
|
|
$resized = Resize-OrcaBitmap $Bitmap $width $height
|
|
$candidateBytes = ConvertTo-OrcaPngBytes $resized
|
|
if ($candidateBytes.Length -le $MaxScreenshotPngBytes) {
|
|
return New-OrcaScreenshotPayload $candidateBytes $width $height ($width / [double]$originalWidth)
|
|
}
|
|
} finally {
|
|
if ($null -ne $resized) { $resized.Dispose() }
|
|
}
|
|
|
|
$scale *= $ScreenshotScaleStep
|
|
}
|
|
|
|
[pscustomobject]@{
|
|
error = [pscustomobject]@{
|
|
code = "screenshot_failed"
|
|
message = "screenshot exceeded the computer-use payload cap after downscaling; retry with --no-screenshot or target a smaller window"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame) {
|
|
if (-not $IncludeScreenshot -or $null -eq $WindowFrame) { return $null }
|
|
$bitmap = $null
|
|
$graphics = $null
|
|
try {
|
|
$width = [int][Math]::Max(1, [Math]::Round($WindowFrame.width))
|
|
$height = [int][Math]::Max(1, [Math]::Round($WindowFrame.height))
|
|
$bitmap = New-Object System.Drawing.Bitmap $width, $height
|
|
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
|
$graphics.CopyFromScreen([int][Math]::Round($WindowFrame.x), [int][Math]::Round($WindowFrame.y), 0, 0, $bitmap.Size)
|
|
Get-OrcaBoundedScreenshotPayload $bitmap
|
|
} catch {
|
|
$null
|
|
} finally {
|
|
if ($null -ne $graphics) { $graphics.Dispose() }
|
|
if ($null -ne $bitmap) { $bitmap.Dispose() }
|
|
}
|
|
}
|
|
|
|
function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) {
|
|
$process = Find-OrcaProcess $Query
|
|
if ($RestoreWindow) { Restore-OrcaWindow $process }
|
|
Assert-OrcaWindowTarget $process $WindowId $WindowIndex
|
|
$root = Get-OrcaRootElement $process
|
|
$windowFrame = Get-OrcaWindowFrame $process $root
|
|
$tree = Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process)
|
|
$screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame
|
|
|
|
[pscustomobject]@{
|
|
snapshotId = [guid]::NewGuid().ToString()
|
|
app = New-OrcaAppRecord $process
|
|
windowTitle = $process.MainWindowTitle
|
|
windowId = Get-OrcaWindowId $process
|
|
windowBounds = $windowFrame
|
|
screenshotPngBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null }
|
|
screenshotWidth = if ($null -ne $screenshot) { $screenshot.width } else { $null }
|
|
screenshotHeight = if ($null -ne $screenshot) { $screenshot.height } else { $null }
|
|
screenshotScale = if ($null -ne $screenshot) { $screenshot.scale } else { $null }
|
|
screenshotError = if ($null -ne $screenshot) { $screenshot.error } else { $null }
|
|
coordinateSpace = "window"
|
|
truncation = $tree.truncation
|
|
treeLines = @($tree.lines)
|
|
focusedSummary = $null
|
|
focusedElementId = $null
|
|
selectedText = $null
|
|
elements = @($tree.elements)
|
|
}
|
|
}
|
|
|
|
function Get-OrcaAppList {
|
|
@(Get-OrcaWindowProcesses | ForEach-Object {
|
|
New-OrcaAppRecord $_
|
|
})
|
|
}
|
|
|
|
function Get-OrcaWindowList([string]$Query) {
|
|
$process = Find-OrcaProcess $Query
|
|
$root = Get-OrcaRootElement $process
|
|
$windowFrame = Get-OrcaWindowFrame $process $root
|
|
$x = $null
|
|
$y = $null
|
|
$width = 0
|
|
$height = 0
|
|
if ($null -ne $windowFrame) {
|
|
$x = [int][Math]::Round($windowFrame.x)
|
|
$y = [int][Math]::Round($windowFrame.y)
|
|
$width = [int][Math]::Max(0, [Math]::Round($windowFrame.width))
|
|
$height = [int][Math]::Max(0, [Math]::Round($windowFrame.height))
|
|
}
|
|
$app = New-OrcaAppRecord $process
|
|
[pscustomobject]@{
|
|
app = $app
|
|
windows = @([pscustomobject]@{
|
|
index = 0
|
|
app = $app
|
|
id = Get-OrcaWindowId $process
|
|
title = $process.MainWindowTitle
|
|
x = $x
|
|
y = $y
|
|
width = $width
|
|
height = $height
|
|
isMinimized = $false
|
|
isOffscreen = $false
|
|
screenIndex = $null
|
|
platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = Get-OrcaWindowId $process }
|
|
})
|
|
}
|
|
}
|
|
|
|
function Get-OrcaHandshake {
|
|
[pscustomobject]@{
|
|
platform = "win32"
|
|
provider = "orca-computer-use-windows"
|
|
providerVersion = "1.0.0"
|
|
protocolVersion = 1
|
|
supports = [pscustomobject]@{
|
|
apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true }
|
|
windows = [pscustomobject]@{ list = $true; targetById = $true; targetByIndex = $true; focus = $false; moveResize = $false }
|
|
observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false }
|
|
actions = [pscustomobject]@{
|
|
click = $true
|
|
typeText = $true
|
|
pressKey = $true
|
|
hotkey = $true
|
|
pasteText = $true
|
|
scroll = $true
|
|
drag = $true
|
|
setValue = $true
|
|
performAction = $true
|
|
}
|
|
surfaces = [pscustomobject]@{ menus = $false; dialogs = $false; dock = $false; menubar = $false }
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-OrcaSameRuntimeId($Left, $Right) {
|
|
if ($null -eq $Left -or $null -eq $Right -or $Left.Count -ne $Right.Count) { return $false }
|
|
for ($i = 0; $i -lt $Left.Count; $i++) {
|
|
if ([int]$Left[$i] -ne [int]$Right[$i]) { return $false }
|
|
}
|
|
$true
|
|
}
|
|
|
|
function Find-OrcaElement($RootElement, $Record) {
|
|
if ($null -eq $Record) { return $null }
|
|
if ($Record.index -eq 0) { return $RootElement }
|
|
|
|
try {
|
|
$descendants = $RootElement.FindAll([Windows.Automation.TreeScope]::Descendants, [Windows.Automation.Condition]::TrueCondition)
|
|
for ($i = 0; $i -lt $descendants.Count; $i++) {
|
|
$candidate = $descendants.Item($i)
|
|
if (Test-OrcaSameRuntimeId @($candidate.GetRuntimeId()) @($Record.runtimeId)) {
|
|
return $candidate
|
|
}
|
|
}
|
|
} catch {}
|
|
$null
|
|
}
|
|
|
|
function Invoke-OrcaPrimaryAction($Element) {
|
|
foreach ($pattern in @(
|
|
[Windows.Automation.InvokePattern]::Pattern,
|
|
[Windows.Automation.SelectionItemPattern]::Pattern,
|
|
[Windows.Automation.TogglePattern]::Pattern
|
|
)) {
|
|
try {
|
|
$instance = $Element.GetCurrentPattern($pattern)
|
|
if ($pattern -eq [Windows.Automation.InvokePattern]::Pattern) { $instance.Invoke(); return $true }
|
|
if ($pattern -eq [Windows.Automation.SelectionItemPattern]::Pattern) { $instance.Select(); return $true }
|
|
if ($pattern -eq [Windows.Automation.TogglePattern]::Pattern) { $instance.Toggle(); return $true }
|
|
} catch {}
|
|
}
|
|
$false
|
|
}
|
|
|
|
function Invoke-OrcaNamedAction($Element, [string]$Action) {
|
|
$wanted = ""
|
|
if ($null -ne $Action) { $wanted = $Action.Trim().ToLowerInvariant() }
|
|
switch ($wanted) {
|
|
"invoke" {
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.InvokePattern]::Pattern)
|
|
$pattern.Invoke()
|
|
return $true
|
|
}
|
|
"select" {
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.SelectionItemPattern]::Pattern)
|
|
$pattern.Select()
|
|
return $true
|
|
}
|
|
"toggle" {
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.TogglePattern]::Pattern)
|
|
$pattern.Toggle()
|
|
return $true
|
|
}
|
|
default {
|
|
return $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Set-OrcaElementValue($Element, [string]$Value) {
|
|
try {
|
|
$pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
|
|
if (-not $pattern.Current.IsReadOnly) {
|
|
$pattern.SetValue($Value)
|
|
return $true
|
|
}
|
|
} catch {}
|
|
$false
|
|
}
|
|
|
|
function Get-OrcaRequiredNumber($Value, [string]$Name) {
|
|
if ($null -eq $Value) { throw "$Name is required" }
|
|
$number = [double]$Value
|
|
if ([double]::IsNaN($number) -or [double]::IsInfinity($number)) {
|
|
throw "$Name must be a finite number"
|
|
}
|
|
$number
|
|
}
|
|
|
|
function Get-OrcaPositiveInteger($Value, [string]$Name) {
|
|
if ($null -eq $Value) { $Value = 1 }
|
|
$number = [int]$Value
|
|
if ($number -le 0) { throw "$Name must be a positive integer" }
|
|
$number
|
|
}
|
|
|
|
function Get-OrcaPositiveNumber($Value, [string]$Name) {
|
|
if ($null -eq $Value) { $Value = 1 }
|
|
$number = Get-OrcaRequiredNumber $Value $Name
|
|
if ($number -le 0) { throw "$Name must be a positive number" }
|
|
$number
|
|
}
|
|
|
|
function Get-OrcaRequiredString($Value, [string]$Name) {
|
|
if ($null -eq $Value) { throw "$Name is required" }
|
|
$text = [string]$Value
|
|
if ($text.Length -eq 0) { throw "$Name is required" }
|
|
$text
|
|
}
|
|
|
|
function Get-OrcaScreenPoint($Operation, $WindowFrame) {
|
|
if ($null -ne $Operation.element) {
|
|
throw "stale element frame; run get-app-state again and use a fresh element index"
|
|
}
|
|
$x = Get-OrcaRequiredNumber $Operation.x "x"
|
|
$y = Get-OrcaRequiredNumber $Operation.y "y"
|
|
@{
|
|
x = [int][Math]::Round($WindowFrame.x + $x)
|
|
y = [int][Math]::Round($WindowFrame.y + $y)
|
|
}
|
|
}
|
|
|
|
function Get-OrcaElementScreenPoint($Element) {
|
|
if ($null -eq $Element) { return $null }
|
|
try {
|
|
$rect = $Element.Current.BoundingRectangle
|
|
if ($rect.Width -gt 0 -and $rect.Height -gt 0) {
|
|
return @{
|
|
x = [int][Math]::Round($rect.X + ($rect.Width / 2))
|
|
y = [int][Math]::Round($rect.Y + ($rect.Height / 2))
|
|
}
|
|
}
|
|
} catch {}
|
|
$null
|
|
}
|
|
|
|
function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) {
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
|
[void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY)
|
|
$buttonName = if ([string]::IsNullOrWhiteSpace($Button)) { "left" } else { $Button.ToLowerInvariant() }
|
|
switch ($buttonName) {
|
|
"left" { $down = $MouseEvents.LeftDown; $up = $MouseEvents.LeftUp }
|
|
"right" { $down = $MouseEvents.RightDown; $up = $MouseEvents.RightUp }
|
|
"middle" { $down = $MouseEvents.MiddleDown; $up = $MouseEvents.MiddleUp }
|
|
default { throw "unsupported mouse button: $Button" }
|
|
}
|
|
|
|
for ($i = 0; $i -lt (Get-OrcaPositiveInteger $Count "click_count"); $i++) {
|
|
[OrcaDesktopWin32]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
|
|
Start-Sleep -Milliseconds 35
|
|
[OrcaDesktopWin32]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
|
|
}
|
|
}
|
|
|
|
function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
|
$startX = [int]$From.x
|
|
$startY = [int]$From.y
|
|
$endX = [int]$To.x
|
|
$endY = [int]$To.y
|
|
[void][OrcaDesktopWin32]::SetCursorPos($startX, $startY)
|
|
[OrcaDesktopWin32]::mouse_event($MouseEvents.LeftDown, 0, 0, 0, [UIntPtr]::Zero)
|
|
for ($step = 1; $step -le 12; $step++) {
|
|
$x = [int][Math]::Round($startX + (($endX - $startX) * $step / 12))
|
|
$y = [int][Math]::Round($startY + (($endY - $startY) * $step / 12))
|
|
[void][OrcaDesktopWin32]::SetCursorPos($x, $y)
|
|
Start-Sleep -Milliseconds 20
|
|
}
|
|
[OrcaDesktopWin32]::mouse_event($MouseEvents.LeftUp, 0, 0, 0, [UIntPtr]::Zero)
|
|
}
|
|
|
|
function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) {
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
|
$hasNonAscii = $false
|
|
foreach ($character in $Text.ToCharArray()) {
|
|
if ([int][char]$character -gt 0x7F) { $hasNonAscii = $true; break }
|
|
}
|
|
if ($hasNonAscii) {
|
|
foreach ($character in $Text.ToCharArray()) {
|
|
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero)
|
|
Start-Sleep -Milliseconds 8
|
|
}
|
|
return
|
|
}
|
|
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysText $Text))
|
|
}
|
|
|
|
function Get-OrcaVirtualKey([string]$Key) {
|
|
$normalized = $Key.ToLowerInvariant()
|
|
$map = @{
|
|
"return" = 0x0D; "enter" = 0x0D; "tab" = 0x09; "escape" = 0x1B; "esc" = 0x1B
|
|
"backspace" = 0x08; "delete" = 0x2E; "space" = 0x20; "left" = 0x25
|
|
"up" = 0x26; "right" = 0x27; "down" = 0x28; "home" = 0x24; "end" = 0x23
|
|
}
|
|
if ($map.ContainsKey($normalized)) { return $map[$normalized] }
|
|
if ($normalized.Length -eq 1) { return [int][char]$normalized.ToUpperInvariant()[0] }
|
|
throw "Unsupported key: $Key"
|
|
}
|
|
|
|
function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) {
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
|
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysKey $Key))
|
|
}
|
|
|
|
function Get-OrcaModifierVirtualKey([string]$Modifier) {
|
|
switch ($Modifier.ToLowerInvariant()) {
|
|
{ $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return 0x11 }
|
|
{ $_ -in @("shift") } { return 0x10 }
|
|
{ $_ -in @("alt", "option") } { return 0x12 }
|
|
{ $_ -in @("meta", "super", "win", "cmd", "command") } { return 0x5B }
|
|
default { throw "Unsupported modifier: $Modifier" }
|
|
}
|
|
}
|
|
|
|
function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
|
|
$parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
|
|
if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" }
|
|
$key = $parts[$parts.Count - 1]
|
|
$prefix = ""
|
|
if ($parts.Count -gt 1) {
|
|
foreach ($modifier in $parts[0..($parts.Count - 2)]) {
|
|
$prefix += ConvertTo-OrcaSendKeysModifier $modifier
|
|
}
|
|
}
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
|
[System.Windows.Forms.SendKeys]::SendWait($prefix + (ConvertTo-OrcaSendKeysKey $key))
|
|
}
|
|
|
|
function ConvertTo-OrcaSendKeysText([string]$Text) {
|
|
$builder = New-Object System.Text.StringBuilder
|
|
foreach ($character in $Text.ToCharArray()) {
|
|
$value = [string]$character
|
|
if ($value -eq "`r") { continue }
|
|
if ($value -eq "`n") { [void]$builder.Append("{ENTER}"); continue }
|
|
if ("+^%~(){}[]".Contains($value)) {
|
|
[void]$builder.Append("{").Append($value).Append("}")
|
|
} else {
|
|
[void]$builder.Append($value)
|
|
}
|
|
}
|
|
$builder.ToString()
|
|
}
|
|
|
|
function ConvertTo-OrcaSendKeysKey([string]$Key) {
|
|
switch ($Key.ToLowerInvariant()) {
|
|
{ $_ -in @("return", "enter") } { return "{ENTER}" }
|
|
"tab" { return "{TAB}" }
|
|
{ $_ -in @("escape", "esc") } { return "{ESC}" }
|
|
"backspace" { return "{BACKSPACE}" }
|
|
"delete" { return "{DELETE}" }
|
|
"space" { return " " }
|
|
"left" { return "{LEFT}" }
|
|
"up" { return "{UP}" }
|
|
"right" { return "{RIGHT}" }
|
|
"down" { return "{DOWN}" }
|
|
"home" { return "{HOME}" }
|
|
"end" { return "{END}" }
|
|
{ $_ -in @("pageup", "page_up") } { return "{PGUP}" }
|
|
{ $_ -in @("pagedown", "page_down") } { return "{PGDN}" }
|
|
"insert" { return "{INSERT}" }
|
|
default {
|
|
if ($Key.Length -eq 1) { return (ConvertTo-OrcaSendKeysText $Key) }
|
|
throw "Unsupported key: $Key"
|
|
}
|
|
}
|
|
}
|
|
|
|
function ConvertTo-OrcaSendKeysModifier([string]$Modifier) {
|
|
switch ($Modifier.ToLowerInvariant()) {
|
|
{ $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return "^" }
|
|
"shift" { return "+" }
|
|
{ $_ -in @("alt", "option") } { return "%" }
|
|
default { throw "Unsupported modifier: $Modifier" }
|
|
}
|
|
}
|
|
|
|
function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) {
|
|
$previous = $null
|
|
$hadPrevious = $false
|
|
try { $previous = [System.Windows.Forms.Clipboard]::GetDataObject() } catch {}
|
|
$hadPrevious = $null -ne $previous
|
|
try {
|
|
Set-Clipboard -Value $Text
|
|
Send-OrcaHotkey $WindowHandle "Ctrl+v"
|
|
} finally {
|
|
if ($hadPrevious) {
|
|
try { [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true) } catch {}
|
|
} else {
|
|
try { [System.Windows.Forms.Clipboard]::Clear() } catch {}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-OrcaOperation($Operation) {
|
|
$includeScreenshot = -not [bool]$Operation.noScreenshot
|
|
if ($Operation.tool -eq "handshake") {
|
|
return [pscustomobject]@{ ok = $true; capabilities = Get-OrcaHandshake }
|
|
}
|
|
if ($Operation.tool -eq "list_apps") {
|
|
return [pscustomobject]@{ ok = $true; apps = @(Get-OrcaAppList) }
|
|
}
|
|
if ($Operation.tool -eq "list_windows") {
|
|
$list = Get-OrcaWindowList $Operation.app
|
|
return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) }
|
|
}
|
|
if ($Operation.tool -eq "get_app_state") {
|
|
return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) }
|
|
}
|
|
|
|
$process = Find-OrcaProcess $Operation.app
|
|
if ([bool]$Operation.restoreWindow) { Restore-OrcaWindow $process }
|
|
Assert-OrcaWindowTarget $process $Operation.windowId $Operation.windowIndex
|
|
$root = Get-OrcaRootElement $process
|
|
$windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $process $root }
|
|
$element = Find-OrcaElement $root $Operation.element
|
|
$fromElement = Find-OrcaElement $root $Operation.fromElement
|
|
$toElement = Find-OrcaElement $root $Operation.toElement
|
|
$handle = [IntPtr]$process.MainWindowHandle
|
|
if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) {
|
|
Assert-OrcaKeyboardFocus $handle $Operation
|
|
}
|
|
$action = $null
|
|
|
|
switch ($Operation.tool) {
|
|
"click" {
|
|
# Why: agents expect a click into a target app to make the next
|
|
# keyboard action safe, even when UI Automation handles the click.
|
|
Restore-OrcaWindow $process
|
|
$handledByPattern = $false
|
|
$clickCount = Get-OrcaPositiveInteger $Operation.click_count "click_count"
|
|
if ($null -ne $element -and $Operation.mouse_button -ne "right" -and $Operation.mouse_button -ne "middle" -and $clickCount -le 1) {
|
|
$handledByPattern = Invoke-OrcaPrimaryAction $element
|
|
}
|
|
if (-not $handledByPattern) {
|
|
$point = Get-OrcaElementScreenPoint $element
|
|
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
|
|
Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button $clickCount
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }
|
|
} else {
|
|
$action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
|
|
}
|
|
}
|
|
"perform_secondary_action" {
|
|
if ($null -eq $element) { throw "unknown element_index" }
|
|
if (-not (Invoke-OrcaNamedAction $element $Operation.action)) {
|
|
throw "$($Operation.action) is not a valid secondary action"
|
|
}
|
|
$action = [pscustomobject]@{ path = "accessibility"; actionName = $Operation.action; fallbackReason = $null }
|
|
}
|
|
"scroll" {
|
|
$delta = 120 * [int][Math]::Ceiling((Get-OrcaPositiveNumber $Operation.pages "pages"))
|
|
if ($Operation.direction -eq "down" -or $Operation.direction -eq "right") {
|
|
$delta = -1 * $delta
|
|
} elseif ($Operation.direction -ne "up" -and $Operation.direction -ne "left") {
|
|
throw "unsupported scroll direction: $($Operation.direction)"
|
|
}
|
|
$point = Get-OrcaElementScreenPoint $element
|
|
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
|
|
[void][OrcaDesktopWin32]::SetForegroundWindow($handle)
|
|
[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)
|
|
[OrcaDesktopWin32]::mouse_event($MouseEvents.Wheel, 0, 0, $delta, [UIntPtr]::Zero)
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
|
|
}
|
|
"drag" {
|
|
$from = Get-OrcaElementScreenPoint $fromElement
|
|
if ($null -eq $from -and $null -ne $Operation.fromElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
|
|
if ($null -eq $from) {
|
|
$from = @{
|
|
x = $windowFrame.x + (Get-OrcaRequiredNumber $Operation.from_x "from_x")
|
|
y = $windowFrame.y + (Get-OrcaRequiredNumber $Operation.from_y "from_y")
|
|
}
|
|
}
|
|
$to = Get-OrcaElementScreenPoint $toElement
|
|
if ($null -eq $to -and $null -ne $Operation.toElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
|
|
if ($null -eq $to) {
|
|
$to = @{
|
|
x = $windowFrame.x + (Get-OrcaRequiredNumber $Operation.to_x "to_x")
|
|
y = $windowFrame.y + (Get-OrcaRequiredNumber $Operation.to_y "to_y")
|
|
}
|
|
}
|
|
Send-OrcaDrag $handle $from $to
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }
|
|
}
|
|
"type_text" {
|
|
Send-OrcaText $handle (Get-OrcaRequiredString $Operation.text "text")
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
|
|
}
|
|
"press_key" {
|
|
Send-OrcaKey $handle (Get-OrcaRequiredString $Operation.key "key")
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = "pressKey"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
|
|
}
|
|
"hotkey" {
|
|
Send-OrcaHotkey $handle (Get-OrcaRequiredString $Operation.key "key")
|
|
$action = [pscustomobject]@{ path = "synthetic"; actionName = "hotkey"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
|
|
}
|
|
"paste_text" {
|
|
Send-OrcaPasteText $handle (Get-OrcaRequiredString $Operation.text "text")
|
|
$action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } }
|
|
}
|
|
"set_value" {
|
|
if ($null -eq $element -or -not (Set-OrcaElementValue $element ([string]$Operation.value))) {
|
|
throw "element value is not settable"
|
|
}
|
|
$action = [pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null }
|
|
}
|
|
default {
|
|
throw "unsupported tool: $($Operation.tool)"
|
|
}
|
|
}
|
|
|
|
try {
|
|
$snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex
|
|
} catch {
|
|
if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw }
|
|
if ($null -eq $action.verification) {
|
|
$action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" })
|
|
}
|
|
$snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $null $null
|
|
}
|
|
[pscustomobject]@{ ok = $true; action = $action; snapshot = $snapshot }
|
|
}
|
|
|
|
try {
|
|
$operation = Read-OrcaOperation $OperationPath
|
|
Write-OrcaJson (Invoke-OrcaOperation $operation)
|
|
} catch {
|
|
Write-OrcaJson ([pscustomobject]@{ ok = $false; error = [string]$_.Exception.Message })
|
|
}
|