wehub-resource-sync
601 lines
23 KiB
Python
601 lines
23 KiB
Python
import ast
|
|
import re
|
|
from collections import defaultdict
|
|
from io import BytesIO
|
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
|
|
import pytesseract
|
|
from PIL import Image
|
|
from pytesseract import Output
|
|
|
|
from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY
|
|
from gui_agents.s2.core.mllm import LMMAgent
|
|
from gui_agents.s2.utils.common_utils import (
|
|
call_llm_safe,
|
|
parse_single_code_from_string,
|
|
)
|
|
|
|
|
|
class ACI:
|
|
def __init__(self):
|
|
self.notes: List[str] = []
|
|
|
|
|
|
# Agent action decorator
|
|
def agent_action(func):
|
|
func.is_agent_action = True
|
|
return func
|
|
|
|
|
|
UBUNTU_APP_SETUP = f"""import subprocess;
|
|
import difflib;
|
|
import pyautogui;
|
|
pyautogui.press('escape');
|
|
time.sleep(0.5);
|
|
output = subprocess.check_output(['wmctrl', '-lx']);
|
|
output = output.decode('utf-8').splitlines();
|
|
window_titles = [line.split(None, 4)[2] for line in output];
|
|
closest_matches = difflib.get_close_matches('APP_NAME', window_titles, n=1, cutoff=0.1);
|
|
if closest_matches:
|
|
closest_match = closest_matches[0];
|
|
for line in output:
|
|
if closest_match in line:
|
|
window_id = line.split()[0]
|
|
break;
|
|
subprocess.run(['wmctrl', '-ia', window_id])
|
|
subprocess.run(['wmctrl', '-ir', window_id, '-b', 'add,maximized_vert,maximized_horz'])
|
|
"""
|
|
|
|
|
|
SET_CELL_VALUES_CMD = """import uno
|
|
import subprocess
|
|
|
|
def identify_document_type(component):
|
|
if component.supportsService("com.sun.star.sheet.SpreadsheetDocument"):
|
|
return "Calc"
|
|
|
|
if component.supportsService("com.sun.star.text.TextDocument"):
|
|
return "Writer"
|
|
|
|
if component.supportsService("com.sun.star.sheet.PresentationDocument"):
|
|
return "Impress"
|
|
|
|
return None
|
|
|
|
def cell_ref_to_indices(cell_ref):
|
|
column_letters = ''.join(filter(str.isalpha, cell_ref))
|
|
row_number = ''.join(filter(str.isdigit, cell_ref))
|
|
|
|
col = sum((ord(char.upper()) - ord('A') + 1) * (26**idx) for idx, char in enumerate(reversed(column_letters))) - 1
|
|
row = int(row_number) - 1
|
|
return col, row
|
|
|
|
def set_cell_values(new_cell_values: dict[str, str], app_name: str = "Untitled 1", sheet_name: str = "Sheet1"):
|
|
new_cell_values_idx = {{}}
|
|
for k, v in new_cell_values.items():
|
|
try:
|
|
col, row = cell_ref_to_indices(k)
|
|
except:
|
|
col = row = None
|
|
|
|
if col is not None and row is not None:
|
|
new_cell_values_idx[(col, row)] = v
|
|
|
|
# Clean up previous TCP connections.
|
|
subprocess.run(
|
|
'echo \"password\" | sudo -S ss --kill --tcp state TIME-WAIT sport = :2002',
|
|
shell=True,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True
|
|
)
|
|
|
|
# Dynamically allow soffice to listen on port 2002.
|
|
subprocess.run(
|
|
[
|
|
"soffice",
|
|
"--accept=socket,host=localhost,port=2002;urp;StarOffice.Service"
|
|
]
|
|
)
|
|
|
|
local_context = uno.getComponentContext()
|
|
resolver = local_context.ServiceManager.createInstanceWithContext(
|
|
"com.sun.star.bridge.UnoUrlResolver", local_context
|
|
)
|
|
context = resolver.resolve(
|
|
f"uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext"
|
|
)
|
|
desktop = context.ServiceManager.createInstanceWithContext(
|
|
"com.sun.star.frame.Desktop", context
|
|
)
|
|
|
|
# Collect all LibreOffice-related opened windows.
|
|
documents = []
|
|
for i, component in enumerate(desktop.Components):
|
|
title = component.Title
|
|
doc_type = identify_document_type(component)
|
|
documents.append((i, component, title, doc_type))
|
|
|
|
# Find the LibreOffice Calc app and the sheet of interest.
|
|
spreadsheet = [doc for doc in documents if doc[3] == "Calc"]
|
|
selected_spreadsheet = [doc for doc in spreadsheet if doc[2] == app_name]
|
|
if spreadsheet:
|
|
try:
|
|
if selected_spreadsheet:
|
|
spreadsheet = selected_spreadsheet[0][1]
|
|
else:
|
|
spreadsheet = spreadsheet[0][1]
|
|
|
|
sheet = spreadsheet.Sheets.getByName(sheet_name)
|
|
except:
|
|
raise ValueError(f"Could not find sheet {{sheet_name}} in {{app_name}}.")
|
|
|
|
for (col, row), value in new_cell_values_idx.items():
|
|
cell = sheet.getCellByPosition(col, row)
|
|
|
|
# Set the cell value.
|
|
if isinstance(value, (int, float)):
|
|
cell.Value = value
|
|
elif isinstance(value, str):
|
|
if value.startswith("="):
|
|
cell.Formula = value
|
|
else:
|
|
cell.String = value
|
|
elif isinstance(value, bool):
|
|
cell.Value = 1 if value else 0
|
|
elif value is None:
|
|
cell.clearContents(0)
|
|
else:
|
|
raise ValueError(f"Unsupported cell value type: {{type(value)}}")
|
|
|
|
else:
|
|
raise ValueError(f"Could not find LibreOffice Calc app corresponding to {{app_name}}.")
|
|
|
|
set_cell_values(new_cell_values={cell_values}, app_name="{app_name}", sheet_name="{sheet_name}")
|
|
"""
|
|
|
|
|
|
# ACI primitives are parameterized by description, and coordinate generation uses a pretrained grounding model
|
|
class OSWorldACI(ACI):
|
|
def __init__(
|
|
self,
|
|
platform: str,
|
|
engine_params_for_generation: Dict,
|
|
engine_params_for_grounding: Dict,
|
|
width: int = 1920,
|
|
height: int = 1080,
|
|
):
|
|
self.platform = (
|
|
platform # Dictates how the switch_applications agent action works.
|
|
)
|
|
|
|
# Configure scaling
|
|
self.width = width
|
|
self.height = height
|
|
|
|
# Maintain state for save_to_knowledge
|
|
self.notes = []
|
|
|
|
# Coordinates used during ACI execution
|
|
self.coords1 = None
|
|
self.coords2 = None
|
|
|
|
# Configure the visual grounding model responsible for coordinate generation
|
|
self.grounding_model = LMMAgent(engine_params_for_grounding)
|
|
self.engine_params_for_grounding = engine_params_for_grounding
|
|
|
|
# Configure text grounding agent
|
|
self.text_span_agent = LMMAgent(
|
|
engine_params=engine_params_for_generation,
|
|
system_prompt=PROCEDURAL_MEMORY.PHRASE_TO_WORD_COORDS_PROMPT,
|
|
)
|
|
|
|
# Given the state and worker's referring expression, use the grounding model to generate (x,y)
|
|
def generate_coords(self, ref_expr: str, obs: Dict) -> List[int]:
|
|
|
|
# Reset the grounding model state
|
|
self.grounding_model.reset()
|
|
|
|
# Configure the context, UI-TARS demo does not use system prompt
|
|
prompt = f"Query:{ref_expr}\nOutput only the coordinate of one point in your response.\n"
|
|
self.grounding_model.add_message(
|
|
text_content=prompt, image_content=obs["screenshot"], put_text_last=True
|
|
)
|
|
|
|
# Generate and parse coordinates
|
|
response = call_llm_safe(self.grounding_model)
|
|
print("RAW GROUNDING MODEL RESPONSE:", response)
|
|
numericals = re.findall(r"\d+", response)
|
|
assert len(numericals) >= 2
|
|
return [int(numericals[0]), int(numericals[1])]
|
|
|
|
# Calls pytesseract to generate word level bounding boxes for text grounding
|
|
def get_ocr_elements(self, b64_image_data: str) -> Tuple[str, List]:
|
|
image = Image.open(BytesIO(b64_image_data))
|
|
image_data = pytesseract.image_to_data(image, output_type=Output.DICT)
|
|
|
|
# Clean text by removing leading and trailing spaces and non-alphabetical characters, but keeping punctuation
|
|
for i, word in enumerate(image_data["text"]):
|
|
image_data["text"][i] = re.sub(
|
|
r"^[^a-zA-Z\s.,!?;:\-\+]+|[^a-zA-Z\s.,!?;:\-\+]+$", "", word
|
|
)
|
|
|
|
ocr_elements = []
|
|
ocr_table = "Text Table:\nWord id\tText\n"
|
|
# Obtain the <id, text, group number, word number> for each valid element
|
|
grouping_map = defaultdict(list)
|
|
ocr_id = 0
|
|
for i in range(len(image_data["text"])):
|
|
block_num = image_data["block_num"][i]
|
|
if image_data["text"][i]:
|
|
grouping_map[block_num].append(image_data["text"][i])
|
|
ocr_table += f"{ocr_id}\t{image_data['text'][i]}\n"
|
|
ocr_elements.append(
|
|
{
|
|
"id": ocr_id,
|
|
"text": image_data["text"][i],
|
|
"group_num": block_num,
|
|
"word_num": len(grouping_map[block_num]),
|
|
"left": image_data["left"][i],
|
|
"top": image_data["top"][i],
|
|
"width": image_data["width"][i],
|
|
"height": image_data["height"][i],
|
|
}
|
|
)
|
|
ocr_id += 1
|
|
|
|
return ocr_table, ocr_elements
|
|
|
|
# Given the state and worker's text phrase, generate the coords of the first/last word in the phrase
|
|
def generate_text_coords(
|
|
self, phrase: str, obs: Dict, alignment: str = ""
|
|
) -> List[int]:
|
|
|
|
ocr_table, ocr_elements = self.get_ocr_elements(obs["screenshot"])
|
|
|
|
alignment_prompt = ""
|
|
if alignment == "start":
|
|
alignment_prompt = "**Important**: Output the word id of the FIRST word in the provided phrase.\n"
|
|
elif alignment == "end":
|
|
alignment_prompt = "**Important**: Output the word id of the LAST word in the provided phrase.\n"
|
|
|
|
# Load LLM prompt
|
|
self.text_span_agent.reset()
|
|
self.text_span_agent.add_message(
|
|
alignment_prompt + "Phrase: " + phrase + "\n" + ocr_table, role="user"
|
|
)
|
|
self.text_span_agent.add_message(
|
|
"Screenshot:\n", image_content=obs["screenshot"], role="user"
|
|
)
|
|
|
|
# Obtain the target element
|
|
response = call_llm_safe(self.text_span_agent)
|
|
print("TEXT SPAN AGENT RESPONSE:", response)
|
|
numericals = re.findall(r"\d+", response)
|
|
if len(numericals) > 0:
|
|
text_id = int(numericals[-1])
|
|
else:
|
|
text_id = 0
|
|
elem = ocr_elements[text_id]
|
|
|
|
# Compute the element coordinates
|
|
if alignment == "start":
|
|
coords = [elem["left"], elem["top"] + (elem["height"] // 2)]
|
|
elif alignment == "end":
|
|
coords = [elem["left"] + elem["width"], elem["top"] + (elem["height"] // 2)]
|
|
else:
|
|
coords = [
|
|
elem["left"] + (elem["width"] // 2),
|
|
elem["top"] + (elem["height"] // 2),
|
|
]
|
|
return coords
|
|
|
|
# Takes a description based action and assigns the coordinates for any coordinate based action
|
|
# Raises an error if function can't be parsed
|
|
def assign_coordinates(self, plan: str, obs: Dict):
|
|
|
|
# Reset coords from previous action generation
|
|
self.coords1, self.coords2 = None, None
|
|
|
|
try:
|
|
# Extract the function name and args
|
|
action = parse_single_code_from_string(plan.split("Grounded Action")[-1])
|
|
function_name = re.match(r"(\w+\.\w+)\(", action).group(1)
|
|
args = self.parse_function_args(action)
|
|
except Exception as e:
|
|
raise RuntimeError(f"Error in parsing grounded action: {e}") from e
|
|
|
|
# arg0 is a description
|
|
if (
|
|
function_name in ["agent.click", "agent.type", "agent.scroll"]
|
|
and len(args) >= 1
|
|
and args[0] != None
|
|
):
|
|
self.coords1 = self.generate_coords(args[0], obs)
|
|
# arg0 and arg1 are descriptions
|
|
elif function_name == "agent.drag_and_drop" and len(args) >= 2:
|
|
self.coords1 = self.generate_coords(args[0], obs)
|
|
self.coords2 = self.generate_coords(args[1], obs)
|
|
# arg0 and arg1 are text phrases
|
|
elif function_name == "agent.highlight_text_span" and len(args) >= 2:
|
|
self.coords1 = self.generate_text_coords(args[0], obs, alignment="start")
|
|
self.coords2 = self.generate_text_coords(args[1], obs, alignment="end")
|
|
|
|
# Resize from grounding model dim into OSWorld dim (1920 * 1080)
|
|
def resize_coordinates(self, coordinates: List[int]) -> List[int]:
|
|
# User explicitly passes the grounding model dimensions
|
|
if {"grounding_width", "grounding_height"}.issubset(
|
|
self.engine_params_for_grounding
|
|
):
|
|
grounding_width = self.engine_params_for_grounding["grounding_width"]
|
|
grounding_height = self.engine_params_for_grounding["grounding_height"]
|
|
# Default to (1000, 1000), which is UI-TARS resizing
|
|
else:
|
|
grounding_width = 1000
|
|
grounding_height = 1000
|
|
|
|
return [
|
|
round(coordinates[0] * self.width / grounding_width),
|
|
round(coordinates[1] * self.height / grounding_height),
|
|
]
|
|
|
|
# Given a generated ACI function, returns a list of argument values, where descriptions are at the front of the list
|
|
def parse_function_args(self, function: str) -> List[str]:
|
|
tree = ast.parse(function)
|
|
call_node = tree.body[0].value
|
|
|
|
def safe_eval(node):
|
|
if isinstance(
|
|
node, ast.Constant
|
|
): # Handles literals like numbers, strings, etc.
|
|
return node.value
|
|
else:
|
|
return ast.unparse(node) # Return as a string if not a literal
|
|
|
|
positional_args = [safe_eval(arg) for arg in call_node.args]
|
|
keyword_args = {kw.arg: safe_eval(kw.value) for kw in call_node.keywords}
|
|
|
|
res = []
|
|
|
|
for key, val in keyword_args.items():
|
|
if "description" in key:
|
|
res.append(val)
|
|
|
|
for arg in positional_args:
|
|
res.append(arg)
|
|
|
|
return res
|
|
|
|
@agent_action
|
|
def click(
|
|
self,
|
|
element_description: str,
|
|
num_clicks: int = 1,
|
|
button_type: str = "left",
|
|
hold_keys: List = [],
|
|
):
|
|
"""Click on the element
|
|
Args:
|
|
element_description:str, a detailed descriptions of which element to click on. This description should be at least a full sentence.
|
|
num_clicks:int, number of times to click the element
|
|
button_type:str, which mouse button to press can be "left", "middle", or "right"
|
|
hold_keys:List, list of keys to hold while clicking
|
|
"""
|
|
x, y = self.resize_coordinates(self.coords1)
|
|
command = "import pyautogui; "
|
|
|
|
# TODO: specified duration?
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyDown({repr(k)}); "
|
|
command += f"""import pyautogui; pyautogui.click({x}, {y}, clicks={num_clicks}, button={repr(button_type)}); """
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyUp({repr(k)}); "
|
|
# Return pyautoguicode to click on the element
|
|
return command
|
|
|
|
@agent_action
|
|
def switch_applications(self, app_code):
|
|
"""Switch to a different application that is already open
|
|
Args:
|
|
app_code:str the code name of the application to switch to from the provided list of open applications
|
|
"""
|
|
if self.platform == "darwin":
|
|
return f"import pyautogui; import time; pyautogui.hotkey('command', 'space', interval=0.5); pyautogui.typewrite({repr(app_code)}); pyautogui.press('enter'); time.sleep(1.0)"
|
|
elif self.platform == "linux":
|
|
return UBUNTU_APP_SETUP.replace("APP_NAME", app_code)
|
|
elif self.platform == "windows":
|
|
return f"import pyautogui; import time; pyautogui.hotkey('win', 'd', interval=0.5); pyautogui.typewrite({repr(app_code)}); pyautogui.press('enter'); time.sleep(1.0)"
|
|
|
|
@agent_action
|
|
def open(self, app_or_filename: str):
|
|
"""Open any application or file with name app_or_filename. Use this action to open applications or files on the desktop, do not open manually.
|
|
Args:
|
|
app_or_filename:str, the name of the application or filename to open
|
|
"""
|
|
return f"import pyautogui; pyautogui.hotkey('win'); time.sleep(0.5); pyautogui.write({repr(app_or_filename)}); time.sleep(1.0); pyautogui.hotkey('enter'); time.sleep(0.5)"
|
|
|
|
@agent_action
|
|
def type(
|
|
self,
|
|
element_description: Optional[str] = None,
|
|
text: str = "",
|
|
overwrite: bool = False,
|
|
enter: bool = False,
|
|
):
|
|
"""Type text into a specific element
|
|
Args:
|
|
element_description:str, a detailed description of which element to enter text in. This description should be at least a full sentence.
|
|
text:str, the text to type
|
|
overwrite:bool, Assign it to True if the text should overwrite the existing text, otherwise assign it to False. Using this argument clears all text in an element.
|
|
enter:bool, Assign it to True if the enter key should be pressed after typing the text, otherwise assign it to False.
|
|
"""
|
|
|
|
if self.coords1 is not None:
|
|
# If a node is found, retrieve its coordinates and size
|
|
# Start typing at the center of the element
|
|
|
|
x, y = self.resize_coordinates(self.coords1)
|
|
|
|
command = "import pyautogui; "
|
|
command += f"pyautogui.click({x}, {y}); "
|
|
|
|
if overwrite:
|
|
command += (
|
|
f"pyautogui.hotkey('ctrl', 'a'); pyautogui.press('backspace'); "
|
|
)
|
|
|
|
command += f"pyautogui.write({repr(text)}); "
|
|
|
|
if enter:
|
|
command += "pyautogui.press('enter'); "
|
|
else:
|
|
# If no element is found, start typing at the current cursor location
|
|
command = "import pyautogui; "
|
|
|
|
if overwrite:
|
|
command += (
|
|
f"pyautogui.hotkey('ctrl', 'a'); pyautogui.press('backspace'); "
|
|
)
|
|
|
|
command += f"pyautogui.write({repr(text)}); "
|
|
|
|
if enter:
|
|
command += "pyautogui.press('enter'); "
|
|
|
|
return command
|
|
|
|
@agent_action
|
|
def save_to_knowledge(self, text: List[str]):
|
|
"""Save facts, elements, texts, etc. to a long-term knowledge bank for reuse during this task. Can be used for copy-pasting text, saving elements, etc.
|
|
Args:
|
|
text:List[str] the text to save to the knowledge
|
|
"""
|
|
self.notes.extend(text)
|
|
return """WAIT"""
|
|
|
|
@agent_action
|
|
def drag_and_drop(
|
|
self, starting_description: str, ending_description: str, hold_keys: List = []
|
|
):
|
|
"""Drag from the starting description to the ending description
|
|
Args:
|
|
starting_description:str, a very detailed description of where to start the drag action. This description should be at least a full sentence.
|
|
ending_description:str, a very detailed description of where to end the drag action. This description should be at least a full sentence.
|
|
hold_keys:List list of keys to hold while dragging
|
|
"""
|
|
x1, y1 = self.resize_coordinates(self.coords1)
|
|
x2, y2 = self.resize_coordinates(self.coords2)
|
|
|
|
command = "import pyautogui; "
|
|
|
|
command += f"pyautogui.moveTo({x1}, {y1}); "
|
|
# TODO: specified duration?
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyDown({repr(k)}); "
|
|
command += f"pyautogui.dragTo({x2}, {y2}, duration=1.); pyautogui.mouseUp(); "
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyUp({repr(k)}); "
|
|
|
|
# Return pyautoguicode to drag and drop the elements
|
|
|
|
return command
|
|
|
|
@agent_action
|
|
def highlight_text_span(self, starting_phrase: str, ending_phrase: str):
|
|
"""Highlight a text span between a provided starting phrase and ending phrase. Use this to highlight words, lines, and paragraphs.
|
|
Args:
|
|
starting_phrase:str, the phrase that denotes the start of the text span you want to highlight. If you only want to highlight one word, just pass in that single word.
|
|
ending_phrase:str, the phrase that denotes the end of the text span you want to highlight. If you only want to highlight one word, just pass in that single word.
|
|
"""
|
|
|
|
x1, y1 = self.coords1
|
|
x2, y2 = self.coords2
|
|
|
|
command = "import pyautogui; "
|
|
command += f"pyautogui.moveTo({x1}, {y1}); "
|
|
command += f"pyautogui.dragTo({x2}, {y2}, duration=1.); pyautogui.mouseUp(); "
|
|
|
|
# Return pyautoguicode to drag and drop the elements
|
|
return command
|
|
|
|
@agent_action
|
|
def set_cell_values(
|
|
self, cell_values: Dict[str, Any], app_name: str, sheet_name: str
|
|
):
|
|
"""Use this to set individual cell values in a spreadsheet. For example, setting A2 to "hello" would be done by passing {"A2": "hello"} as cell_values. The sheet must be opened before this command can be used.
|
|
Args:
|
|
cell_values: Dict[str, Any], A dictionary of cell values to set in the spreadsheet. The keys are the cell coordinates in the format "A1", "B2", etc.
|
|
Supported value types include: float, int, string, bool, formulas.
|
|
app_name: str, The name of the spreadsheet application. For example, "Some_sheet.xlsx".
|
|
sheet_name: str, The name of the sheet in the spreadsheet. For example, "Sheet1".
|
|
"""
|
|
return SET_CELL_VALUES_CMD.format(
|
|
cell_values=cell_values, app_name=app_name, sheet_name=sheet_name
|
|
)
|
|
|
|
@agent_action
|
|
def scroll(self, element_description: str, clicks: int, shift: bool = False):
|
|
"""Scroll the element in the specified direction
|
|
Args:
|
|
element_description:str, a very detailed description of which element to enter scroll in. This description should be at least a full sentence.
|
|
clicks:int, the number of clicks to scroll can be positive (up) or negative (down).
|
|
shift:bool, whether to use shift+scroll for horizontal scrolling
|
|
"""
|
|
|
|
x, y = self.resize_coordinates(self.coords1)
|
|
|
|
if shift:
|
|
return f"import pyautogui; import time; pyautogui.moveTo({x}, {y}); time.sleep(0.5); pyautogui.hscroll({clicks})"
|
|
else:
|
|
return f"import pyautogui; import time; pyautogui.moveTo({x}, {y}); time.sleep(0.5); pyautogui.vscroll({clicks})"
|
|
|
|
@agent_action
|
|
def hotkey(self, keys: List):
|
|
"""Press a hotkey combination
|
|
Args:
|
|
keys:List the keys to press in combination in a list format (e.g. ['ctrl', 'c'])
|
|
"""
|
|
# add quotes around the keys
|
|
keys = [f"'{key}'" for key in keys]
|
|
return f"import pyautogui; pyautogui.hotkey({', '.join(keys)})"
|
|
|
|
@agent_action
|
|
def hold_and_press(self, hold_keys: List, press_keys: List):
|
|
"""Hold a list of keys and press a list of keys
|
|
Args:
|
|
hold_keys:List, list of keys to hold
|
|
press_keys:List, list of keys to press in a sequence
|
|
"""
|
|
|
|
press_keys_str = "[" + ", ".join([f"'{key}'" for key in press_keys]) + "]"
|
|
command = "import pyautogui; "
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyDown({repr(k)}); "
|
|
command += f"pyautogui.press({press_keys_str}); "
|
|
for k in hold_keys:
|
|
command += f"pyautogui.keyUp({repr(k)}); "
|
|
|
|
return command
|
|
|
|
@agent_action
|
|
def wait(self, time: float):
|
|
"""Wait for a specified amount of time
|
|
Args:
|
|
time:float the amount of time to wait in seconds
|
|
"""
|
|
return f"""import time; time.sleep({time})"""
|
|
|
|
@agent_action
|
|
def done(
|
|
self,
|
|
return_value: Optional[Union[Dict, str, List, Tuple, int, float, bool]] = None,
|
|
):
|
|
"""End the current task with a success and the required return value"""
|
|
self.returned_info = return_value
|
|
return """DONE"""
|
|
|
|
@agent_action
|
|
def fail(self):
|
|
"""End the current task with a failure, and replan the whole task."""
|
|
return """FAIL"""
|