import ast import re from collections import defaultdict from io import BytesIO from typing import Any, Dict, List, Optional, Tuple, Union import pytesseract from PIL import Image from pytesseract import Output from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY from gui_agents.s2.core.mllm import LMMAgent from gui_agents.s2.utils.common_utils import ( call_llm_safe, parse_single_code_from_string, ) class ACI: def __init__(self): self.notes: List[str] = [] # Agent action decorator def agent_action(func): func.is_agent_action = True return func UBUNTU_APP_SETUP = f"""import subprocess; import difflib; import pyautogui; pyautogui.press('escape'); time.sleep(0.5); output = subprocess.check_output(['wmctrl', '-lx']); output = output.decode('utf-8').splitlines(); window_titles = [line.split(None, 4)[2] for line in output]; closest_matches = difflib.get_close_matches('APP_NAME', window_titles, n=1, cutoff=0.1); if closest_matches: closest_match = closest_matches[0]; for line in output: if closest_match in line: window_id = line.split()[0] break; subprocess.run(['wmctrl', '-ia', window_id]) subprocess.run(['wmctrl', '-ir', window_id, '-b', 'add,maximized_vert,maximized_horz']) """ SET_CELL_VALUES_CMD = """import uno import subprocess def identify_document_type(component): if component.supportsService("com.sun.star.sheet.SpreadsheetDocument"): return "Calc" if component.supportsService("com.sun.star.text.TextDocument"): return "Writer" if component.supportsService("com.sun.star.sheet.PresentationDocument"): return "Impress" return None def cell_ref_to_indices(cell_ref): column_letters = ''.join(filter(str.isalpha, cell_ref)) row_number = ''.join(filter(str.isdigit, cell_ref)) col = sum((ord(char.upper()) - ord('A') + 1) * (26**idx) for idx, char in enumerate(reversed(column_letters))) - 1 row = int(row_number) - 1 return col, row def set_cell_values(new_cell_values: dict[str, str], app_name: str = "Untitled 1", sheet_name: str = "Sheet1"): new_cell_values_idx = {{}} for k, v in new_cell_values.items(): try: col, row = cell_ref_to_indices(k) except: col = row = None if col is not None and row is not None: new_cell_values_idx[(col, row)] = v # Clean up previous TCP connections. subprocess.run( 'echo \"password\" | sudo -S ss --kill --tcp state TIME-WAIT sport = :2002', shell=True, check=True, text=True, capture_output=True ) # Dynamically allow soffice to listen on port 2002. subprocess.run( [ "soffice", "--accept=socket,host=localhost,port=2002;urp;StarOffice.Service" ] ) local_context = uno.getComponentContext() resolver = local_context.ServiceManager.createInstanceWithContext( "com.sun.star.bridge.UnoUrlResolver", local_context ) context = resolver.resolve( f"uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext" ) desktop = context.ServiceManager.createInstanceWithContext( "com.sun.star.frame.Desktop", context ) # Collect all LibreOffice-related opened windows. documents = [] for i, component in enumerate(desktop.Components): title = component.Title doc_type = identify_document_type(component) documents.append((i, component, title, doc_type)) # Find the LibreOffice Calc app and the sheet of interest. spreadsheet = [doc for doc in documents if doc[3] == "Calc"] selected_spreadsheet = [doc for doc in spreadsheet if doc[2] == app_name] if spreadsheet: try: if selected_spreadsheet: spreadsheet = selected_spreadsheet[0][1] else: spreadsheet = spreadsheet[0][1] sheet = spreadsheet.Sheets.getByName(sheet_name) except: raise ValueError(f"Could not find sheet {{sheet_name}} in {{app_name}}.") for (col, row), value in new_cell_values_idx.items(): cell = sheet.getCellByPosition(col, row) # Set the cell value. if isinstance(value, (int, float)): cell.Value = value elif isinstance(value, str): if value.startswith("="): cell.Formula = value else: cell.String = value elif isinstance(value, bool): cell.Value = 1 if value else 0 elif value is None: cell.clearContents(0) else: raise ValueError(f"Unsupported cell value type: {{type(value)}}") else: raise ValueError(f"Could not find LibreOffice Calc app corresponding to {{app_name}}.") set_cell_values(new_cell_values={cell_values}, app_name="{app_name}", sheet_name="{sheet_name}") """ # ACI primitives are parameterized by description, and coordinate generation uses a pretrained grounding model class OSWorldACI(ACI): def __init__( self, platform: str, engine_params_for_generation: Dict, engine_params_for_grounding: Dict, width: int = 1920, height: int = 1080, ): self.platform = ( platform # Dictates how the switch_applications agent action works. ) # Configure scaling self.width = width self.height = height # Maintain state for save_to_knowledge self.notes = [] # Coordinates used during ACI execution self.coords1 = None self.coords2 = None # Configure the visual grounding model responsible for coordinate generation self.grounding_model = LMMAgent(engine_params_for_grounding) self.engine_params_for_grounding = engine_params_for_grounding # Configure text grounding agent self.text_span_agent = LMMAgent( engine_params=engine_params_for_generation, system_prompt=PROCEDURAL_MEMORY.PHRASE_TO_WORD_COORDS_PROMPT, ) # Given the state and worker's referring expression, use the grounding model to generate (x,y) def generate_coords(self, ref_expr: str, obs: Dict) -> List[int]: # Reset the grounding model state self.grounding_model.reset() # Configure the context, UI-TARS demo does not use system prompt prompt = f"Query:{ref_expr}\nOutput only the coordinate of one point in your response.\n" self.grounding_model.add_message( text_content=prompt, image_content=obs["screenshot"], put_text_last=True ) # Generate and parse coordinates response = call_llm_safe(self.grounding_model) print("RAW GROUNDING MODEL RESPONSE:", response) numericals = re.findall(r"\d+", response) assert len(numericals) >= 2 return [int(numericals[0]), int(numericals[1])] # Calls pytesseract to generate word level bounding boxes for text grounding def get_ocr_elements(self, b64_image_data: str) -> Tuple[str, List]: image = Image.open(BytesIO(b64_image_data)) image_data = pytesseract.image_to_data(image, output_type=Output.DICT) # Clean text by removing leading and trailing spaces and non-alphabetical characters, but keeping punctuation for i, word in enumerate(image_data["text"]): image_data["text"][i] = re.sub( r"^[^a-zA-Z\s.,!?;:\-\+]+|[^a-zA-Z\s.,!?;:\-\+]+$", "", word ) ocr_elements = [] ocr_table = "Text Table:\nWord id\tText\n" # Obtain the for each valid element grouping_map = defaultdict(list) ocr_id = 0 for i in range(len(image_data["text"])): block_num = image_data["block_num"][i] if image_data["text"][i]: grouping_map[block_num].append(image_data["text"][i]) ocr_table += f"{ocr_id}\t{image_data['text'][i]}\n" ocr_elements.append( { "id": ocr_id, "text": image_data["text"][i], "group_num": block_num, "word_num": len(grouping_map[block_num]), "left": image_data["left"][i], "top": image_data["top"][i], "width": image_data["width"][i], "height": image_data["height"][i], } ) ocr_id += 1 return ocr_table, ocr_elements # Given the state and worker's text phrase, generate the coords of the first/last word in the phrase def generate_text_coords( self, phrase: str, obs: Dict, alignment: str = "" ) -> List[int]: ocr_table, ocr_elements = self.get_ocr_elements(obs["screenshot"]) alignment_prompt = "" if alignment == "start": alignment_prompt = "**Important**: Output the word id of the FIRST word in the provided phrase.\n" elif alignment == "end": alignment_prompt = "**Important**: Output the word id of the LAST word in the provided phrase.\n" # Load LLM prompt self.text_span_agent.reset() self.text_span_agent.add_message( alignment_prompt + "Phrase: " + phrase + "\n" + ocr_table, role="user" ) self.text_span_agent.add_message( "Screenshot:\n", image_content=obs["screenshot"], role="user" ) # Obtain the target element response = call_llm_safe(self.text_span_agent) print("TEXT SPAN AGENT RESPONSE:", response) numericals = re.findall(r"\d+", response) if len(numericals) > 0: text_id = int(numericals[-1]) else: text_id = 0 elem = ocr_elements[text_id] # Compute the element coordinates if alignment == "start": coords = [elem["left"], elem["top"] + (elem["height"] // 2)] elif alignment == "end": coords = [elem["left"] + elem["width"], elem["top"] + (elem["height"] // 2)] else: coords = [ elem["left"] + (elem["width"] // 2), elem["top"] + (elem["height"] // 2), ] return coords # Takes a description based action and assigns the coordinates for any coordinate based action # Raises an error if function can't be parsed def assign_coordinates(self, plan: str, obs: Dict): # Reset coords from previous action generation self.coords1, self.coords2 = None, None try: # Extract the function name and args action = parse_single_code_from_string(plan.split("Grounded Action")[-1]) function_name = re.match(r"(\w+\.\w+)\(", action).group(1) args = self.parse_function_args(action) except Exception as e: raise RuntimeError(f"Error in parsing grounded action: {e}") from e # arg0 is a description if ( function_name in ["agent.click", "agent.type", "agent.scroll"] and len(args) >= 1 and args[0] != None ): self.coords1 = self.generate_coords(args[0], obs) # arg0 and arg1 are descriptions elif function_name == "agent.drag_and_drop" and len(args) >= 2: self.coords1 = self.generate_coords(args[0], obs) self.coords2 = self.generate_coords(args[1], obs) # arg0 and arg1 are text phrases elif function_name == "agent.highlight_text_span" and len(args) >= 2: self.coords1 = self.generate_text_coords(args[0], obs, alignment="start") self.coords2 = self.generate_text_coords(args[1], obs, alignment="end") # Resize from grounding model dim into OSWorld dim (1920 * 1080) def resize_coordinates(self, coordinates: List[int]) -> List[int]: # User explicitly passes the grounding model dimensions if {"grounding_width", "grounding_height"}.issubset( self.engine_params_for_grounding ): grounding_width = self.engine_params_for_grounding["grounding_width"] grounding_height = self.engine_params_for_grounding["grounding_height"] # Default to (1000, 1000), which is UI-TARS resizing else: grounding_width = 1000 grounding_height = 1000 return [ round(coordinates[0] * self.width / grounding_width), round(coordinates[1] * self.height / grounding_height), ] # Given a generated ACI function, returns a list of argument values, where descriptions are at the front of the list def parse_function_args(self, function: str) -> List[str]: tree = ast.parse(function) call_node = tree.body[0].value def safe_eval(node): if isinstance( node, ast.Constant ): # Handles literals like numbers, strings, etc. return node.value else: return ast.unparse(node) # Return as a string if not a literal positional_args = [safe_eval(arg) for arg in call_node.args] keyword_args = {kw.arg: safe_eval(kw.value) for kw in call_node.keywords} res = [] for key, val in keyword_args.items(): if "description" in key: res.append(val) for arg in positional_args: res.append(arg) return res @agent_action def click( self, element_description: str, num_clicks: int = 1, button_type: str = "left", hold_keys: List = [], ): """Click on the element Args: element_description:str, a detailed descriptions of which element to click on. This description should be at least a full sentence. num_clicks:int, number of times to click the element button_type:str, which mouse button to press can be "left", "middle", or "right" hold_keys:List, list of keys to hold while clicking """ x, y = self.resize_coordinates(self.coords1) command = "import pyautogui; " # TODO: specified duration? for k in hold_keys: command += f"pyautogui.keyDown({repr(k)}); " command += f"""import pyautogui; pyautogui.click({x}, {y}, clicks={num_clicks}, button={repr(button_type)}); """ for k in hold_keys: command += f"pyautogui.keyUp({repr(k)}); " # Return pyautoguicode to click on the element return command @agent_action def switch_applications(self, app_code): """Switch to a different application that is already open Args: app_code:str the code name of the application to switch to from the provided list of open applications """ if self.platform == "darwin": return f"import pyautogui; import time; pyautogui.hotkey('command', 'space', interval=0.5); pyautogui.typewrite({repr(app_code)}); pyautogui.press('enter'); time.sleep(1.0)" elif self.platform == "linux": return UBUNTU_APP_SETUP.replace("APP_NAME", app_code) elif self.platform == "windows": return f"import pyautogui; import time; pyautogui.hotkey('win', 'd', interval=0.5); pyautogui.typewrite({repr(app_code)}); pyautogui.press('enter'); time.sleep(1.0)" @agent_action def open(self, app_or_filename: str): """Open any application or file with name app_or_filename. Use this action to open applications or files on the desktop, do not open manually. Args: app_or_filename:str, the name of the application or filename to open """ return f"import pyautogui; pyautogui.hotkey('win'); time.sleep(0.5); pyautogui.write({repr(app_or_filename)}); time.sleep(1.0); pyautogui.hotkey('enter'); time.sleep(0.5)" @agent_action def type( self, element_description: Optional[str] = None, text: str = "", overwrite: bool = False, enter: bool = False, ): """Type text into a specific element Args: element_description:str, a detailed description of which element to enter text in. This description should be at least a full sentence. text:str, the text to type overwrite:bool, Assign it to True if the text should overwrite the existing text, otherwise assign it to False. Using this argument clears all text in an element. enter:bool, Assign it to True if the enter key should be pressed after typing the text, otherwise assign it to False. """ if self.coords1 is not None: # If a node is found, retrieve its coordinates and size # Start typing at the center of the element x, y = self.resize_coordinates(self.coords1) command = "import pyautogui; " command += f"pyautogui.click({x}, {y}); " if overwrite: command += ( f"pyautogui.hotkey('ctrl', 'a'); pyautogui.press('backspace'); " ) command += f"pyautogui.write({repr(text)}); " if enter: command += "pyautogui.press('enter'); " else: # If no element is found, start typing at the current cursor location command = "import pyautogui; " if overwrite: command += ( f"pyautogui.hotkey('ctrl', 'a'); pyautogui.press('backspace'); " ) command += f"pyautogui.write({repr(text)}); " if enter: command += "pyautogui.press('enter'); " return command @agent_action def save_to_knowledge(self, text: List[str]): """Save facts, elements, texts, etc. to a long-term knowledge bank for reuse during this task. Can be used for copy-pasting text, saving elements, etc. Args: text:List[str] the text to save to the knowledge """ self.notes.extend(text) return """WAIT""" @agent_action def drag_and_drop( self, starting_description: str, ending_description: str, hold_keys: List = [] ): """Drag from the starting description to the ending description Args: starting_description:str, a very detailed description of where to start the drag action. This description should be at least a full sentence. ending_description:str, a very detailed description of where to end the drag action. This description should be at least a full sentence. hold_keys:List list of keys to hold while dragging """ x1, y1 = self.resize_coordinates(self.coords1) x2, y2 = self.resize_coordinates(self.coords2) command = "import pyautogui; " command += f"pyautogui.moveTo({x1}, {y1}); " # TODO: specified duration? for k in hold_keys: command += f"pyautogui.keyDown({repr(k)}); " command += f"pyautogui.dragTo({x2}, {y2}, duration=1.); pyautogui.mouseUp(); " for k in hold_keys: command += f"pyautogui.keyUp({repr(k)}); " # Return pyautoguicode to drag and drop the elements return command @agent_action def highlight_text_span(self, starting_phrase: str, ending_phrase: str): """Highlight a text span between a provided starting phrase and ending phrase. Use this to highlight words, lines, and paragraphs. Args: starting_phrase:str, the phrase that denotes the start of the text span you want to highlight. If you only want to highlight one word, just pass in that single word. ending_phrase:str, the phrase that denotes the end of the text span you want to highlight. If you only want to highlight one word, just pass in that single word. """ x1, y1 = self.coords1 x2, y2 = self.coords2 command = "import pyautogui; " command += f"pyautogui.moveTo({x1}, {y1}); " command += f"pyautogui.dragTo({x2}, {y2}, duration=1.); pyautogui.mouseUp(); " # Return pyautoguicode to drag and drop the elements return command @agent_action def set_cell_values( self, cell_values: Dict[str, Any], app_name: str, sheet_name: str ): """Use this to set individual cell values in a spreadsheet. For example, setting A2 to "hello" would be done by passing {"A2": "hello"} as cell_values. The sheet must be opened before this command can be used. Args: cell_values: Dict[str, Any], A dictionary of cell values to set in the spreadsheet. The keys are the cell coordinates in the format "A1", "B2", etc. Supported value types include: float, int, string, bool, formulas. app_name: str, The name of the spreadsheet application. For example, "Some_sheet.xlsx". sheet_name: str, The name of the sheet in the spreadsheet. For example, "Sheet1". """ return SET_CELL_VALUES_CMD.format( cell_values=cell_values, app_name=app_name, sheet_name=sheet_name ) @agent_action def scroll(self, element_description: str, clicks: int, shift: bool = False): """Scroll the element in the specified direction Args: element_description:str, a very detailed description of which element to enter scroll in. This description should be at least a full sentence. clicks:int, the number of clicks to scroll can be positive (up) or negative (down). shift:bool, whether to use shift+scroll for horizontal scrolling """ x, y = self.resize_coordinates(self.coords1) if shift: return f"import pyautogui; import time; pyautogui.moveTo({x}, {y}); time.sleep(0.5); pyautogui.hscroll({clicks})" else: return f"import pyautogui; import time; pyautogui.moveTo({x}, {y}); time.sleep(0.5); pyautogui.vscroll({clicks})" @agent_action def hotkey(self, keys: List): """Press a hotkey combination Args: keys:List the keys to press in combination in a list format (e.g. ['ctrl', 'c']) """ # add quotes around the keys keys = [f"'{key}'" for key in keys] return f"import pyautogui; pyautogui.hotkey({', '.join(keys)})" @agent_action def hold_and_press(self, hold_keys: List, press_keys: List): """Hold a list of keys and press a list of keys Args: hold_keys:List, list of keys to hold press_keys:List, list of keys to press in a sequence """ press_keys_str = "[" + ", ".join([f"'{key}'" for key in press_keys]) + "]" command = "import pyautogui; " for k in hold_keys: command += f"pyautogui.keyDown({repr(k)}); " command += f"pyautogui.press({press_keys_str}); " for k in hold_keys: command += f"pyautogui.keyUp({repr(k)}); " return command @agent_action def wait(self, time: float): """Wait for a specified amount of time Args: time:float the amount of time to wait in seconds """ return f"""import time; time.sleep({time})""" @agent_action def done( self, return_value: Optional[Union[Dict, str, List, Tuple, int, float, bool]] = None, ): """End the current task with a success and the required return value""" self.returned_info = return_value return """DONE""" @agent_action def fail(self): """End the current task with a failure, and replan the whole task.""" return """FAIL"""