chore: import upstream snapshot with attribution

2026-07-13 12:02:32 +08:00
commit 4cd2d4af2b
475 changed files with 121829 additions and 0 deletions
@@ -0,0 +1,299 @@
+"""
+Debug test for iframe scrolling issue where DOM tree only shows top elements after scrolling.
+
+This test verifies that after scrolling inside an iframe, the selector_map correctly
+contains lower input elements like City, State, Zip Code, etc.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import browser_use modules
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import ActionModel
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.events import BrowserStateRequestEvent
+
+# Import the mock LLM helper from conftest
+from tests.ci.conftest import create_mock_llm
+
+
+async def debug_iframe_scrolling():
+	"""Debug iframe scrolling and DOM visibility issue."""
+
+	print('Starting iframe scrolling debug test...')
+
+	# Create the sequence of actions for the mock LLM
+	# We need to format these as the LLM would return them
+	actions = [
+		# First action: Navigate to the test URL
+		"""
+		{
+			"thinking": "Navigating to the iframe test page",
+			"evaluation_previous_goal": null,
+			"memory": "Starting test",
+			"next_goal": "Navigate to the iframe test page",
+			"action": [
+				{
+					"navigate": {
+						"url": "https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html",
+						"new_tab": false
+					}
+				}
+			]
+		}
+		""",
+		# Second action: Input text in the first name field (to verify we can interact)
+		"""
+		{
+			"thinking": "Inputting text in the first name field to test interaction",
+			"evaluation_previous_goal": "Successfully navigated to the page",
+			"memory": "Page loaded with nested iframes",
+			"next_goal": "Type text in the first name field",
+			"action": [
+				{
+					"input_text": {
+						"index": 1,
+						"text": "TestName"
+					}
+				}
+			]
+		}
+		""",
+		# Third action: Scroll the iframe (element_index=2 should be the iframe)
+		"""
+		{
+			"thinking": "Scrolling inside the iframe to reveal lower form elements",
+			"evaluation_previous_goal": "Successfully typed in first name field",
+			"memory": "Typed TestName in first field",
+			"next_goal": "Scroll inside the innermost iframe to see more form fields",
+			"action": [
+				{
+					"scroll": {
+						"down": true,
+						"num_pages": 1.0,
+						"index": 2
+					}
+				}
+			]
+		}
+		""",
+		# Fourth action: Done
+		"""
+		{
+			"thinking": "Completed scrolling, ready to inspect DOM",
+			"evaluation_previous_goal": "Successfully scrolled inside iframe",
+			"memory": "Scrolled to reveal lower form fields",
+			"next_goal": "Task completed",
+			"action": [
+				{
+					"done": {
+						"text": "Scrolling completed",
+						"success": true
+					}
+				}
+			]
+		}
+		""",
+	]
+
+	# Create mock LLM with our action sequence
+	mock_llm = create_mock_llm(actions=actions)
+
+	# Create browser session with headless=False so we can see what's happening
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=False,  # Set to False to see the browser
+			user_data_dir=None,  # Use temporary directory
+			keep_alive=True,
+			enable_default_extensions=True,
+			cross_origin_iframes=True,  # Enable cross-origin iframe support
+		)
+	)
+
+	try:
+		# Start the browser session
+		await browser_session.start()
+		print('Browser session started')
+
+		# Create an agent with the mock LLM
+		agent = Agent(
+			task='Navigate to the iframe test page and scroll inside the iframe',
+			llm=mock_llm,
+			browser_session=browser_session,
+		)
+
+		# Helper function to capture and analyze DOM state
+		async def capture_dom_state(label: str) -> dict:
+			"""Capture DOM state and return analysis"""
+			print(f'\n📸 Capturing DOM state: {label}')
+			state_event = browser_session.event_bus.dispatch(
+				BrowserStateRequestEvent(include_dom=True, include_screenshot=False, include_recent_events=False)
+			)
+			browser_state = await state_event.event_result()
+
+			if browser_state and browser_state.dom_state and browser_state.dom_state.selector_map:
+				selector_map = browser_state.dom_state.selector_map
+				element_count = len(selector_map)
+
+				# Check for specific elements
+				found_elements = {}
+				expected_checks = [
+					('First Name', ['firstName', 'first name']),
+					('Last Name', ['lastName', 'last name']),
+					('Email', ['email']),
+					('City', ['city']),
+					('State', ['state']),
+					('Zip', ['zip', 'zipCode']),
+				]
+
+				for name, keywords in expected_checks:
+					for index, element in selector_map.items():
+						element_str = str(element).lower()
+						if any(kw.lower() in element_str for kw in keywords):
+							found_elements[name] = True
+							break
+
+				return {
+					'label': label,
+					'total_elements': element_count,
+					'found_elements': found_elements,
+					'selector_map': selector_map,
+				}
+			return {'label': label, 'error': 'No DOM state available'}
+
+		# Capture initial state before any actions
+		print('\n' + '=' * 80)
+		print('PHASE 1: INITIAL PAGE LOAD')
+		print('=' * 80)
+
+		# Navigate to the page first
+		from browser_use.tools.service import Tools
+
+		tools = Tools()
+
+		# Create the action model for navigation
+		goto_action = ActionModel.model_validate_json(actions[0])
+		await tools.act(goto_action, browser_session)
+		await asyncio.sleep(2)  # Wait for page to fully load
+
+		initial_state = await capture_dom_state('INITIAL (after page load)')
+
+		# Now run the rest of the actions via the agent
+		print('\n' + '=' * 80)
+		print('PHASE 2: EXECUTING ACTIONS')
+		print('=' * 80)
+
+		# Create new agent with remaining actions
+		remaining_actions = actions[1:]  # Skip the navigation we already did
+		mock_llm_remaining = create_mock_llm(actions=remaining_actions)
+		agent = Agent(
+			task='Input text and scroll inside the iframe',
+			llm=mock_llm_remaining,
+			browser_session=browser_session,
+		)
+
+		# Hook into agent actions to capture state after each one
+		states = []
+		original_act = tools.act
+
+		async def wrapped_act(action, session):
+			result = await original_act(action, session)
+			# Capture state after each action
+			action_type = 'unknown'
+			if hasattr(action, 'input_text') and action.input_text:
+				action_type = 'input_text'
+				await asyncio.sleep(1)  # Give time for DOM to update
+				state = await capture_dom_state('AFTER INPUT_TEXT')
+				states.append(state)
+			elif hasattr(action, 'scroll') and action.scroll:
+				action_type = 'scroll'
+				await asyncio.sleep(2)  # Give more time after scroll
+				state = await capture_dom_state('AFTER SCROLL')
+				states.append(state)
+			return result
+
+		tools.act = wrapped_act
+
+		# Run the agent with remaining actions
+		result = await agent.run()
+		print(f'\nAgent completed with result: {result}')
+
+		# Analyze all captured states
+		print('\n' + '=' * 80)
+		print('PHASE 3: ANALYSIS OF DOM STATES')
+		print('=' * 80)
+
+		all_states = [initial_state] + states
+
+		for state in all_states:
+			if 'error' in state:
+				print(f'\n❌ {state["label"]}: {state["error"]}')
+			else:
+				print(f'\n📊 {state["label"]}:')
+				print(f'  Total elements: {state["total_elements"]}')
+				print('  Found elements:')
+				for elem_name, found in state['found_elements'].items():
+					status = '✓' if found else '✗'
+					print(f'    {status} {elem_name}')
+
+		# Compare states
+		print('\n' + '=' * 80)
+		print('COMPARISON SUMMARY')
+		print('=' * 80)
+
+		if len(all_states) >= 3:
+			initial = all_states[0]
+			after_input = all_states[1] if len(all_states) > 1 else None
+			after_scroll = all_states[2] if len(all_states) > 2 else None
+
+			print('\nElement count changes:')
+			print(f'  Initial: {initial.get("total_elements", 0)} elements')
+			if after_input:
+				print(f'  After input_text: {after_input.get("total_elements", 0)} elements')
+			if after_scroll:
+				print(f'  After scroll: {after_scroll.get("total_elements", 0)} elements')
+
+			# Check if lower form fields appear after scroll
+			if after_scroll and 'found_elements' in after_scroll:
+				lower_fields = ['City', 'State', 'Zip']
+				missing_fields = [f for f in lower_fields if not after_scroll['found_elements'].get(f, False)]
+
+				if missing_fields:
+					print('\n⚠️  BUG CONFIRMED: Lower form fields missing after scroll:')
+					for field in missing_fields:
+						print(f'    ✗ {field}')
+					print('\nThis confirms that scrolling inside iframes does not update the DOM tree properly.')
+				else:
+					print('\n✅ SUCCESS: All lower form fields are visible after scrolling!')
+
+		# Show first few elements from final state for debugging
+		if states and 'selector_map' in states[-1]:
+			print('\n' + '=' * 80)
+			print('DEBUG: First 5 elements in final selector_map')
+			print('=' * 80)
+			final_map = states[-1]['selector_map']
+			for i, (index, element) in enumerate(list(final_map.items())[:5]):
+				elem_preview = str(element)[:150]
+				print(f'\n  [{index}]: {elem_preview}...')
+
+		# Keep browser open for manual inspection if needed
+		print('\n' + '=' * 80)
+		print('Test complete. Browser will remain open for 10 seconds for inspection...')
+		print('=' * 80)
+		await asyncio.sleep(10)
+
+	finally:
+		# Clean up
+		print('\nCleaning up...')
+		await browser_session.kill()
+		await browser_session.event_bus.stop(clear=True, timeout=5)
+		print('Browser session closed')
+
+
+if __name__ == '__main__':
+	# Run the debug test
+	asyncio.run(debug_iframe_scrolling())
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""Test frame hierarchy for any URL passed as argument."""
+
+import asyncio
+import sys
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import BrowserStartEvent
+from browser_use.browser.profile import BrowserProfile
+
+
+async def analyze_frame_hierarchy(url):
+	"""Analyze and display complete frame hierarchy for a URL."""
+
+	profile = BrowserProfile(headless=True, user_data_dir=None)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		print('🚀 Starting browser...')
+		await session.on_BrowserStartEvent(BrowserStartEvent())
+
+		print(f'📍 Navigating to: {url}')
+		await session._cdp_navigate(url)
+		await asyncio.sleep(3)
+
+		print('\n' + '=' * 80)
+		print('FRAME HIERARCHY ANALYSIS')
+		print('=' * 80)
+
+		# Get all targets from SessionManager
+		all_targets = session.session_manager.get_all_targets()
+
+		# Separate by type
+		page_targets = [target for target in all_targets.values() if target.target_type == 'page']
+		iframe_targets = [target for target in all_targets.values() if target.target_type == 'iframe']
+
+		print('\n📊 Target Summary:')
+		print(f'  Total targets: {len(all_targets)}')
+		print(f'  Page targets: {len(page_targets)}')
+		print(f'  Iframe targets (OOPIFs): {len(iframe_targets)}')
+
+		# Show all targets
+		print('\n📋 All Targets:')
+		for i, (target_id, target) in enumerate(all_targets.items()):
+			if target.target_type in ['page', 'iframe']:
+				print(f'\n  [{i + 1}] Type: {target.target_type}')
+				print(f'      URL: {target.url}')
+				print(f'      Target ID: {target.target_id[:30]}...')
+
+				# Check if target has active sessions using the public API
+				try:
+					cdp_session = await session.get_or_create_cdp_session(target.target_id, focus=False)
+					has_session = cdp_session is not None
+				except Exception:
+					has_session = False
+				print(f'      Has Session: {has_session}')
+
+		# Get main page frame tree
+		main_target = next((t for t in page_targets if url in t.url), page_targets[0] if page_targets else None)
+
+		if main_target:
+			print('\n📐 Main Page Frame Tree:')
+			print(f'  Target: {main_target.url}')
+			print(f'  Target ID: {main_target.target_id[:30]}...')
+
+			s = await session.cdp_client.send.Target.attachToTarget(params={'targetId': main_target.target_id, 'flatten': True})
+			sid = s['sessionId']
+
+			try:
+				await session.cdp_client.send.Page.enable(session_id=sid)
+				tree = await session.cdp_client.send.Page.getFrameTree(session_id=sid)
+
+				print('\n  Frame Tree Structure:')
+
+				def print_tree(node, indent=0, parent_id=None):
+					frame = node['frame']
+					frame_id = frame.get('id', 'unknown')
+					frame_url = frame.get('url', 'none')
+
+					prefix = '  ' * indent + ('└─ ' if indent > 0 else '')
+					print(f'{prefix}Frame: {frame_url}')
+					print(f'{"  " * (indent + 1)}ID: {frame_id[:30]}...')
+
+					if parent_id:
+						print(f'{"  " * (indent + 1)}Parent: {parent_id[:30]}...')
+
+					# Check cross-origin status
+					cross_origin = frame.get('crossOriginIsolatedContextType', 'unknown')
+					if cross_origin != 'NotIsolated':
+						print(f'{"  " * (indent + 1)}⚠️  Cross-Origin: {cross_origin}')
+
+					# Process children
+					for child in node.get('childFrames', []):
+						print_tree(child, indent + 1, frame_id)
+
+				print_tree(tree['frameTree'])
+
+			finally:
+				await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sid})
+
+		# Show iframe target trees
+		if iframe_targets:
+			print('\n🔸 OOPIF Target Frame Trees:')
+
+			for iframe_target in iframe_targets:
+				print(f'\n  OOPIF Target: {iframe_target.url}')
+				print(f'  Target ID: {iframe_target.target_id[:30]}...')
+
+				s = await session.cdp_client.send.Target.attachToTarget(
+					params={'targetId': iframe_target.target_id, 'flatten': True}
+				)
+				sid = s['sessionId']
+
+				try:
+					await session.cdp_client.send.Page.enable(session_id=sid)
+					tree = await session.cdp_client.send.Page.getFrameTree(session_id=sid)
+
+					frame = tree['frameTree']['frame']
+					print(f'  Frame ID: {frame.get("id", "unknown")[:30]}...')
+					print(f'  Frame URL: {frame.get("url", "none")}')
+					print('  ⚠️  This frame runs in a separate process (OOPIF)')
+
+				except Exception as e:
+					print(f'  Error: {e}')
+				finally:
+					await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sid})
+
+		# Now show unified view from get_all_frames
+		print('\n' + '=' * 80)
+		print('UNIFIED FRAME HIERARCHY (get_all_frames method)')
+		print('=' * 80)
+
+		all_frames, target_sessions = await session.get_all_frames()
+
+		# Clean up sessions
+		for tid, sess_id in target_sessions.items():
+			try:
+				await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sess_id})
+			except Exception:
+				pass
+
+		print('\n📊 Frame Statistics:')
+		print(f'  Total frames discovered: {len(all_frames)}')
+
+		# Separate root and child frames
+		root_frames = []
+		child_frames = []
+
+		for frame_id, frame_info in all_frames.items():
+			if not frame_info.get('parentFrameId'):
+				root_frames.append((frame_id, frame_info))
+			else:
+				child_frames.append((frame_id, frame_info))
+
+		print(f'  Root frames: {len(root_frames)}')
+		print(f'  Child frames: {len(child_frames)}')
+
+		# Display all frames with details
+		print('\n📋 All Frames:')
+
+		for i, (frame_id, frame_info) in enumerate(all_frames.items()):
+			url = frame_info.get('url', 'none')
+			parent = frame_info.get('parentFrameId')
+			target_id = frame_info.get('frameTargetId', 'unknown')
+			is_cross = frame_info.get('isCrossOrigin', False)
+
+			print(f'\n  [{i + 1}] Frame URL: {url}')
+			print(f'      Frame ID: {frame_id[:30]}...')
+			print(f'      Parent Frame ID: {parent[:30] + "..." if parent else "None (ROOT)"}')
+			print(f'      Target ID: {target_id[:30]}...')
+			print(f'      Cross-Origin: {is_cross}')
+
+			# Highlight problems
+			if not parent and 'v0-simple-landing' in url:
+				print('      ❌ PROBLEM: Cross-origin frame incorrectly marked as root!')
+			elif not parent and url != 'about:blank' and url not in ['chrome://newtab/', 'about:blank']:
+				# Check if this should be the main frame
+				if any(url in t.url for t in page_targets):
+					print('      ✅ Correctly identified as root frame')
+
+			if is_cross:
+				print('      🔸 This is a cross-origin frame (OOPIF)')
+
+		# Show parent-child relationships
+		print('\n🌳 Frame Relationships:')
+
+		# Build a tree structure
+		def print_frame_tree(frame_id, frame_info, indent=0, visited=None):
+			if visited is None:
+				visited = set()
+
+			if frame_id in visited:
+				return
+			visited.add(frame_id)
+
+			url = frame_info.get('url', 'none')
+			prefix = '  ' * indent + ('└─ ' if indent > 0 else '')
+
+			print(f'{prefix}{url[:60]}...')
+			print(f'{"  " * (indent + 1)}[{frame_id[:20]}...]')
+
+			# Find children
+			for child_id, child_info in all_frames.items():
+				if child_info.get('parentFrameId') == frame_id:
+					print_frame_tree(child_id, child_info, indent + 1, visited)
+
+		# Print trees starting from roots
+		for frame_id, frame_info in root_frames:
+			print('\n  Tree starting from root:')
+			print_frame_tree(frame_id, frame_info)
+
+		print('\n' + '=' * 80)
+		print('✅ Analysis complete!')
+		print('=' * 80)
+
+	except Exception as e:
+		print(f'❌ Error: {e}')
+		import traceback
+
+		traceback.print_exc()
+	finally:
+		# Stop the CDP client first before killing the browser
+		print('\n🛑 Shutting down...')
+
+		# Close CDP connection first while browser is still alive
+		if session._cdp_client_root:
+			try:
+				await session._cdp_client_root.stop()
+			except Exception:
+				pass  # Ignore errors if already disconnected
+
+		# Then stop the browser process
+		from browser_use.browser.events import BrowserStopEvent
+
+		stop_event = session.event_bus.dispatch(BrowserStopEvent())
+		try:
+			await asyncio.wait_for(stop_event, timeout=2.0)
+		except TimeoutError:
+			print('⚠️ Browser stop timed out')
+
+
+def main():
+	if len(sys.argv) != 2:
+		print('Usage: python test_frame_hierarchy.py <URL>')
+		print('\nExample URLs to test:')
+		print('  https://v0-website-with-clickable-elements.vercel.app/nested-iframe')
+		print('  https://v0-website-with-clickable-elements.vercel.app/cross-origin')
+		print('  https://v0-website-with-clickable-elements.vercel.app/shadow-dom')
+		sys.exit(1)
+
+	url = sys.argv[1]
+	asyncio.run(analyze_frame_hierarchy(url))
+
+	# Ensure clean exit
+	print('✅ Script completed')
+	sys.exit(0)
+
+
+if __name__ == '__main__':
+	main()