chore: import upstream snapshot with attribution
Validate YAML Workflows / Validate YAML Configuration Files (push) Has been cancelled
Validate YAML Workflows / Validate YAML Configuration Files (push) Has been cancelled
This commit is contained in:
Executable
+66
@@ -0,0 +1,66 @@
|
||||
def execute_code(code: str, time_out: int = 60) -> str:
|
||||
"""
|
||||
Execute code and return std outputs and std error.
|
||||
|
||||
Args:
|
||||
code (str): Code to execute.
|
||||
time_out (int): time out, in second.
|
||||
|
||||
Returns:
|
||||
str: std output and std error
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
def __write_script_file(_code: str):
|
||||
_workspace = Path(os.getenv('TEMP_CODE_DIR', 'temp')).resolve()
|
||||
_workspace.mkdir(exist_ok=True)
|
||||
filename = f"{uuid.uuid4()}.py"
|
||||
code_path = _workspace / filename
|
||||
code_content = _code if _code.endswith("\n") else _code + "\n"
|
||||
code_path.write_text(code_content, encoding="utf-8")
|
||||
return code_path
|
||||
|
||||
def __default_interpreter() -> str:
|
||||
return sys.executable or "python3"
|
||||
|
||||
script_path = None
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
|
||||
try:
|
||||
script_path = __write_script_file(code)
|
||||
workspace = script_path.parent
|
||||
|
||||
cmd = [__default_interpreter(), str(script_path.resolve())]
|
||||
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(workspace),
|
||||
capture_output=True,
|
||||
timeout=time_out,
|
||||
check=False
|
||||
)
|
||||
stdout = completed.stdout.decode('utf-8', errors="replace")
|
||||
stderr = completed.stderr.decode('utf-8', errors="replace")
|
||||
except subprocess.TimeoutExpired as e:
|
||||
stdout = e.stdout.decode('utf-8', errors="replace") if e.stdout else ""
|
||||
stderr = e.stderr.decode('utf-8', errors="replace") if e.stderr else ""
|
||||
stderr += f"\nError: Execution timed out after {time_out} seconds."
|
||||
except Exception as e:
|
||||
stderr = f"Execution error: {str(e)}"
|
||||
|
||||
except Exception as e:
|
||||
stderr = f"Setup error: {str(e)}"
|
||||
finally:
|
||||
if script_path and script_path.exists():
|
||||
try:
|
||||
os.remove(script_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return stdout + stderr
|
||||
Executable
+645
@@ -0,0 +1,645 @@
|
||||
"""Deep research tools for search results and report management."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Dict, List, Optional, Tuple
|
||||
|
||||
from filelock import FileLock
|
||||
|
||||
from entity.messages import MessageBlock, MessageBlockType
|
||||
from functions.function_calling.file import FileToolContext
|
||||
from utils.function_catalog import ParamMeta
|
||||
|
||||
# Constants for file paths (relative to workspace root)
|
||||
SEARCH_RESULTS_FILE = "deep_research/search_results.json"
|
||||
SEARCH_LOCK_FILE = "deep_research/search_results.lock"
|
||||
REPORT_FILE = "deep_research/report.md"
|
||||
REPORT_LOCK_FILE = "deep_research/report.lock"
|
||||
|
||||
|
||||
def _get_files(ctx: FileToolContext) -> Tuple[Path, Path]:
|
||||
search_file = ctx.resolve_under_workspace(SEARCH_RESULTS_FILE)
|
||||
report_file = ctx.resolve_under_workspace(REPORT_FILE)
|
||||
return search_file, report_file
|
||||
|
||||
|
||||
def _get_locks(ctx: FileToolContext) -> Tuple[Path, Path]:
|
||||
search_lock = ctx.resolve_under_workspace(SEARCH_LOCK_FILE)
|
||||
report_lock = ctx.resolve_under_workspace(REPORT_LOCK_FILE)
|
||||
return search_lock, report_lock
|
||||
|
||||
|
||||
def _load_search_results(file_path: Path) -> Dict[str, Any]:
|
||||
if not file_path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(file_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_search_results(file_path: Path, data: Dict[str, Any]) -> None:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _format_search_result(url: str, data: Dict[str, Any], concise: bool) -> str:
|
||||
keys = data.get("highlight_keys", [])
|
||||
highlight_str = f" [IMPORTANT MATCHES: {', '.join(keys)}]" if keys else ""
|
||||
|
||||
if concise:
|
||||
return (
|
||||
f"URL: {url}{highlight_str}\n"
|
||||
f"Title: {data.get('title', '')}\n"
|
||||
f"Abstract: {data.get('abs', '')}\n"
|
||||
f"{'-' * 40}"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"URL: {url}{highlight_str}\n"
|
||||
f"Title: {data.get('title', '')}\n"
|
||||
f"Abstract: {data.get('abs', '')}\n"
|
||||
f"Detail: {data.get('detail', '')}\n"
|
||||
f"{'-' * 40}"
|
||||
)
|
||||
|
||||
|
||||
def search_save_result(
|
||||
url: Annotated[str, ParamMeta(description="URL of the search result (used as key)")],
|
||||
title: Annotated[str, ParamMeta(description="Title of the search result")],
|
||||
abs: Annotated[str, ParamMeta(description="Abstract/Summary of the content")],
|
||||
detail: Annotated[str, ParamMeta(description="Detailed content")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Save or update a search result.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
search_file, _ = _get_files(ctx)
|
||||
search_lock, _ = _get_locks(ctx)
|
||||
|
||||
with FileLock(search_lock):
|
||||
data = _load_search_results(search_file)
|
||||
current = data.get(url, {})
|
||||
|
||||
# Preserve existing keys if updating
|
||||
highlight_keys = current.get("highlight_keys", [])
|
||||
|
||||
data[url] = {
|
||||
"title": title,
|
||||
"abs": abs,
|
||||
"detail": detail,
|
||||
"highlight_keys": highlight_keys,
|
||||
}
|
||||
|
||||
_save_search_results(search_file, data)
|
||||
return f"Saved result for {url}"
|
||||
|
||||
|
||||
def search_load_all(
|
||||
# concise: Annotated[bool, ParamMeta(description="If True, only show concise information")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Load all saved search results.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
search_file, _ = _get_files(ctx)
|
||||
search_lock, _ = _get_locks(ctx)
|
||||
|
||||
with FileLock(search_lock):
|
||||
data = _load_search_results(search_file)
|
||||
|
||||
if not data:
|
||||
return "No search results found."
|
||||
|
||||
results = []
|
||||
for url, content in data.items():
|
||||
results.append(_format_search_result(url, content, concise=True))
|
||||
|
||||
return "\n\n".join(results)
|
||||
|
||||
|
||||
def search_load_by_url(
|
||||
url: Annotated[str, ParamMeta(description="URL to retrieve")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Load a specific search result by URL.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
search_file, _ = _get_files(ctx)
|
||||
search_lock, _ = _get_locks(ctx)
|
||||
|
||||
with FileLock(search_lock):
|
||||
data = _load_search_results(search_file)
|
||||
|
||||
if url not in data:
|
||||
return f"No result found for {url}"
|
||||
|
||||
return _format_search_result(url, data[url], concise=False)
|
||||
|
||||
|
||||
def search_high_light_key(
|
||||
url: Annotated[str, ParamMeta(description="URL to highlight keys for")],
|
||||
keys: Annotated[List[str], ParamMeta(description="List of keys/terms to highlight")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Save highlighted keys for a specific search result.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
search_file, _ = _get_files(ctx)
|
||||
search_lock, _ = _get_locks(ctx)
|
||||
|
||||
with FileLock(search_lock):
|
||||
data = _load_search_results(search_file)
|
||||
|
||||
if url not in data:
|
||||
return f"URL {url} not found in results. Please save it first."
|
||||
|
||||
current_keys = set(data[url].get("highlight_keys", []))
|
||||
current_keys.update(keys)
|
||||
data[url]["highlight_keys"] = list(current_keys)
|
||||
|
||||
_save_search_results(search_file, data)
|
||||
return f"Updated highlights for {url}: {list(current_keys)}"
|
||||
|
||||
|
||||
# Report Helpers
|
||||
|
||||
def _read_report_lines(file_path: Path) -> List[str]:
|
||||
if not file_path.exists():
|
||||
return []
|
||||
return file_path.read_text(encoding="utf-8").splitlines()
|
||||
|
||||
|
||||
def _save_report(file_path: Path, lines: List[str]) -> None:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Ensure final newline
|
||||
content = "\n".join(lines)
|
||||
if content and not content.endswith("\n"):
|
||||
content += "\n"
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _parse_header(line: str) -> Tuple[int, str]:
|
||||
"""Returns (level, title) if line is a header, else (0, "")."""
|
||||
match = re.match(r"^(#+)\s+(.+)$", line)
|
||||
if match:
|
||||
return len(match.group(1)), match.group(2).strip()
|
||||
return 0, ""
|
||||
|
||||
|
||||
def _find_chapter_range(lines: List[str], title_path: str) -> Tuple[int, int]:
|
||||
"""
|
||||
Find the start and end indices (inclusive, exclusive) of a chapter.
|
||||
title_path is like "Chapter 1/Section 2"
|
||||
"""
|
||||
titles = [t.strip() for t in title_path.split("/")]
|
||||
current_level_idx = 0
|
||||
start_idx = -1
|
||||
|
||||
# We need to find the sequence of headers
|
||||
search_start = 0
|
||||
|
||||
for i, target_title in enumerate(titles):
|
||||
found = False
|
||||
|
||||
for idx in range(search_start, len(lines)):
|
||||
level, text = _parse_header(lines[idx])
|
||||
if level > 0 and text == target_title:
|
||||
# Found the current segment
|
||||
search_start = idx + 1
|
||||
found = True
|
||||
if i == len(titles) - 1:
|
||||
start_idx = idx
|
||||
current_level_idx = level
|
||||
break
|
||||
|
||||
if not found:
|
||||
return -1, -1
|
||||
|
||||
if start_idx == -1:
|
||||
return -1, -1
|
||||
|
||||
# Find end: next header of same or lower level (higher importance, smaller integer)
|
||||
end_idx = len(lines)
|
||||
for idx in range(start_idx + 1, len(lines)):
|
||||
level, _ = _parse_header(lines[idx])
|
||||
if level > 0 and level <= current_level_idx:
|
||||
end_idx = idx
|
||||
break
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def report_read(
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Read the current content of the report.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
if not report_file.exists():
|
||||
return "Report is empty."
|
||||
return report_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def report_read_chapter(
|
||||
title: Annotated[str, ParamMeta(description="Chapter title to read (supports multi-level index e.g. 'Intro/Background')")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Read the content of a specific chapter.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
start, end = _find_chapter_range(lines, title)
|
||||
if start == -1:
|
||||
return f"Chapter '{title}' not found."
|
||||
|
||||
# Return content (excluding header)
|
||||
# start is the header line, so start+1
|
||||
return "\\n".join(lines[start+1:end])
|
||||
|
||||
|
||||
def report_outline(
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the outline of the report (headers).
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
outline = []
|
||||
for line in lines:
|
||||
level, title = _parse_header(line)
|
||||
if level > 0:
|
||||
outline.append(f"{'#' * level} {title}")
|
||||
|
||||
if not outline:
|
||||
return "No headers found in report."
|
||||
return "\n".join(outline)
|
||||
|
||||
|
||||
def report_create_chapter(
|
||||
title: Annotated[str, ParamMeta(description="Chapter title (supports 'Parent/NewChild' to insert into existing). Use '|' to specify insertion point e.g. 'Prev|New' to insert after 'Prev', or '|New' to insert at start.")],
|
||||
level: Annotated[int, ParamMeta(description="Header level (1-6)")],
|
||||
content: Annotated[str, ParamMeta(description="Content of the chapter")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Create a new chapter in the report.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
# Check for routing path
|
||||
parent_path = None
|
||||
display_title = title
|
||||
p_start, p_end = -1, len(lines)
|
||||
|
||||
if "/" in title:
|
||||
# Handle recursive "Parent/Child" structure, where Child might contain "|"
|
||||
parent_path, new_title = title.rsplit("/", 1)
|
||||
p_start, p_end = _find_chapter_range(lines, parent_path)
|
||||
|
||||
if p_start == -1:
|
||||
return f"Parent chapter '{parent_path}' not found. Cannot create '{new_title}' inside it."
|
||||
|
||||
display_title = new_title
|
||||
|
||||
# Check for "|" syntax in the leaf title
|
||||
insert_after_target = None # None means append, "" means start, "str" means after that chapter
|
||||
if "|" in display_title:
|
||||
target, real_title = display_title.split("|", 1)
|
||||
display_title = real_title
|
||||
insert_after_target = target
|
||||
|
||||
# Determine insertion index
|
||||
insert_idx = -1
|
||||
|
||||
if insert_after_target is not None:
|
||||
if insert_after_target == "":
|
||||
# Insert at the beginning of the context
|
||||
if parent_path:
|
||||
# Inside parent: Insert after parent header (and its intro text), before first subchapter
|
||||
insert_idx = p_end # Default to appending if no subchapters found
|
||||
|
||||
# Scan for first header inside parent
|
||||
for idx in range(p_start + 1, len(lines)):
|
||||
if idx >= p_end:
|
||||
break
|
||||
lvl, _ = _parse_header(lines[idx])
|
||||
if lvl > 0:
|
||||
insert_idx = idx
|
||||
break
|
||||
else:
|
||||
# Top level: Insert at start of file
|
||||
insert_idx = 0
|
||||
else:
|
||||
# Insert after the specified chapter
|
||||
# If we are inside a parent, the target must be relative to the parent?
|
||||
# The user requirement says "Prev|New".
|
||||
# If inside "Parent", "Prev" should be a sibling inside "Parent".
|
||||
|
||||
search_target = insert_after_target
|
||||
if parent_path:
|
||||
# Construct full path for search if we are scoped
|
||||
search_target = f"{parent_path}/{insert_after_target}"
|
||||
|
||||
a_start, a_end = _find_chapter_range(lines, search_target)
|
||||
if a_start == -1:
|
||||
return f"Target chapter '{search_target}' not found."
|
||||
insert_idx = a_end
|
||||
else:
|
||||
# Default: Append to parent context or file end
|
||||
insert_idx = p_end if parent_path else len(lines)
|
||||
|
||||
header = f"{'#' * level} {display_title}"
|
||||
new_section = [header] + content.splitlines() + [""]
|
||||
|
||||
# Insert
|
||||
lines[insert_idx:insert_idx] = new_section
|
||||
|
||||
_save_report(report_file, lines)
|
||||
|
||||
final_path = f"{parent_path}/{display_title}" if parent_path else display_title
|
||||
return f"Created chapter '{final_path}' at level {level}"
|
||||
|
||||
|
||||
def report_rewrite_chapter(
|
||||
title: Annotated[str, ParamMeta(description="Chapter title to rewrite (supports multi-level index e.g. 'Intro/Background')")],
|
||||
content: Annotated[str, ParamMeta(description="New content")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Rewrite the content of an existing chapter.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
start, end = _find_chapter_range(lines, title)
|
||||
if start == -1:
|
||||
return f"Chapter '{title}' not found."
|
||||
|
||||
# Keep the header, replace the body
|
||||
# new body should not contain the header itself, just the content
|
||||
new_body = [lines[start]] + content.splitlines() + [""]
|
||||
|
||||
# Replace slice
|
||||
lines[start:end] = new_body
|
||||
|
||||
_save_report(report_file, lines)
|
||||
return f"Rewrote chapter '{title}'"
|
||||
|
||||
|
||||
def report_continue_chapter(
|
||||
title: Annotated[str, ParamMeta(description="Chapter title to append to (supports multi-level index e.g. 'Intro/Background')")],
|
||||
content: Annotated[str, ParamMeta(description="Content to append")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Append content to an existing chapter.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
start, end = _find_chapter_range(lines, title)
|
||||
if start == -1:
|
||||
return f"Chapter '{title}' not found."
|
||||
|
||||
# Append content before 'end' (which is the start of next section or end of file)
|
||||
new_lines = content.splitlines() + [""]
|
||||
lines[end:end] = new_lines
|
||||
|
||||
_save_report(report_file, lines)
|
||||
return f"Appended content to chapter '{title}'"
|
||||
|
||||
|
||||
def report_reorder_chapters(
|
||||
new_order: Annotated[List[str], ParamMeta(description="List of chapter titles in the new desired order")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Reorder chapters in the report.
|
||||
This swaps the positions of the specified chapters, preserving their content and valid text between them.
|
||||
All specified chapters must exist and must not overlap (e.g. cannot reorder a parent and its child).
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
# 1. Find all ranges
|
||||
chapters = [] # (index in new_order, title, start, end)
|
||||
for i, title in enumerate(new_order):
|
||||
s, e = _find_chapter_range(lines, title)
|
||||
if s == -1:
|
||||
return f"Chapter '{title}' not found."
|
||||
chapters.append({
|
||||
"target_order_idx": i,
|
||||
"title": title,
|
||||
"content": lines[s:e],
|
||||
"start": s,
|
||||
"end": e
|
||||
})
|
||||
|
||||
# 2. Sort by original position in file to identify slots
|
||||
chapters_sorted_by_pos = sorted(chapters, key=lambda x: x["start"])
|
||||
|
||||
# 3. Validation: Check for overlaps
|
||||
for i in range(len(chapters_sorted_by_pos) - 1):
|
||||
curr = chapters_sorted_by_pos[i]
|
||||
next_ch = chapters_sorted_by_pos[i+1]
|
||||
if curr["end"] > next_ch["start"]:
|
||||
return f"Chapters '{curr['title']}' and '{next_ch['title']}' overlap. Cannot reorder nested or overlapping chapters."
|
||||
|
||||
# 4. Construct new line list
|
||||
result_lines = []
|
||||
current_idx = 0
|
||||
|
||||
for k, original_slot_holder in enumerate(chapters_sorted_by_pos):
|
||||
# Append text before this slot
|
||||
result_lines.extend(lines[current_idx : original_slot_holder["start"]])
|
||||
|
||||
# Append the content of the chapter that belongs in this k-th slot
|
||||
# The slot sequence corresponds to the input list order
|
||||
desired_chapter = chapters[k]
|
||||
result_lines.extend(desired_chapter["content"])
|
||||
|
||||
current_idx = original_slot_holder["end"]
|
||||
|
||||
# Append remaining file content
|
||||
result_lines.extend(lines[current_idx:])
|
||||
|
||||
_save_report(report_file, result_lines)
|
||||
|
||||
return "Reordered chapters successfully."
|
||||
|
||||
|
||||
def report_del_chapter(
|
||||
title: Annotated[str, ParamMeta(description="Chapter title to delete (supports multi-level index e.g. 'Intro/Background')")],
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Delete a chapter and its content.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
lines = _read_report_lines(report_file)
|
||||
|
||||
start, end = _find_chapter_range(lines, title)
|
||||
if start == -1:
|
||||
return f"Chapter '{title}' not found."
|
||||
|
||||
del lines[start:end]
|
||||
|
||||
_save_report(report_file, lines)
|
||||
return f"Deleted chapter '{title}'"
|
||||
|
||||
def report_export_pdf(
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> List[MessageBlock]:
|
||||
"""
|
||||
Export the report to PDF.
|
||||
"""
|
||||
ctx = FileToolContext(_context)
|
||||
_, report_file = _get_files(ctx)
|
||||
_, report_lock = _get_locks(ctx)
|
||||
|
||||
with FileLock(report_lock):
|
||||
if not report_file.exists():
|
||||
raise FileNotFoundError("Report file does not exist.")
|
||||
text = report_file.read_text(encoding="utf-8")
|
||||
|
||||
text = re.sub(r"([^\n])\n(#{1,6}\s)", r"\1\n\n\2", text)
|
||||
text = re.sub(r"(?m)^(?!\s*(?:[*+-]|\d+\.)\s)(.+)\n(\s*(?:[*+-]|\d+\.)\s)", r"\1\n\n\2", text)
|
||||
|
||||
try:
|
||||
import markdown
|
||||
from xhtml2pdf import pisa
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Error: strict dependencies 'markdown' and 'xhtml2pdf' are missing."
|
||||
)
|
||||
|
||||
pdf_file = report_file.with_suffix(".pdf")
|
||||
|
||||
# Convert to HTML
|
||||
extensions = ["extra", "codehilite", "nl2br", "tables"]
|
||||
html_content = markdown.markdown(text, extensions=extensions)
|
||||
|
||||
styled_html = f"""
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
@page {{
|
||||
size: A4;
|
||||
margin: 2cm;
|
||||
}}
|
||||
body {{
|
||||
font-family: sans-serif;
|
||||
line-height: 1.6;
|
||||
font-size: 10pt;
|
||||
word-wrap: break-word;
|
||||
word-break: break-all;
|
||||
}}
|
||||
h1, h2, h3 {{
|
||||
color: #2c3e50;
|
||||
margin-top: 25px; /* Add spacing above the title */
|
||||
margin-bottom: 15px;
|
||||
border-bottom: 1px solid #eee; /* Add an underline to the main title for clarity */
|
||||
padding-bottom: 5px;
|
||||
}}
|
||||
|
||||
/* --- Table style fixes --- */
|
||||
table {{
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-bottom: 20px;
|
||||
border: 1px solid #ddd;
|
||||
}}
|
||||
th, td {{
|
||||
border: 1px solid #ddd; /* Explicitly add borders */
|
||||
padding: 8px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}}
|
||||
th {{
|
||||
background-color: #f2f2f2;
|
||||
font-weight: bold;
|
||||
color: #333;
|
||||
}}
|
||||
/* ------------------ */
|
||||
|
||||
code {{ background-color: #f4f4f4; padding: 2px 5px; border-radius: 3px; font-family: monospace; }}
|
||||
pre {{ background-color: #f4f4f4; padding: 10px; border-radius: 5px; overflow-x: auto; white-space: pre-wrap; }}
|
||||
ul, ol {{ margin-top: 8px; margin-bottom: 8px; padding-left: 20px; }}
|
||||
li {{ margin-bottom: 4px; }}
|
||||
blockquote {{ border-left: 4px solid #ccc; padding-left: 10px; color: #666; margin: 10px 0; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{html_content}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Convert to PDF
|
||||
try:
|
||||
with open(pdf_file, "wb") as f:
|
||||
pisa_status = pisa.CreatePDF(styled_html, dest=f)
|
||||
|
||||
if pisa_status.err:
|
||||
raise RuntimeError("Failed to generate PDF: xhtml2pdf error")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate PDF: {e}")
|
||||
|
||||
record = ctx.attachment_store.register_file(
|
||||
pdf_file,
|
||||
kind=MessageBlockType.FILE,
|
||||
display_name=pdf_file.name,
|
||||
mime_type="application/pdf",
|
||||
copy_file=False,
|
||||
persist=False,
|
||||
deduplicate=True,
|
||||
extra={
|
||||
"source": "generated_report",
|
||||
"workspace_path": str(pdf_file),
|
||||
},
|
||||
)
|
||||
return [record.as_message_block()]
|
||||
Executable
+1100
File diff suppressed because it is too large
Load Diff
Executable
+17
@@ -0,0 +1,17 @@
|
||||
def call_user(instruction: str, _context: dict | None = None) -> str:
|
||||
"""
|
||||
If you think it's necessary to get input from the user, use this function to send the instruction to the user and get their response.
|
||||
|
||||
Args:
|
||||
instruction: The instruction to send to the user.
|
||||
"""
|
||||
prompt = _context.get("human_prompt") if _context else None
|
||||
if prompt is None:
|
||||
return f"Human prompt unavailable, default response for instruction: {instruction}"
|
||||
result = prompt.request(
|
||||
node_id=_context.get("node_id", "model_function_calling"),
|
||||
task_description="Please response to the model instruction.",
|
||||
inputs=instruction,
|
||||
metadata={"source": "function_tool"},
|
||||
)
|
||||
return result.text
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
import time
|
||||
from typing import Union
|
||||
|
||||
|
||||
def wait(seconds: float):
|
||||
"""
|
||||
Wait for a specified number of seconds.
|
||||
|
||||
Args:
|
||||
seconds: The number of seconds to wait.
|
||||
"""
|
||||
if isinstance(seconds, str): # Convert string to float if necessary
|
||||
try:
|
||||
if "." in seconds:
|
||||
seconds = float(seconds)
|
||||
else:
|
||||
seconds = int(seconds)
|
||||
except ValueError:
|
||||
# Fallback to float if int conversion fails or other string formats
|
||||
seconds = float(seconds)
|
||||
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def get_current_time():
|
||||
"""
|
||||
Get the current time in the format: YYYY-MM-DD HH:MM:SS.
|
||||
|
||||
Returns:
|
||||
str: The current time in the format: YYYY-MM-DD HH:MM:SS.
|
||||
"""
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||||
Executable
+313
@@ -0,0 +1,313 @@
|
||||
"""Utility tool to manage Python environments via uv."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Mapping, Sequence
|
||||
|
||||
_SAFE_PACKAGE_RE = re.compile(r"^[A-Za-z0-9_.\-+=<>!\[\],@:/]+$")
|
||||
_DEFAULT_TIMEOUT = float(os.getenv("LIB_INSTALL_TIMEOUT", "120"))
|
||||
_OUTPUT_SNIPPET_LIMIT = 240
|
||||
|
||||
|
||||
def _trim_output_preview(stdout: str, stderr: str) -> str | None:
|
||||
"""Return a short preview from stdout or stderr for error messaging."""
|
||||
|
||||
preview_source = stdout.strip() or stderr.strip()
|
||||
if not preview_source:
|
||||
return None
|
||||
if len(preview_source) <= _OUTPUT_SNIPPET_LIMIT:
|
||||
return preview_source
|
||||
return f"{preview_source[:_OUTPUT_SNIPPET_LIMIT].rstrip()}... [truncated]"
|
||||
|
||||
|
||||
def _build_timeout_message(step: str | None, timeout_value: float, stdout: str, stderr: str) -> str:
|
||||
"""Create a descriptive timeout error message with optional output preview."""
|
||||
|
||||
label = "uv command"
|
||||
if step:
|
||||
label = f"{label} ({step})"
|
||||
message = f"{label} timed out after {timeout_value} seconds"
|
||||
preview = _trim_output_preview(stdout, stderr)
|
||||
if preview:
|
||||
return f"{message}. Last output: {preview}"
|
||||
return message
|
||||
|
||||
|
||||
class WorkspaceCommandContext:
|
||||
"""Resolve the workspace root from the injected runtime context."""
|
||||
|
||||
def __init__(self, ctx: Dict[str, Any] | None):
|
||||
if ctx is None:
|
||||
raise ValueError("_context is required for uv tools")
|
||||
self.workspace_root = self._require_workspace(ctx.get("python_workspace_root"))
|
||||
self._raw_ctx = ctx
|
||||
|
||||
@staticmethod
|
||||
def _require_workspace(raw_path: Any) -> Path:
|
||||
if raw_path is None:
|
||||
raise ValueError("python_workspace_root missing from _context")
|
||||
path = Path(raw_path).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
def resolve_under_workspace(self, relative_path: str | Path) -> Path:
|
||||
candidate = Path(relative_path)
|
||||
absolute = candidate if candidate.is_absolute() else self.workspace_root / candidate
|
||||
absolute = absolute.expanduser().resolve()
|
||||
if self.workspace_root not in absolute.parents and absolute != self.workspace_root:
|
||||
raise ValueError("script path is outside workspace root")
|
||||
return absolute
|
||||
|
||||
|
||||
def _validate_packages(packages: Sequence[str]) -> List[str]:
|
||||
normalized: List[str] = []
|
||||
for pkg in packages:
|
||||
if not isinstance(pkg, str):
|
||||
raise ValueError("package entries must be strings")
|
||||
stripped = pkg.strip()
|
||||
if not stripped:
|
||||
raise ValueError("package names cannot be empty")
|
||||
if not _SAFE_PACKAGE_RE.match(stripped):
|
||||
raise ValueError(f"unsafe characters detected in package spec {pkg}")
|
||||
if stripped.startswith("-"):
|
||||
raise ValueError(f"flags are not allowed in packages list: {pkg}")
|
||||
normalized.append(stripped)
|
||||
if not normalized:
|
||||
raise ValueError("at least one package is required")
|
||||
return normalized
|
||||
|
||||
|
||||
def _coerce_timeout_seconds(timeout_seconds: Any) -> float | None:
|
||||
if timeout_seconds is None:
|
||||
return None
|
||||
if isinstance(timeout_seconds, bool):
|
||||
raise ValueError("timeout_seconds must be a number")
|
||||
if isinstance(timeout_seconds, (int, float)):
|
||||
value = float(timeout_seconds)
|
||||
elif isinstance(timeout_seconds, str):
|
||||
raw = timeout_seconds.strip()
|
||||
if not raw:
|
||||
raise ValueError("timeout_seconds cannot be empty")
|
||||
try:
|
||||
if re.fullmatch(r"[+-]?\d+", raw):
|
||||
value = float(int(raw))
|
||||
else:
|
||||
value = float(raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError("timeout_seconds must be a number") from exc
|
||||
else:
|
||||
raise ValueError("timeout_seconds must be a number")
|
||||
|
||||
if value <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_flag_args(args: Sequence[str] | None) -> List[str]:
|
||||
normalized: List[str] = []
|
||||
if not args:
|
||||
return normalized
|
||||
for arg in args:
|
||||
if not isinstance(arg, str):
|
||||
raise ValueError("extra args must be strings")
|
||||
stripped = arg.strip()
|
||||
if not stripped:
|
||||
raise ValueError("extra args cannot be empty")
|
||||
if not stripped.startswith("-"):
|
||||
raise ValueError(f"extra args must be flags, got {arg}")
|
||||
normalized.append(stripped)
|
||||
return normalized
|
||||
|
||||
|
||||
def _validate_args(args: Sequence[str] | None) -> List[str]:
|
||||
normalized: List[str] = []
|
||||
if not args:
|
||||
return normalized
|
||||
for arg in args:
|
||||
if not isinstance(arg, str):
|
||||
raise ValueError("args entries must be strings")
|
||||
stripped = arg.strip()
|
||||
if not stripped:
|
||||
raise ValueError("args entries cannot be empty")
|
||||
normalized.append(stripped)
|
||||
return normalized
|
||||
|
||||
|
||||
def _validate_env(env: Mapping[str, str] | None) -> Dict[str, str]:
|
||||
if env is None:
|
||||
return {}
|
||||
result: Dict[str, str] = {}
|
||||
for key, value in env.items():
|
||||
if not isinstance(key, str) or not key:
|
||||
raise ValueError("environment variable keys must be non-empty strings")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("environment variable values must be strings")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _run_uv_command(
|
||||
cmd: List[str],
|
||||
workspace_root: Path,
|
||||
*,
|
||||
step: str | None = None,
|
||||
env: Dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
timeout_value = _DEFAULT_TIMEOUT if timeout is None else timeout
|
||||
env_vars = None if env is None else {**os.environ, **env}
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(workspace_root),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_value,
|
||||
check=False,
|
||||
env=env_vars,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError("uv command not found in PATH") from exc
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
stdout_text = exc.stdout
|
||||
if stdout_text is None:
|
||||
stdout_text = getattr(exc, "output", "") or ""
|
||||
stderr_text = exc.stderr or ""
|
||||
message = _build_timeout_message(step, timeout_value, stdout_text, stderr_text)
|
||||
return {
|
||||
"command": cmd,
|
||||
"stdout": stdout_text,
|
||||
"stderr": stderr_text,
|
||||
"returncode": None,
|
||||
"step": step,
|
||||
"timed_out": True,
|
||||
"timeout": timeout_value,
|
||||
"error": message,
|
||||
}
|
||||
|
||||
return {
|
||||
"command": cmd,
|
||||
"stdout": completed.stdout or "",
|
||||
"stderr": completed.stderr or "",
|
||||
"returncode": completed.returncode,
|
||||
# "cwd": str(workspace_root),
|
||||
"step": step,
|
||||
}
|
||||
|
||||
|
||||
def install_python_packages(
|
||||
packages: Sequence[str],
|
||||
*,
|
||||
upgrade: bool = False,
|
||||
# extra_args: Sequence[str] | None = None,
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Install Python packages inside the workspace using uv add."""
|
||||
|
||||
ctx = WorkspaceCommandContext(_context)
|
||||
safe_packages = _validate_packages(packages)
|
||||
cmd: List[str] = ["uv", "add"]
|
||||
if upgrade:
|
||||
cmd.append("--upgrade")
|
||||
|
||||
# if extra_args:
|
||||
# flags = _validate_flag_args(extra_args)
|
||||
# cmd.extend(flags)
|
||||
|
||||
cmd.extend(safe_packages)
|
||||
result = _run_uv_command(cmd, ctx.workspace_root, step="uv add")
|
||||
# result["workspace_root"] = str(ctx.workspace_root)
|
||||
return result
|
||||
|
||||
|
||||
def init_python_env(
|
||||
*,
|
||||
# recreate: bool = False,
|
||||
python_version: str | None = None,
|
||||
# lock_args: Sequence[str] | None = None,
|
||||
# venv_args: Sequence[str] | None = None,
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run uv lock and uv venv inside the workspace."""
|
||||
|
||||
ctx = WorkspaceCommandContext(_context)
|
||||
steps: List[Dict[str, Any]] = []
|
||||
|
||||
lock_cmd: List[str] = ["uv", "lock"]
|
||||
# lock_cmd.extend(_validate_flag_args(lock_args))
|
||||
lock_result = _run_uv_command(lock_cmd, ctx.workspace_root, step="uv lock")
|
||||
steps.append(lock_result)
|
||||
if lock_result["returncode"] != 0:
|
||||
return {
|
||||
"workspace_root": str(ctx.workspace_root),
|
||||
"steps": steps,
|
||||
}
|
||||
|
||||
venv_cmd: List[str] = ["uv", "venv"]
|
||||
# if recreate:
|
||||
# venv_cmd.append("--recreate")
|
||||
# venv_cmd.extend(_validate_flag_args(venv_args))
|
||||
if python_version is not None:
|
||||
python_spec = python_version.strip()
|
||||
if not python_spec:
|
||||
raise ValueError("python argument cannot be empty")
|
||||
venv_cmd.extend(["--python", python_spec])
|
||||
|
||||
venv_result = _run_uv_command(venv_cmd, ctx.workspace_root, step="uv venv")
|
||||
steps.append(venv_result)
|
||||
|
||||
init_cmd: List[str] = ["uv", "init", "--bare", "--no-workspace"]
|
||||
init_result = _run_uv_command(init_cmd, ctx.workspace_root, step="uv init")
|
||||
steps.append(init_result)
|
||||
|
||||
return {
|
||||
"workspace_root": str(ctx.workspace_root),
|
||||
"steps": steps,
|
||||
}
|
||||
|
||||
|
||||
def uv_run(
|
||||
*,
|
||||
module: str | None = None,
|
||||
script: str | None = None,
|
||||
args: Sequence[str] | None = None,
|
||||
env: Mapping[str, str] | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute uv run for a module or script inside the workspace root."""
|
||||
|
||||
ctx = WorkspaceCommandContext(_context)
|
||||
timeout_seconds = _coerce_timeout_seconds(timeout_seconds)
|
||||
|
||||
has_module = module is not None
|
||||
has_script = script is not None
|
||||
if has_module == has_script:
|
||||
raise ValueError("Provide exactly one of module or script")
|
||||
|
||||
cmd: List[str] = ["uv", "run"]
|
||||
if has_module:
|
||||
module_name = module.strip()
|
||||
if not module_name:
|
||||
raise ValueError("module cannot be empty")
|
||||
cmd.extend(["python", "-m", module_name])
|
||||
else:
|
||||
script_value = script.strip() if isinstance(script, str) else script
|
||||
if not script_value:
|
||||
raise ValueError("script cannot be empty")
|
||||
script_path = ctx.resolve_under_workspace(script_value)
|
||||
cmd.append(str(script_path))
|
||||
|
||||
cmd.extend(_validate_args(args))
|
||||
env_overrides = _validate_env(env)
|
||||
result = _run_uv_command(
|
||||
cmd,
|
||||
ctx.workspace_root,
|
||||
step="uv run",
|
||||
env=env_overrides,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
result["workspace_root"] = str(ctx.workspace_root)
|
||||
return result
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import ast
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
def _get_class_names(py_file: str) -> list[str]:
|
||||
file_path = Path(py_file)
|
||||
source = file_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(file_path))
|
||||
return [node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef)]
|
||||
|
||||
def render_manim(
|
||||
script_path: str,
|
||||
quality: str = "h",
|
||||
preview: bool = True,
|
||||
) -> Path:
|
||||
output_dir = Path.cwd() / "media"
|
||||
print("Clearing media folder:", output_dir)
|
||||
shutil.rmtree(output_dir, ignore_errors=True)
|
||||
script_path = Path(script_path).resolve()
|
||||
if not script_path.exists():
|
||||
raise FileNotFoundError(script_path, " does not exist.")
|
||||
scene_name = _get_class_names(str(script_path))[0]
|
||||
cmd = [
|
||||
sys.executable, "-m", "manim",
|
||||
f"-pq{quality}",
|
||||
]
|
||||
|
||||
if preview:
|
||||
cmd.insert(3, "-p")
|
||||
|
||||
cmd.extend([str(script_path), scene_name])
|
||||
|
||||
print("Running:", " ".join(cmd))
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Manim rendering failed:")
|
||||
print("stdout:", e.stdout)
|
||||
print("stderr:", e.stderr)
|
||||
error_info = f"Error rendering {scene_name} from {script_path}: {e.stderr}"
|
||||
# shutil.rmtree(output_dir, ignore_errors=True)
|
||||
raise RuntimeError(error_info)
|
||||
|
||||
# Find valid mp4 files where no parent directory contains a partial_movie_files folder
|
||||
video_file = None
|
||||
for mp4_file in (Path.cwd() / "media" / "videos").parent.rglob("*.mp4"):
|
||||
if mp4_file.name == f"{scene_name}.mp4":
|
||||
video_file = mp4_file
|
||||
break
|
||||
|
||||
target_path = script_path.parent / video_file.name
|
||||
print(f"Copying video to {target_path}")
|
||||
shutil.copy2(video_file, target_path)
|
||||
shutil.rmtree(output_dir, ignore_errors=True)
|
||||
return target_path
|
||||
|
||||
def concat_videos(video_paths: list[Path]) -> Path:
|
||||
if not video_paths:
|
||||
raise ValueError("No video files to concatenate")
|
||||
|
||||
video_paths = [Path(p).resolve() for p in video_paths]
|
||||
output_path = video_paths[0].parent / "combined_video.mp4"
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
|
||||
for p in video_paths:
|
||||
f.write(f"file '{p.as_posix()}'\n")
|
||||
list_file = f.name
|
||||
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-f", "concat",
|
||||
"-safe", "0",
|
||||
"-i", list_file,
|
||||
"-c", "copy",
|
||||
str(output_path)
|
||||
]
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
return output_path
|
||||
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
def get_city_num(city: str) -> dict:
|
||||
"""
|
||||
Fetch the city code for a given city name.
|
||||
Example response:
|
||||
{
|
||||
"city": "Beijing",
|
||||
"city_num": "1010",
|
||||
}
|
||||
"""
|
||||
return {
|
||||
"city_num": 3701
|
||||
}
|
||||
|
||||
def get_weather(city_num: int, unit: str = "celsius") -> dict:
|
||||
"""
|
||||
Fetch weather information for the city represented by ``city_num``.
|
||||
Example response:
|
||||
{
|
||||
"city_num": "1010",
|
||||
"temperature": 20,
|
||||
"unit": "celsius"
|
||||
}
|
||||
"""
|
||||
temperature_c = 15 # Hardcode the temperature value
|
||||
if unit == "fahrenheit":
|
||||
temperature = temperature_c * 9 / 5 + 32
|
||||
else:
|
||||
temperature = temperature_c
|
||||
|
||||
return {
|
||||
"city_num": city_num,
|
||||
"temperature": temperature,
|
||||
"unit": unit
|
||||
}
|
||||
Executable
+172
@@ -0,0 +1,172 @@
|
||||
import os
|
||||
|
||||
|
||||
def web_search(query: str, page: int = 1, language: str = "en", country: str = "us") -> str:
|
||||
"""
|
||||
Performs a web search based on the user-provided query with pagination.
|
||||
|
||||
Args:
|
||||
query (str): The keyword(s) to search for.
|
||||
page (int): The page number of the results to return. Defaults to 1.
|
||||
language (str): The language of the search results. Defaults to "en", can be "en", "zh-cn", "zh-tw", "ja", "ko".
|
||||
country (str): The country of the search results. Defaults to "us", can be "us", "cn", "jp", "kr".
|
||||
|
||||
Returns:
|
||||
str: A formatted string containing the title, link, and snippet of the search results for the specified page.
|
||||
"""
|
||||
import requests
|
||||
import json
|
||||
|
||||
url = "https://google.serper.dev/search"
|
||||
|
||||
payload = json.dumps({
|
||||
"q": query,
|
||||
"page": page,
|
||||
"hl": language,
|
||||
"gl": country
|
||||
})
|
||||
headers = {
|
||||
'X-API-KEY': os.getenv("SERPER_DEV_API_KEY"),
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
response = requests.request("POST", url, headers=headers, data=payload)
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
return __format_serper_results(data)
|
||||
except json.JSONDecodeError:
|
||||
return response.text
|
||||
|
||||
|
||||
def __format_serper_results(data: dict) -> str:
|
||||
"""
|
||||
Formats the raw JSON response from Serper.dev into a structured string.
|
||||
"""
|
||||
formatted_output = []
|
||||
|
||||
# 1. Knowledge Graph
|
||||
if "knowledgeGraph" in data:
|
||||
kg = data["knowledgeGraph"]
|
||||
formatted_output.append("## Knowledge Graph")
|
||||
if "title" in kg:
|
||||
formatted_output.append(f"**Title**: {kg['title']}")
|
||||
if "type" in kg:
|
||||
formatted_output.append(f"**Type**: {kg['type']}")
|
||||
if "description" in kg:
|
||||
if "descriptionSource" in kg and "descriptionLink" in kg:
|
||||
formatted_output.append(f"**Description**: {kg['description']} (Source: [{kg['descriptionSource']}]({kg['descriptionLink']}))")
|
||||
else:
|
||||
formatted_output.append(f"**Description**: {kg['description']}")
|
||||
|
||||
if "attributes" in kg:
|
||||
formatted_output.append("**Attributes**:")
|
||||
for key, value in kg["attributes"].items():
|
||||
formatted_output.append(f"- {key}: {value}")
|
||||
formatted_output.append("") # Add spacing
|
||||
|
||||
# 2. Organic Results
|
||||
if "organic" in data and data["organic"]:
|
||||
formatted_output.append("## Organic Results")
|
||||
for i, result in enumerate(data["organic"], 1):
|
||||
title = result.get("title", "No Title")
|
||||
link = result.get("link", "#")
|
||||
snippet = result.get("snippet", "")
|
||||
formatted_output.append(f"{i}. **[{title}]({link})**")
|
||||
if snippet:
|
||||
formatted_output.append(f" {snippet}")
|
||||
|
||||
# Optional: Include attributes if useful, but keep it concise
|
||||
if "attributes" in result:
|
||||
for key, value in result["attributes"].items():
|
||||
formatted_output.append(f" - {key}: {value}")
|
||||
formatted_output.append("")
|
||||
|
||||
# 3. People Also Ask
|
||||
if "peopleAlsoAsk" in data and data["peopleAlsoAsk"]:
|
||||
formatted_output.append("## People Also Ask")
|
||||
for item in data["peopleAlsoAsk"]:
|
||||
question = item.get("question")
|
||||
snippet = item.get("snippet")
|
||||
link = item.get("link")
|
||||
title = item.get("title")
|
||||
|
||||
if question:
|
||||
formatted_output.append(f"- **{question}**")
|
||||
if snippet:
|
||||
formatted_output.append(f" {snippet}")
|
||||
if link and title:
|
||||
formatted_output.append(f" Source: [{title}]({link})")
|
||||
formatted_output.append("")
|
||||
|
||||
# 4. Related Searches
|
||||
if "relatedSearches" in data and data["relatedSearches"]:
|
||||
formatted_output.append("## Related Searches")
|
||||
queries = [item["query"] for item in data["relatedSearches"] if "query" in item]
|
||||
formatted_output.append(", ".join(queries))
|
||||
|
||||
return "\n".join(formatted_output).strip()
|
||||
|
||||
|
||||
def read_webpage_content(url: str) -> str:
|
||||
"""
|
||||
Reads the content of a webpage and returns it as a string.
|
||||
"""
|
||||
import requests
|
||||
import time
|
||||
from collections import deque
|
||||
import threading
|
||||
|
||||
# Rate limiting configuration
|
||||
RATE_LIMIT = 20 # requests
|
||||
TIME_WINDOW = 60 # seconds
|
||||
|
||||
# Global state for rate limiting (thread-safe)
|
||||
if not hasattr(read_webpage_content, "_request_timestamps"):
|
||||
read_webpage_content._request_timestamps = deque()
|
||||
read_webpage_content._lock = threading.Lock()
|
||||
|
||||
target_url = f"https://r.jina.ai/{url}"
|
||||
key = os.getenv("JINA_API_KEY")
|
||||
|
||||
headers = {}
|
||||
if key:
|
||||
headers["Authorization"] = key
|
||||
else:
|
||||
# Apply rate limiting if no key is present
|
||||
with read_webpage_content._lock:
|
||||
current_time = time.time()
|
||||
|
||||
# Remove timestamps older than the time window
|
||||
while read_webpage_content._request_timestamps and \
|
||||
current_time - read_webpage_content._request_timestamps[0] > TIME_WINDOW:
|
||||
read_webpage_content._request_timestamps.popleft()
|
||||
|
||||
# Check if limit reached
|
||||
if len(read_webpage_content._request_timestamps) >= RATE_LIMIT:
|
||||
# Calculate sleep time
|
||||
oldest_request = read_webpage_content._request_timestamps[0]
|
||||
sleep_time = TIME_WINDOW - (current_time - oldest_request)
|
||||
if sleep_time > 0:
|
||||
time.sleep(sleep_time)
|
||||
|
||||
# After sleeping, we can pop the oldest since it expired (logically)
|
||||
# Re-check time/clean just to be safe and accurate,
|
||||
# but effectively we just waited for the slot to free up.
|
||||
# Ideally, we add the *new* request time now.
|
||||
# Note: after sleep, the current_time has advanced.
|
||||
current_time = time.time()
|
||||
# Clean up again
|
||||
while read_webpage_content._request_timestamps and \
|
||||
current_time - read_webpage_content._request_timestamps[0] > TIME_WINDOW:
|
||||
read_webpage_content._request_timestamps.popleft()
|
||||
|
||||
# Record the execution
|
||||
read_webpage_content._request_timestamps.append(time.time())
|
||||
|
||||
response = requests.get(target_url, headers=headers)
|
||||
return response.text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
Reference in New Issue
Block a user