chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,481 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
import math
|
||||
from io import BytesIO
|
||||
import os
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from tqdm import tqdm
|
||||
from contextlib import redirect_stdout, redirect_stderr
|
||||
import sys
|
||||
from openai import OpenAI
|
||||
from PIL import Image, ImageDraw
|
||||
from transformers import AutoProcessor
|
||||
import argparse
|
||||
|
||||
from mmrag_r1.llm_agent.qwen_tool_call import Qwen_agent
|
||||
import datetime
|
||||
|
||||
sys_prompt = """\
|
||||
You are a Web Information Seeking Master. Your task is to thoroughly seek the internet for information and provide accurate answers to visual questions.
|
||||
As you proceed, adhere to the following principles:
|
||||
|
||||
1. Decompose the original visual question into sub-questions and solve them step by step. Summarize the knowledge obtained from the previous round of dialogue, then think about what is next sub-question.
|
||||
2. Whether you can answer the question or not, you should describe the image in detail. if the image includes multiple sub-image, you should describe each one separately.
|
||||
3. You should provide the final answer within 10 turns, regardless of whether all valid information has been collected.
|
||||
"""
|
||||
|
||||
prompt_ins = '''\
|
||||
You are an intelligent agent engaged in a conversation with a user. The user poses a question and provides a corresponding image for context. As an agent, you approach the problem with care and methodical precision, following a multi-step process to arrive at a solution. You utilize a variety of tools, ensuring that the information gathered from each one is cross-validated before you reach a final answer. Rather than relying on any single tool for accuracy, you employ multiple tools iteratively to prioritize the comprehensiveness and reliability of your responses.
|
||||
<tools>
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Call this tool to interact with the web_search API. You will receive the top 10 text excerpts from Google's text search engine using text as the search query.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"queries": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"description": "The search query."
|
||||
},
|
||||
"description": "The list of search queries."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"queries"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "VLSearchImage",
|
||||
"description": "Call this tool to receive the top 10 images and corresponding descriptions from Google's image search engine. You can only search the input image and cannot conduct additional searches on the results obtained from the initial search. You'd better use this tool only once",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"image_urls": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "description": "The search image url."},
|
||||
"description": "The list of search image url."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"image_urls"
|
||||
]
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "visit",
|
||||
"description": "visit a webpage and return the summary of webpage.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "the url you want to explore."
|
||||
},
|
||||
"goal": {
|
||||
"type": "string",
|
||||
"description": "the goal of the visit for the webpage."
|
||||
}
|
||||
},
|
||||
"required": ["url","goal"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "code_interpreter",
|
||||
"description": "Call this tool to execute Python code for calculation, data analysis, or content extraction tasks.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string",
|
||||
"description": "The Python code to execute."
|
||||
},
|
||||
"required": ["code"]
|
||||
}
|
||||
}
|
||||
</tools>
|
||||
|
||||
The assistant starts with one or more cycles of (thinking about which tool to use -> performing tool call -> waiting for tool response), and ends with (thinking about the answer -> answer of the question). The thinking processes, tool calls, tool responses, and answer are enclosed within their tags. There could be multiple thinking processes, tool calls, tool call parameters and tool response parameters.
|
||||
|
||||
Example response:
|
||||
<think> thinking process here </think>
|
||||
<tool_call>
|
||||
{"name": "tool name here", "arguments": {"parameter name here": parameter value here, "another parameter name here": another parameter value here, ...}}
|
||||
</tool_call>
|
||||
<tool_response>
|
||||
{"name": "tool name here", "content": {"result name here": result value here, "another result name here": another
|
||||
result value here, ...}}
|
||||
</tool_response>
|
||||
<think> thinking process here </think>
|
||||
<tool_call>
|
||||
{"name": "another tool name here", "arguments": {...}}
|
||||
</tool_call>
|
||||
<tool_response>
|
||||
{"name": "another tool name here", "content": {...}}
|
||||
</tool_response>
|
||||
(more thinking processes, tool calls and tool responses here)
|
||||
<think> thinking process here </think>
|
||||
<answer> answer here </answer>
|
||||
|
||||
Input Question:{Question}
|
||||
Input image:{Image_url}
|
||||
'''
|
||||
|
||||
|
||||
class OmniSearch:
|
||||
def __init__(self,
|
||||
base_url='http://0.0.0.0:8001/v1',
|
||||
api_key='EMPTY'):
|
||||
|
||||
self.client = OpenAI(base_url=base_url, api_key=api_key)
|
||||
|
||||
self.max_pixels = 1024 * 28 * 28
|
||||
self.min_pixels = 256 * 28 * 28
|
||||
self.repeated_nums = 1
|
||||
self.max_steps = 12
|
||||
|
||||
self.qwen_agent = Qwen_agent(function_list=['web_search','VLSearchImage','visit','code_interpreter'])
|
||||
self.processor = AutoProcessor.from_pretrained(os.getenv('VLLM_MODEL', ''))
|
||||
|
||||
def process_image(self, image):
|
||||
if isinstance(image, dict):
|
||||
image = Image.open(BytesIO(image['bytes']))
|
||||
elif isinstance(image, str):
|
||||
image = Image.open(image)
|
||||
|
||||
if (image.width * image.height) > self.max_pixels:
|
||||
resize_factor = math.sqrt(self.max_pixels / (image.width * image.height))
|
||||
width, height = int(image.width * resize_factor), int(image.height * resize_factor)
|
||||
image = image.resize((width, height))
|
||||
|
||||
if (image.width * image.height) < self.min_pixels:
|
||||
resize_factor = math.sqrt(self.min_pixels / (image.width * image.height))
|
||||
width, height = int(image.width * resize_factor), int(image.height * resize_factor)
|
||||
image = image.resize((width, height))
|
||||
|
||||
if image.mode != 'RGB':
|
||||
image = image.convert('RGB')
|
||||
|
||||
byte_stream = BytesIO()
|
||||
image.save(byte_stream, format="JPEG")
|
||||
byte_array = byte_stream.getvalue()
|
||||
base64_encoded_image = base64.b64encode(byte_array)
|
||||
base64_string = base64_encoded_image.decode("utf-8")
|
||||
base64_qwen = f"data:image;base64,{base64_string}"
|
||||
|
||||
return image, base64_qwen
|
||||
|
||||
def search(self,query):
|
||||
if isinstance(query,str):
|
||||
query = [query]
|
||||
# search_response = requests.get(self.search_url, params={"queries": query})
|
||||
search_results = search_response.json()
|
||||
image_path_list = [result['image_file'] for result in search_results[0]]
|
||||
return image_path_list
|
||||
|
||||
def run_main(self, sample):
|
||||
self.image_raw = []
|
||||
self.image_input = []
|
||||
self.image_path = []
|
||||
test_image_dir = os.getenv("IMAGE_DIR")
|
||||
test_image_name = os.path.basename(sample['file_path'])
|
||||
image_raw = Image.open(os.path.join(test_image_dir, test_image_name))
|
||||
_, test_img_base64 = self.process_image(image_raw)
|
||||
|
||||
messages = [dict(
|
||||
role="system",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": sys_prompt,
|
||||
}
|
||||
]
|
||||
),
|
||||
dict(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt_ins.replace("{Image_url}", sample['file_path']).replace("{Question}", sample['prompt']),
|
||||
},
|
||||
{
|
||||
'type': 'image_url',
|
||||
'image_url': {
|
||||
'url': test_img_base64
|
||||
}
|
||||
}
|
||||
]
|
||||
)]
|
||||
|
||||
max_steps = self.max_steps
|
||||
while True:
|
||||
## assistant
|
||||
gen_times = 10
|
||||
while True:
|
||||
if gen_times < 0:
|
||||
return 'time_out', messages, 'No answer'
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model=os.getenv('VLLM_MODEL', ''),
|
||||
messages=messages,
|
||||
stream=False,
|
||||
top_p=0.95,
|
||||
temperature=0.6
|
||||
)
|
||||
response_content = response.choices[0].message.content
|
||||
# break
|
||||
if response_content:
|
||||
break
|
||||
else:
|
||||
raise Exception('vllm model failed, retrying...')
|
||||
except Exception as e:
|
||||
print(e)
|
||||
gen_times -= 1
|
||||
|
||||
messages.append(dict(
|
||||
role="assistant",
|
||||
content=[{
|
||||
"type": "text",
|
||||
"text": response_content
|
||||
}]
|
||||
))
|
||||
|
||||
|
||||
## think
|
||||
pattern = r'<think>(.*?)</think>'
|
||||
match = re.search(pattern, response_content, re.DOTALL)
|
||||
# thought = match.group(1)
|
||||
if match:
|
||||
thought = match.group(1)
|
||||
else:
|
||||
pattern = r'<think>(.*?)\n\n<Sub-Question>'
|
||||
print("[no valid <think> tag]")
|
||||
# print(f"messages:{messages}")
|
||||
|
||||
print("response_content: ", response_content)
|
||||
## opration
|
||||
pattern = r'<(tool_call|answer)>(.*?)</\1>'
|
||||
match = re.search(pattern, response_content, re.DOTALL)
|
||||
if match:
|
||||
raw_content = match.group(0)
|
||||
content = match.group(2).strip() # Return only the content inside the tags
|
||||
action = match.group(1)
|
||||
else:
|
||||
print(f"[No match tool_call or answer]")
|
||||
content = ''
|
||||
action = None
|
||||
|
||||
print("action: ", action)
|
||||
## whether end
|
||||
if action is None:
|
||||
user_content=[{
|
||||
'type': 'text',
|
||||
'text': 'please use valid tool or answer the question.'
|
||||
}]
|
||||
elif action == 'answer':
|
||||
return 'answer', messages, content
|
||||
elif max_steps==0:
|
||||
return 'time_out', messages, 'No answer'
|
||||
elif action == 'tool_call':
|
||||
# request_para = None
|
||||
try:
|
||||
try:
|
||||
request_para = json.loads(content)
|
||||
print(request_para)
|
||||
except Exception as e:
|
||||
# Step 1: 修复 queries 字段中带双引号的字符串
|
||||
content = re.sub(r'"\s*queries\s*"\s*:\s*\[\s*""(.*?)""\s*\]', r'"queries": ["\1"]', content)
|
||||
# Step 2: 如果还有 "" 替换成 "
|
||||
content = content.replace('""', '"')
|
||||
request_para = json.loads(content)
|
||||
|
||||
if request_para is None:
|
||||
raise Exception(f"Invalid request parameters. request_para is None, content:{content}")
|
||||
|
||||
img_save_path = 'scripts_eval/scripts_eval/images/search_image/' + datetime.datetime.now().strftime("%m%d")
|
||||
if not os.path.exists(img_save_path):
|
||||
os.makedirs(img_save_path)
|
||||
|
||||
if request_para['name'] in ['VLSearchImage', 'vlsearchimage']:
|
||||
user_query = sample.get('prompt', '')
|
||||
search_results = self.qwen_agent._call_tool(
|
||||
request_para['name'],
|
||||
request_para['arguments']
|
||||
)
|
||||
else:
|
||||
search_results = self.qwen_agent._call_tool(
|
||||
request_para['name'],
|
||||
request_para['arguments'],
|
||||
img_save_path=img_save_path,
|
||||
byte=True
|
||||
)
|
||||
print(search_results)
|
||||
|
||||
except Exception as e:
|
||||
print(e, f"Invalid request parameters. request_para is None, content:{content}")
|
||||
request_para = None
|
||||
|
||||
if request_para is None:
|
||||
user_content = [{
|
||||
'type': 'text',
|
||||
'text': 'please use valid tool or answer the question.'
|
||||
}]
|
||||
elif request_para['name'] in ['VLSearchImage']:
|
||||
## prefix
|
||||
user_content = [{
|
||||
'type': 'text',
|
||||
'text': '<tool_response>'
|
||||
}]
|
||||
|
||||
## content
|
||||
images_path = re.findall(r"Image: (.*?), Text:", search_results)
|
||||
text_description = re.findall(r"Text: (.*?)\nImage:", search_results)
|
||||
text_description_last = re.findall(r"Text: (.+)$", search_results)
|
||||
text_description_list = text_description + text_description_last
|
||||
# images_path = []
|
||||
if len(images_path)>0:
|
||||
for image_path in images_path:
|
||||
image_raw = Image.open(image_path)
|
||||
image_input, img_base64 = self.process_image(image_raw)
|
||||
user_content.append({
|
||||
'type': 'image_url',
|
||||
'image_url': {
|
||||
'url': img_base64
|
||||
}
|
||||
})
|
||||
user_content.append({
|
||||
'type': 'text',
|
||||
'text': search_results
|
||||
})
|
||||
else:
|
||||
user_content.append({
|
||||
'type': 'text',
|
||||
'text': search_results
|
||||
})
|
||||
|
||||
## suffix
|
||||
user_content.append({
|
||||
'type': 'text',
|
||||
'text': '</tool_response>'
|
||||
})
|
||||
|
||||
elif request_para['name'] in ['web_search','visit','Visit']:
|
||||
user_content=[{
|
||||
'type': 'text',
|
||||
'text': '<tool_response>' + search_results + '</tool_response>'
|
||||
}]
|
||||
|
||||
elif request_para['name'] in ['Code_Interpreter', 'code_interpreter', 'PythonInterpreter']:
|
||||
if isinstance(search_results, dict):
|
||||
code_result = json.dumps(search_results, ensure_ascii=False)
|
||||
else:
|
||||
code_result = str(search_results)
|
||||
print("Generated Code: ", code_result)
|
||||
user_content = [{
|
||||
'type': 'text',
|
||||
'text': f'<tool_response>{code_result}</tool_response>'
|
||||
}]
|
||||
|
||||
else:
|
||||
user_content = [{
|
||||
'type': 'text',
|
||||
'text': 'please use valid tool or answer the question.'
|
||||
}]
|
||||
|
||||
max_steps -= 1
|
||||
if max_steps == 0:
|
||||
user_content.append({
|
||||
'type': 'text',
|
||||
'text': 'please answer the question now with answer in <answer> ... </answer>'
|
||||
})
|
||||
messages.append(dict(
|
||||
role='user',
|
||||
content=user_content
|
||||
))
|
||||
|
||||
def infer(self, sample):
|
||||
try:
|
||||
status, messages, content = self.run_main(sample)
|
||||
except Exception as e:
|
||||
sample["response"] = e
|
||||
sample["gen"] = 'No Answer'
|
||||
print(e)
|
||||
return sample
|
||||
|
||||
text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
|
||||
sample["response"] = text
|
||||
sample["gen"] = content
|
||||
|
||||
return sample
|
||||
|
||||
def infer_with_timeout_retry(self, sample, max_retry=2, timeout_seconds=300):
|
||||
for attempt in range(max_retry+1):
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(self.infer, sample)
|
||||
try:
|
||||
result = future.result(timeout=timeout_seconds)
|
||||
return result
|
||||
except TimeoutError:
|
||||
print(f"[Timeout] Inference timeout for sample {sample.get('file_path','N/A')}, retry {attempt+1}/{max_retry}")
|
||||
except Exception as e:
|
||||
print(f"[Exception] Inference error for sample {sample.get('file_path', 'N/A')}: {e}, retry {attempt+1}/{max_retry}")
|
||||
print(f"[Fail] Inference failed after {max_retry+1} attempts for sample {sample.get('file_path','N/A')}")
|
||||
sample['response'] = 'Timeout/Error'
|
||||
sample['gen'] = 'Timeout/Error'
|
||||
return sample
|
||||
|
||||
def eval(self, input_file_list, output_file):
|
||||
data = []
|
||||
for input_file in input_file_list:
|
||||
with open(input_file,'r') as f:
|
||||
data.extend([json.loads(line) for line in f])
|
||||
max_workers = 20
|
||||
results = [None] * len(data)
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
future_to_index = {executor.submit(self.infer_with_timeout_retry, sample): idx for idx, sample in enumerate(data)}
|
||||
for future in tqdm(as_completed(future_to_index), total=len(data), desc="Inference Progress"):
|
||||
idx = future_to_index[future]
|
||||
try:
|
||||
res = future.result()
|
||||
except Exception as e:
|
||||
print(f"[Fatal] Sample {data[idx].get('file_path','')} error: {e}")
|
||||
res = {'response': 'Timeout/Error', 'gen': 'Timeout/Error'}
|
||||
results[idx] = res
|
||||
with open(output_file, 'a') as f:
|
||||
json.dump(res, f, ensure_ascii=False)
|
||||
f.write('\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
agent = OmniSearch()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--eval_data", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
output_file = args.output_file
|
||||
|
||||
if args.eval_data == 'hle':
|
||||
agent.eval(['vl_search_r1/eval_data/hle.jsonl'], output_file)
|
||||
elif args.eval_data == 'gaia':
|
||||
agent.eval(['vl_search_r1/eval_data/gaia.jsonl'], output_file)
|
||||
elif args.eval_data == 'livevqa':
|
||||
agent.eval(['vl_search_r1/eval_data/livevqa.jsonl'], output_file)
|
||||
elif args.eval_data == 'mmsearch':
|
||||
agent.eval(['vl_search_r1/eval_data/mmsearch.jsonl'], output_file)
|
||||
elif args.eval_data == 'simplevqa':
|
||||
agent.eval(['vl_search_r1/eval_data/simplevqa.jsonl'], output_file)
|
||||
elif args.eval_data == 'bc_vl_v1':
|
||||
agent.eval(['vl_search_r1/eval_data/bc_vl_v1.jsonl'], output_file)
|
||||
elif args.eval_data == 'bc_vl_v2':
|
||||
agent.eval(['vl_search_r1/eval_data/bc_vl_v2.jsonl'], output_file)
|
||||
else:
|
||||
raise ValueError('Invalid eval_data')
|
||||
@@ -0,0 +1,46 @@
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def download_images_from_jsonl(jsonl_file, output_dir):
|
||||
"""
|
||||
从JSONL文件中提取图片链接，下载图片并保存到本地指定文件夹。
|
||||
|
||||
:param jsonl_file: 输入的jsonl文件路径
|
||||
:param output_dir: 图片保存的目标目录
|
||||
"""
|
||||
# 如果输出目录不存在，创建它
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
# 打开并读取JSONL文件
|
||||
with open(jsonl_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
image_url = data.get('file_path')
|
||||
|
||||
if image_url:
|
||||
# 获取图片文件名（从URL中提取）
|
||||
image_name = os.path.basename(urlparse(image_url).path)
|
||||
|
||||
# 拼接成本地保存路径
|
||||
image_path = os.path.join(output_dir, image_name)
|
||||
|
||||
# 下载图片
|
||||
try:
|
||||
response = requests.get(image_url, stream=True)
|
||||
if response.status_code == 200:
|
||||
with open(image_path, 'wb') as img_file:
|
||||
for chunk in response.iter_content(1024):
|
||||
img_file.write(chunk)
|
||||
print(f"成功下载图片: {image_name}")
|
||||
else:
|
||||
print(f"下载失败: {image_url} (状态码: {response.status_code})")
|
||||
except Exception as e:
|
||||
print(f"下载图片时出错: {image_url} 错误: {e}")
|
||||
|
||||
# 使用示例
|
||||
jsonl_file = 'vl_search_r1/eval_data/hle_50.jsonl' # 你的jsonl文件路径
|
||||
output_dir = 'scripts_eval/images/hle_50' # 图片保存的目标文件夹
|
||||
download_images_from_jsonl(jsonl_file, output_dir)
|
||||
Binary file not shown.
@@ -0,0 +1,17 @@
|
||||
from qwen_agent.tools import BaseTool
|
||||
from .sandbox_module import PythonCodeExecutor
|
||||
|
||||
class CodeInterpreterTool(BaseTool):
|
||||
name = "code_interpreter"
|
||||
description = "Call this tool to execute Python code for calculation, data analysis, or content extraction tasks."
|
||||
file_access = False
|
||||
|
||||
def __init__(self, timeout=50):
|
||||
self.executor = PythonCodeExecutor()
|
||||
|
||||
def call(self, code: str, goal: str = "") -> dict:
|
||||
"""
|
||||
Qwen-agent会自动传入code（和goal）。
|
||||
"""
|
||||
result, raw_resp = self.executor.execute_code(code)
|
||||
return {"result": result}
|
||||
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
def run_code_in_sandbox(code, timeout=50):
|
||||
"""
|
||||
使用 HTTP API 调用沙箱服务编译运行 Python 代码
|
||||
需要跟你的 PythonInterpreter 工具后端兼容
|
||||
"""
|
||||
SANDBOX_FUSION_ENDPOINT = os.environ.get('SANDBOX_FUSION_ENDPOINT', f'{CODE_SERVER_IP}:8080')
|
||||
if not (SANDBOX_FUSION_ENDPOINT.startswith("http://") or SANDBOX_FUSION_ENDPOINT.startswith("https://")):
|
||||
SANDBOX_FUSION_ENDPOINT = "http://" + SANDBOX_FUSION_ENDPOINT
|
||||
|
||||
payload = {
|
||||
"code": code,
|
||||
"language": "python"
|
||||
}
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{SANDBOX_FUSION_ENDPOINT}/run_code",
|
||||
json=payload,
|
||||
timeout=timeout
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
# data 格式： {run_result: {stdout:...,stderr:...}, status:...}
|
||||
except Exception as e:
|
||||
return f"[Code Execution Error]: {e}", None
|
||||
|
||||
stdout = data.get("run_result", {}).get("stdout", "")
|
||||
stderr = data.get("run_result", {}).get("stderr", "")
|
||||
result = ""
|
||||
if stdout.strip():
|
||||
result += f"stdout:\n{stdout}"
|
||||
if stderr.strip():
|
||||
result += f"\nstderr:\n{stderr}"
|
||||
return result if result.strip() else "Finished execution.", data
|
||||
|
||||
def extract_code_from_response(resp: str) -> Optional[str]:
|
||||
code_match = re.search(r"<code>([\s\S]+?)</code>", resp)
|
||||
if code_match:
|
||||
return code_match.group(1)
|
||||
|
||||
code_block_match = re.search(r'```[^\n]*\n(.+?)```', resp, re.DOTALL)
|
||||
if code_block_match:
|
||||
return code_block_match.group(1)
|
||||
return None
|
||||
|
||||
class PythonCodeExecutor:
|
||||
def __init__(self, timeout=50):
|
||||
self.timeout = timeout
|
||||
|
||||
def execute_code(self, code):
|
||||
return run_code_in_sandbox(code, timeout=self.timeout)
|
||||
@@ -0,0 +1,730 @@
|
||||
import torch
|
||||
import re
|
||||
import numpy as np
|
||||
from collections import defaultdict
|
||||
import os
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from .tensor_helper import TensorHelper, TensorConfig
|
||||
from verl import DataProto
|
||||
from verl.utils.tracking import Tracking
|
||||
import shutil
|
||||
import requests
|
||||
from transformers.image_processing_base import BatchFeature
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
import json
|
||||
from .qwen_tool_call import Qwen_agent
|
||||
import concurrent.futures
|
||||
|
||||
|
||||
def process_image(image, max_pixels: int = 2048 * 2048, min_pixels: int = 512 * 512):
|
||||
import math
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
if isinstance(image, dict):
|
||||
image = Image.open(BytesIO(image['bytes']))
|
||||
elif isinstance(image, str):
|
||||
image = Image.open(image)
|
||||
|
||||
|
||||
if (image.width * image.height) > max_pixels:
|
||||
resize_factor = math.sqrt(max_pixels / (image.width * image.height))
|
||||
width, height = int(image.width * resize_factor), int(image.height * resize_factor)
|
||||
image = image.resize((width, height))
|
||||
|
||||
if (image.width * image.height) < min_pixels:
|
||||
resize_factor = math.sqrt(min_pixels / (image.width * image.height))
|
||||
width, height = int(image.width * resize_factor), int(image.height * resize_factor)
|
||||
image = image.resize((width, height))
|
||||
|
||||
if image.mode != 'RGB':
|
||||
image = image.convert('RGB')
|
||||
|
||||
return image
|
||||
|
||||
@dataclass
|
||||
class GenerationConfig:
|
||||
max_turns: int
|
||||
max_start_length: int
|
||||
max_prompt_length: int
|
||||
max_response_length: int
|
||||
max_obs_length: int
|
||||
# logging: dict
|
||||
num_gpus: int
|
||||
no_think_rl: bool=False
|
||||
search_url: str = None
|
||||
topk: int = 3
|
||||
|
||||
class LLMGenerationManager:
|
||||
def __init__(
|
||||
self,
|
||||
processor,
|
||||
actor_rollout_wg,
|
||||
config: GenerationConfig,
|
||||
# logger: Tracking,
|
||||
is_validation: bool = False,
|
||||
text_input: bool=False
|
||||
):
|
||||
self.processor = processor
|
||||
self.tokenizer = processor.tokenizer
|
||||
self.actor_rollout_wg = actor_rollout_wg
|
||||
self.config = config
|
||||
# self.logger = logger
|
||||
self.is_validation = is_validation
|
||||
self.text_input = text_input
|
||||
|
||||
self.tensor_fn = TensorHelper(TensorConfig(
|
||||
pad_token_id=self.tokenizer.pad_token_id,
|
||||
max_prompt_length=config.max_prompt_length,
|
||||
max_obs_length=config.max_obs_length,
|
||||
max_start_length=config.max_start_length
|
||||
))
|
||||
self.qwen_agent = Qwen_agent(function_list=['web_search','VLSearchImage','visit'])
|
||||
|
||||
def _batch_tokenize(self, responses: List[str]) -> torch.Tensor:
|
||||
"""Tokenize a batch of responses."""
|
||||
return self.tokenizer(
|
||||
responses,
|
||||
add_special_tokens=False,
|
||||
return_tensors='pt',
|
||||
padding="longest"
|
||||
)['input_ids']
|
||||
|
||||
def _postprocess_responses_first(self,batch):
|
||||
|
||||
responses_str = self.tokenizer.batch_decode(batch.batch['input_ids'], skip_special_tokens=True)
|
||||
responses_str = ["<search>"+item.split('Question: ')[1].split(' \n\nassistant\n')[0]+"</search>" for item in responses_str]
|
||||
|
||||
responses = self._batch_tokenize(responses_str)
|
||||
return responses, responses_str
|
||||
|
||||
|
||||
def _postprocess_responses(self, responses: torch.Tensor) -> torch.Tensor:
|
||||
"""Process responses to stop at search operation or answer operation."""
|
||||
|
||||
responses_str = self.tokenizer.batch_decode(
|
||||
responses,
|
||||
skip_special_tokens=True
|
||||
)
|
||||
def extract_tags(text):
|
||||
# 定义正则表达式，匹配 <answer>...</answer>、<search>...</search> 和 <think>...</think>
|
||||
pattern = r"<(answer|think|tool_call)>(.*?)</\1>"
|
||||
# 使用 findall 方法找到所有匹配的内容
|
||||
matches = re.findall(pattern, text, re.DOTALL)
|
||||
# 将匹配的内容重新组合成字符串
|
||||
result = "\n".join([f"<{tag}>{content}</{tag}>" for tag, content in matches])
|
||||
return result
|
||||
|
||||
responses_str = [extract_tags(resp) + self.tokenizer.eos_token for resp in responses_str]
|
||||
|
||||
if self.config.no_think_rl:
|
||||
raise ValueError('stop')
|
||||
# if no_think_rl is enabled, only keep action in the str
|
||||
actions, _ = self.env.postprocess_predictions(responses_str)
|
||||
responses_str=[f"<answer>{envs[idx].ACTION_LOOKUP[action]}</answer>" for idx, action in enumerate(actions)]
|
||||
print("RESPONSES:", responses_str)
|
||||
responses = self._batch_tokenize(responses_str)
|
||||
return responses, responses_str
|
||||
|
||||
def _process_next_obs(self, next_obs: List, rollings) -> torch.Tensor:
|
||||
"""Process next observations from environment."""
|
||||
# len([item for item in next_obs if isinstance(item, dict) and item['tool'] not in ['VLSearchImage','VLSearchText','web_search']])
|
||||
next_obs_str = []
|
||||
multi_modal_data = []
|
||||
multi_modal_inputs = []
|
||||
merge_length = self.processor.image_processor.merge_size**2
|
||||
# print(self.retrievaled_images)
|
||||
for idx, obs_item in enumerate(next_obs):
|
||||
if isinstance(obs_item,str):
|
||||
next_obs_str.append(obs_item)
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
elif obs_item['status'] == False:
|
||||
next_obs_str.append('\n<|im_start|>user\nYour previous action is invalid.\n<|im_end|>\n<|im_start|>assistant\n')
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
elif obs_item['status'] == True:
|
||||
text = False
|
||||
if text:
|
||||
results = obs_item['result']
|
||||
obs_str = '\n<|im_start|>user\n' + results + '<|im_end|>\n<|im_start|>assistant\n'
|
||||
next_obs_str.append(obs_str)
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
else:
|
||||
if obs_item['tool'] in ['VLSearchImage']:
|
||||
search_results = obs_item['result']
|
||||
images_path = re.findall(r"Image: (.*?), Text:", search_results)
|
||||
text_description = re.findall(r"Text: (.*?)\nImage:", search_results)
|
||||
text_description_last = re.findall(r"Text: (.+)$", search_results)
|
||||
text_description_list = text_description + text_description_last
|
||||
# images_path = images_path[:5]
|
||||
images_path = []
|
||||
if len(images_path)>0:
|
||||
raw_images_list = [process_image(image, 512*28*28, 2*28*28) for image in images_path]
|
||||
multi_modal_data.append({'image': raw_images_list})
|
||||
image_inputs = self.processor.image_processor(raw_images_list, return_tensors='pt')
|
||||
multi_modal_inputs.append(image_inputs)
|
||||
image_grid_thw = image_inputs['image_grid_thw']
|
||||
|
||||
obs_str = ''.join([f"Input image:<|vision_start|>{self.processor.image_token * (image_grid_thw_item.prod() // merge_length)}<|vision_end|>\nDescription: {description}\n" for image_grid_thw_item,description in zip(image_grid_thw, text_description_list)])
|
||||
# raw_obs_str = f"<|vision_start|>{self.processor.image_token}<|vision_end|>" * len(image_grid_thw)
|
||||
obs_str = '\n<|im_start|>user\n<tool_response>\nContents of retrieved images: \n' + obs_str + '</tool_response><|im_end|>\n<|im_start|>assistant\n'
|
||||
next_obs_str.append(obs_str)
|
||||
else:
|
||||
# no image
|
||||
obs_str = '\n<|im_start|>user\n<tool_response>\nContents of retrieved images: \n' + search_results + '\n</tool_response><|im_end|>\n<|im_start|>assistant\n'
|
||||
next_obs_str.append(obs_str)
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
elif obs_item['tool'] in ['web_search','visit']:
|
||||
results = obs_item['result']
|
||||
obs_str = '\n<|im_start|>user\n<tool_response>\n' + results + '\n</tool_response><|im_end|>\n<|im_start|>assistant\n'
|
||||
next_obs_str.append(obs_str)
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
else:
|
||||
next_obs_str.append('\n<|im_start|>user\nYour previous action is invalid.\n<|im_end|>\n<|im_start|>assistant\n')
|
||||
multi_modal_data.append({'image': []})
|
||||
multi_modal_inputs.append(BatchFeature(dict()))
|
||||
else:
|
||||
raise ValueError('invalid observation')
|
||||
|
||||
|
||||
next_obs_ids = self.tokenizer(
|
||||
next_obs_str,
|
||||
padding='longest',
|
||||
return_tensors='pt',
|
||||
add_special_tokens=False, # Prevents adding special tokens
|
||||
)['input_ids']
|
||||
|
||||
return next_obs_ids, next_obs_str, multi_modal_data, multi_modal_inputs
|
||||
|
||||
def _concat_multi_modal_data(self, rollings, next_obs_multi_modal_data:list, next_obs_multi_modal_inputs:list):
|
||||
if not 'multi_modal_inputs' in rollings.non_tensor_batch.keys():
|
||||
rollings.non_tensor_batch['multi_modal_inputs'] = np.empty(len(next_obs_multi_modal_data), dtype=object)
|
||||
for idx, item in enumerate(next_obs_multi_modal_inputs):
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx] = item
|
||||
rollings.non_tensor_batch['multi_modal_data'] = np.array(next_obs_multi_modal_data, dtype=object)
|
||||
else:
|
||||
for idx, multi_modal_data_item in enumerate(next_obs_multi_modal_data):
|
||||
if len(multi_modal_data_item['image']) > 0:
|
||||
# data
|
||||
rollings.non_tensor_batch['multi_modal_data'][idx]['image'].extend(multi_modal_data_item['image'])
|
||||
if 'pixel_values' in rollings.non_tensor_batch['multi_modal_inputs'][idx]:
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx]['pixel_values'] = torch.cat((rollings.non_tensor_batch['multi_modal_inputs'][idx]['pixel_values'], next_obs_multi_modal_inputs[idx]['pixel_values']),dim=0)
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx]['image_grid_thw'] = torch.cat((rollings.non_tensor_batch['multi_modal_inputs'][idx]['image_grid_thw'], next_obs_multi_modal_inputs[idx]['image_grid_thw']),dim=0)
|
||||
else:
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx]['pixel_values'] = next_obs_multi_modal_inputs[idx]['pixel_values']
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx]['image_grid_thw'] = next_obs_multi_modal_inputs[idx]['image_grid_thw']
|
||||
else:
|
||||
pass
|
||||
|
||||
return rollings
|
||||
|
||||
|
||||
def _update_rolling_state(self, rollings, cur_responses: torch.Tensor,
|
||||
next_obs_ids: torch.Tensor) -> Dict:
|
||||
"""Update rolling state with new responses and observations."""
|
||||
# Concatenate and handle padding
|
||||
if next_obs_ids.shape[1] != 0:
|
||||
new_input_ids = self.tensor_fn.concatenate_with_padding([
|
||||
rollings.batch['input_ids'],
|
||||
cur_responses,
|
||||
next_obs_ids
|
||||
])
|
||||
else:
|
||||
new_input_ids = self.tensor_fn.concatenate_with_padding([
|
||||
rollings.batch['input_ids'],
|
||||
cur_responses
|
||||
])
|
||||
# Create attention mask and position ids
|
||||
new_attention_mask = self.tensor_fn.create_attention_mask(new_input_ids)
|
||||
new_position_ids = self.tensor_fn.create_position_ids(new_attention_mask)
|
||||
|
||||
# Cut to appropriate length
|
||||
effective_len = new_attention_mask.sum(dim=1).max()
|
||||
max_len = min(self.config.max_prompt_length, effective_len)
|
||||
|
||||
return DataProto.from_dict({
|
||||
'input_ids': new_input_ids[:, -max_len:],
|
||||
'position_ids': new_position_ids[:, -max_len:],
|
||||
'attention_mask': new_attention_mask[:, -max_len:]
|
||||
}, rollings.non_tensor_batch)
|
||||
|
||||
def _update_right_side(self, right_side: Dict,
|
||||
cur_responses: torch.Tensor,
|
||||
next_obs_ids: torch.Tensor = None) -> Dict:
|
||||
"""Update right side state."""
|
||||
if next_obs_ids != None and next_obs_ids.shape[1] != 0:
|
||||
responses = self.tensor_fn.concatenate_with_padding([
|
||||
right_side['responses'],
|
||||
cur_responses,
|
||||
next_obs_ids
|
||||
], pad_to_left=False)
|
||||
else:
|
||||
responses = self.tensor_fn.concatenate_with_padding([
|
||||
right_side['responses'],
|
||||
cur_responses,
|
||||
], pad_to_left=False)
|
||||
|
||||
effective_len = self.tensor_fn.create_attention_mask(responses).sum(dim=1).max()
|
||||
max_len = min(self.config.max_prompt_length, effective_len)
|
||||
|
||||
return {'responses': responses[:, :max_len]}
|
||||
|
||||
|
||||
def _generate_with_gpu_padding(self, active_batch: DataProto) -> DataProto:
|
||||
"""
|
||||
Wrapper for generation that handles multi-GPU padding requirements.
|
||||
if num_gpus <= 1, return self.actor_rollout_wg.generate_sequences(active_batch)
|
||||
if active_batch size is not divisible by num_gpus, pad with first sequence
|
||||
then remove padding from output
|
||||
"""
|
||||
num_gpus = self.config.num_gpus
|
||||
if num_gpus <= 1:
|
||||
return self.actor_rollout_wg.generate_sequences(active_batch)
|
||||
|
||||
batch_size = active_batch.batch['input_ids'].shape[0]
|
||||
remainder = batch_size % num_gpus
|
||||
|
||||
if remainder == 0:
|
||||
return self.actor_rollout_wg.generate_sequences(active_batch)
|
||||
|
||||
# Add padding sequences
|
||||
padding_size = num_gpus - remainder
|
||||
padded_batch = {}
|
||||
padded_non_tensor_batch = {}
|
||||
|
||||
padded_ids = self.tokenizer(
|
||||
['<|im_start|>user\nHi, who are u?<|im_end|>\n<|im_start|>assistant\n'],
|
||||
padding='longest',
|
||||
return_tensors='pt',
|
||||
add_special_tokens=False, # Prevents adding special tokens
|
||||
)['input_ids']
|
||||
padded_ids = padded_ids[0]
|
||||
|
||||
pad_input_ids = torch.full_like(active_batch.batch['input_ids'][0], 151643, dtype=torch.int64)
|
||||
pad_input_ids[:len(padded_ids)] = padded_ids
|
||||
pad_attention_mask = self.tensor_fn.create_attention_mask(pad_input_ids)
|
||||
pad_input_ids = pad_input_ids.unsqueeze(0)
|
||||
pad_attention_mask = pad_attention_mask.unsqueeze(0)
|
||||
pad_position_ids = self.tensor_fn.create_position_ids(pad_attention_mask)
|
||||
|
||||
padded_batch['attention_mask'] = torch.cat([active_batch.batch['attention_mask'], pad_attention_mask.repeat(padding_size, *[1] * (len(active_batch.batch['attention_mask'].shape) - 1))], dim=0)
|
||||
padded_batch['input_ids'] = torch.cat([active_batch.batch['input_ids'], pad_input_ids.repeat(padding_size, *[1] * (len(active_batch.batch['input_ids'].shape) - 1))], dim=0)
|
||||
padded_batch['position_ids'] = torch.cat([active_batch.batch['position_ids'], pad_position_ids.repeat(padding_size, *[1] * (len(active_batch.batch['position_ids'].shape) - 1))], dim=0)
|
||||
|
||||
# for k, v in active_batch.batch.items():
|
||||
# # Use first sequence as padding template
|
||||
# pad_sequence = v[0:1].repeat(padding_size, *[1] * (len(v.shape) - 1))
|
||||
# padded_batch[k] = torch.cat([v, pad_sequence], dim=0)
|
||||
|
||||
|
||||
# if len(active_batch.non_tensor_batch['multi_modal_inputs'].shape)>1 and active_batch.non_tensor_batch['multi_modal_inputs'].shape[1] == 0:
|
||||
# active_batch.non_tensor_batch.pop('multi_modal_inputs')
|
||||
# active_batch.non_tensor_batch.pop('multi_modal_data')
|
||||
# else:
|
||||
for k, v in active_batch.non_tensor_batch.items():
|
||||
pad_non_tensor_item = np.empty(padding_size, dtype=object)
|
||||
if k == 'raw_prompt_ids':
|
||||
list_ids = padded_ids.tolist()
|
||||
for idx in range(padding_size):
|
||||
pad_non_tensor_item[idx] = list_ids
|
||||
elif k == 'multi_modal_inputs':
|
||||
for idx in range(padding_size):
|
||||
pad_non_tensor_item[idx] = {}
|
||||
elif k == 'multi_modal_data':
|
||||
for idx in range(padding_size):
|
||||
pad_non_tensor_item[idx] = {'image': []}
|
||||
padded_non_tensor_batch[k] = np.concatenate([v, pad_non_tensor_item])
|
||||
|
||||
padded_active_batch = DataProto.from_dict(padded_batch, padded_non_tensor_batch)
|
||||
|
||||
# padded_active_batch = DataProto.from_dict(padded_batch, active_batch.non_tensor_batch)
|
||||
|
||||
# Generate with padded batch
|
||||
padded_output = self.actor_rollout_wg.generate_sequences(padded_active_batch)
|
||||
|
||||
# Remove padding from output
|
||||
trimmed_batch = {k: v[:-padding_size] for k, v in padded_output.batch.items()}
|
||||
|
||||
# Handle meta_info if present
|
||||
if hasattr(padded_output, 'meta_info') and padded_output.meta_info:
|
||||
trimmed_meta = {}
|
||||
for k, v in padded_output.meta_info.items():
|
||||
if isinstance(v, torch.Tensor):
|
||||
trimmed_meta[k] = v[:-padding_size]
|
||||
else:
|
||||
trimmed_meta[k] = v
|
||||
padded_output.meta_info = trimmed_meta
|
||||
|
||||
padded_output.batch = trimmed_batch
|
||||
return padded_output
|
||||
|
||||
def _raw_prompt_ids(self, rollings):
|
||||
new_raw_prompt_ids = []
|
||||
rollings.batch['input_ids'] = rollings.batch['input_ids'].long()
|
||||
raw_next_obs_ids = [ids[mask == 1].tolist() for ids, mask in zip(np.array(rollings.batch['input_ids']), np.array(rollings.batch['attention_mask']))]
|
||||
def replace_consecutive_elements(arr, target):
|
||||
result = []
|
||||
i = 0
|
||||
while i < len(arr):
|
||||
if arr[i] == target:
|
||||
result.append(target)
|
||||
while i + 1 < len(arr) and arr[i + 1] == target:
|
||||
i += 1
|
||||
else:
|
||||
result.append(arr[i])
|
||||
i += 1
|
||||
return result
|
||||
raw_next_obs_ids = [replace_consecutive_elements(row,151655) for row in raw_next_obs_ids]
|
||||
raw_next_obs_ids = np.array(raw_next_obs_ids, dtype=object)
|
||||
rollings.non_tensor_batch['raw_prompt_ids'] = raw_next_obs_ids
|
||||
return rollings
|
||||
|
||||
def deactivate_batch(self, active_mask,rollings):
|
||||
raw_prompt_ids = rollings.non_tensor_batch['raw_prompt_ids']
|
||||
max_model_len = 22048
|
||||
curr_active_mask = torch.tensor([len(raw_prompt_ids_item) < max_model_len for raw_prompt_ids_item in raw_prompt_ids], dtype=torch.bool)
|
||||
active_mask = active_mask * curr_active_mask
|
||||
return active_mask
|
||||
|
||||
def run_llm_loop(self, gen_batch, initial_input_ids: torch.Tensor) -> Tuple[Dict, Dict]:
|
||||
"""Run main LLM generation loop."""
|
||||
|
||||
original_left_side = {'input_ids': initial_input_ids[:, -self.config.max_start_length:]}
|
||||
original_right_side = {'responses': initial_input_ids[:, []]}
|
||||
|
||||
active_mask = torch.ones(gen_batch.batch['input_ids'].shape[0], dtype=torch.bool)
|
||||
active_num_list = [active_mask.sum().item()]
|
||||
rollings = gen_batch
|
||||
# rollings_multimodal_data = gen_batch.non_tensor_batch.get('multi_modal_inputs', None)
|
||||
# rollings_multimodal_data = gen_batch.non_tensor_batch['multi_modal_inputs']
|
||||
# rollings_multimodal_data = None
|
||||
raw_prompt_ids = rollings.non_tensor_batch['raw_prompt_ids']
|
||||
|
||||
self.retrievaled_images = [[] for _ in range(gen_batch.batch['input_ids'].shape[0])]
|
||||
|
||||
# Main generation loop
|
||||
for step in range(self.config.max_turns):
|
||||
if not active_mask.sum():
|
||||
break
|
||||
rollings.batch = self.tensor_fn.cut_to_effective_len(
|
||||
rollings.batch,
|
||||
keys=['input_ids', 'attention_mask', 'position_ids']
|
||||
)
|
||||
|
||||
rollings = self._raw_prompt_ids(rollings)
|
||||
|
||||
active_mask = self.deactivate_batch(active_mask, rollings)
|
||||
if not active_mask.sum():
|
||||
break
|
||||
|
||||
if 'multi_modal_inputs' in rollings.non_tensor_batch.keys():
|
||||
rollings_active = DataProto.from_dict(
|
||||
tensors={k: v[active_mask] for k, v in rollings.batch.items()},
|
||||
non_tensors={k: v[active_mask] for k, v in rollings.non_tensor_batch.items()}
|
||||
)
|
||||
else:
|
||||
rollings_active = DataProto.from_dict({
|
||||
k: v[active_mask] for k, v in rollings.batch.items()
|
||||
})
|
||||
|
||||
# self.processor.batch_decode(rollings_active.batch['input_ids'])
|
||||
gen_output = self._generate_with_gpu_padding(rollings_active)
|
||||
|
||||
meta_info = gen_output.meta_info
|
||||
responses_ids, responses_str = self._postprocess_responses(gen_output.batch['responses'])
|
||||
|
||||
print(responses_str[0])
|
||||
|
||||
responses_ids, responses_str = self.tensor_fn._example_level_pad(responses_ids, responses_str, active_mask)
|
||||
# Execute in environment and process observations
|
||||
next_obs, dones = self.execute_predictions(responses_str, self.tokenizer.pad_token, active_mask)
|
||||
curr_active_mask = torch.tensor([not done for done in dones], dtype=torch.bool)
|
||||
active_mask = active_mask * curr_active_mask
|
||||
active_num_list.append(active_mask.sum().item())
|
||||
next_obs_ids, next_obs_str, next_obs_multi_modal_data, next_obs_multi_modal_inputs = self._process_next_obs(next_obs, rollings)
|
||||
rollings = self._concat_multi_modal_data(
|
||||
rollings,
|
||||
next_obs_multi_modal_data,
|
||||
next_obs_multi_modal_inputs
|
||||
)
|
||||
# Update states
|
||||
rollings = self._update_rolling_state(
|
||||
rollings,
|
||||
responses_ids,
|
||||
next_obs_ids
|
||||
)
|
||||
original_right_side = self._update_right_side(
|
||||
original_right_side,
|
||||
responses_ids,
|
||||
next_obs_ids
|
||||
)
|
||||
|
||||
|
||||
# final LLM rollout
|
||||
if active_mask.sum():
|
||||
|
||||
rollings.batch = self.tensor_fn.cut_to_effective_len(
|
||||
rollings.batch,
|
||||
keys=['input_ids', 'attention_mask', 'position_ids']
|
||||
)
|
||||
|
||||
rollings = self._raw_prompt_ids(rollings)
|
||||
|
||||
active_mask = self.deactivate_batch(active_mask, rollings)
|
||||
|
||||
if active_mask.sum():
|
||||
|
||||
if 'multi_modal_inputs' in rollings.non_tensor_batch.keys():
|
||||
rollings_active = DataProto.from_dict(
|
||||
tensors={k: v[active_mask] for k, v in rollings.batch.items()},
|
||||
non_tensors={k: v[active_mask] for k, v in rollings.non_tensor_batch.items()}
|
||||
)
|
||||
else:
|
||||
rollings_active = DataProto.from_dict({
|
||||
k: v[active_mask] for k, v in rollings.batch.items()
|
||||
})
|
||||
|
||||
gen_output = self._generate_with_gpu_padding(rollings_active)
|
||||
|
||||
meta_info = gen_output.meta_info
|
||||
responses_ids, responses_str = self._postprocess_responses(gen_output.batch['responses'])
|
||||
responses_ids, responses_str = self.tensor_fn._example_level_pad(responses_ids, responses_str, active_mask)
|
||||
|
||||
# # Execute in environment and process observations
|
||||
_, dones = self.execute_predictions(
|
||||
responses_str, self.tokenizer.pad_token, active_mask, do_search=False
|
||||
)
|
||||
|
||||
curr_active_mask = torch.tensor([not done for done in dones], dtype=torch.bool)
|
||||
active_mask = active_mask * curr_active_mask
|
||||
active_num_list.append(active_mask.sum().item())
|
||||
|
||||
original_right_side = self._update_right_side(
|
||||
original_right_side,
|
||||
responses_ids,
|
||||
)
|
||||
|
||||
print("ACTIVE_TRAJ_NUM:", active_num_list)
|
||||
|
||||
# =================== raw prompt ids ===================
|
||||
rollings.non_tensor_batch['raw_prompt_ids'] = raw_prompt_ids
|
||||
|
||||
if not self.is_validation:
|
||||
rollings, original_right_side = self._add_noisy_multi_modal_data(rollings, original_right_side)
|
||||
|
||||
return self._compose_final_output(original_left_side, original_right_side, meta_info, rollings)
|
||||
|
||||
def _add_noisy_multi_modal_data(self, rollings, original_right_side):
|
||||
# from ray.util import pdb
|
||||
# pdb.set_trace()
|
||||
image_padded = Image.new('RGB', (64, 64), (0, 0, 0))
|
||||
|
||||
image_padded = process_image(image_padded, 256*256, 128*128)
|
||||
image_inputs = self.processor.image_processor([image_padded], return_tensors='pt')
|
||||
image_grid_thw = image_inputs['image_grid_thw']
|
||||
merge_length = self.processor.image_processor.merge_size**2
|
||||
padded_str = f"\n<|im_start|>user\n<|vision_start|>{self.processor.image_token * (image_grid_thw.prod() // merge_length)}<|vision_end|><|im_end|>"
|
||||
|
||||
padded_str_list = []
|
||||
for idx, multi_modal_item in enumerate(rollings.non_tensor_batch['multi_modal_data']):
|
||||
if len(multi_modal_item['image']) == 0:
|
||||
padded_str_list.append(padded_str)
|
||||
rollings.non_tensor_batch['multi_modal_data'][idx]['image'].append(image_padded)
|
||||
rollings.non_tensor_batch['multi_modal_inputs'][idx] = image_inputs
|
||||
else:
|
||||
padded_str_list.append('')
|
||||
|
||||
padded_ids = self.tokenizer(
|
||||
padded_str_list,
|
||||
padding='longest',
|
||||
return_tensors='pt',
|
||||
add_special_tokens=False, # Prevents adding special tokens
|
||||
)['input_ids']
|
||||
|
||||
original_right_side = self._update_right_side(
|
||||
original_right_side,
|
||||
padded_ids
|
||||
)
|
||||
return rollings, original_right_side
|
||||
|
||||
|
||||
def _compose_final_output(self, left_side: Dict,
|
||||
right_side: Dict,
|
||||
meta_info: Dict,
|
||||
rollings) -> Tuple[Dict, Dict]:
|
||||
"""Compose final generation output."""
|
||||
final_output = right_side.copy()
|
||||
final_output['prompts'] = left_side['input_ids']
|
||||
|
||||
# Combine input IDs
|
||||
final_output['input_ids'] = torch.cat([
|
||||
left_side['input_ids'],
|
||||
right_side['responses']
|
||||
], dim=1)
|
||||
|
||||
# Create attention mask and position ids
|
||||
final_output['attention_mask'] = torch.cat([
|
||||
self.tensor_fn.create_attention_mask(left_side['input_ids']),
|
||||
self.tensor_fn.create_attention_mask(final_output['responses'])
|
||||
], dim=1)
|
||||
|
||||
final_output['position_ids'] = self.tensor_fn.create_position_ids(
|
||||
final_output['attention_mask']
|
||||
)
|
||||
|
||||
final_output = DataProto.from_dict(final_output,rollings.non_tensor_batch)
|
||||
final_output.meta_info.update(meta_info)
|
||||
|
||||
return final_output
|
||||
|
||||
def execute_predictions(self, predictions: List[str], pad_token: str, active_mask=None, do_search=True) -> List[str]:
|
||||
"""
|
||||
Execute predictions across multiple environments.
|
||||
NOTE: the function is the actual `step` function in the environment
|
||||
NOTE penalty_for_invalid is not included in observation shown to the LLM
|
||||
|
||||
Args:
|
||||
envs: List of environment instances
|
||||
predictions: List of action predictions
|
||||
pad_token: Token to use for padding
|
||||
|
||||
Returns:
|
||||
List of observation strings
|
||||
"""
|
||||
cur_actions, contents = self.postprocess_predictions(predictions)
|
||||
next_obs, dones = [], []
|
||||
|
||||
tool_queries = [content for action, content in zip(cur_actions, contents) if action == 'tool_call']
|
||||
|
||||
# qwen-agent
|
||||
search_results = []
|
||||
if len(tool_queries) > 0:
|
||||
def tool_call(request_para):
|
||||
try:
|
||||
request_para = json.loads(request_para)
|
||||
# reuslt = self.qwen_agent._call_tool(request_para['name'], request_para['arguments'],img_save_path='/mnt/data/qiuchen.wqc/code/ds_verl/images_data',byte=False)
|
||||
reuslt = self.qwen_agent._call_tool(request_para['name'], request_para['arguments'])
|
||||
search_result = dict(
|
||||
status=True,
|
||||
tool=request_para['name'],
|
||||
tool_arguments=request_para['arguments'],
|
||||
result=reuslt
|
||||
)
|
||||
return search_result
|
||||
except Exception as e:
|
||||
return dict(status=False)
|
||||
|
||||
max_workers = min(128, len(tool_queries))
|
||||
if max_workers == 1:
|
||||
# single thread
|
||||
search_results = []
|
||||
for request_para in tool_queries:
|
||||
search_results.append(tool_call(request_para))
|
||||
else:
|
||||
# multi_thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=128) as executor:
|
||||
futures = {executor.submit(tool_call, query): i for i, query in enumerate(tool_queries)}
|
||||
search_results = [None] * len(tool_queries)
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
index = futures[future] # 获取原始顺序
|
||||
result = future.result() # 获取任务结果
|
||||
search_results[index] = result # 将结果放入对应位置
|
||||
|
||||
for i, (action, active) in enumerate(zip(cur_actions, active_mask)):
|
||||
|
||||
if not active:
|
||||
next_obs.append('')
|
||||
dones.append(1)
|
||||
else:
|
||||
if action == 'answer':
|
||||
next_obs.append('')
|
||||
dones.append(1)
|
||||
elif action == 'tool_call':
|
||||
obs_dict = search_results.pop(0)
|
||||
try:
|
||||
if obs_dict['status']:
|
||||
next_obs.append(obs_dict)
|
||||
else:
|
||||
raise Exception('Tool call failed')
|
||||
except Exception as e:
|
||||
next_obs.append('\n<|im_start|>user\nThe tool is error.\n<|im_end|>\n<|im_start|>assistant\n')
|
||||
dones.append(0)
|
||||
else:
|
||||
next_obs.append('\n<|im_start|>user\nYour previous action is invalid.\n<|im_end|>\n<|im_start|>assistant\n')
|
||||
dones.append(0)
|
||||
|
||||
assert len(search_results) == 0
|
||||
return next_obs, dones
|
||||
|
||||
def postprocess_predictions(self, predictions: List[Any]) -> Tuple[List[int], List[bool]]:
|
||||
"""
|
||||
Process (text-based) predictions from llm into actions and validity flags.
|
||||
|
||||
Args:
|
||||
predictions: List of raw predictions
|
||||
|
||||
Returns:
|
||||
Tuple of (actions list, validity flags list)
|
||||
"""
|
||||
actions = []
|
||||
contents = []
|
||||
|
||||
for prediction in predictions:
|
||||
if isinstance(prediction, str): # for llm output
|
||||
pattern = r'<(tool_call|answer)>(.*?)</\1>'
|
||||
match = re.search(pattern, prediction, re.DOTALL)
|
||||
if match:
|
||||
content = match.group(2).strip() # Return only the content inside the tags
|
||||
action = match.group(1)
|
||||
else:
|
||||
content = ''
|
||||
action = None
|
||||
else:
|
||||
raise ValueError(f"Invalid prediction type: {type(prediction)}")
|
||||
|
||||
actions.append(action)
|
||||
contents.append(content)
|
||||
|
||||
return actions, contents
|
||||
|
||||
def batch_search(self, queries: List[str] = None) -> str:
|
||||
"""
|
||||
Batchified search for queries.
|
||||
Args:
|
||||
queries: queries to call the search engine
|
||||
Returns:
|
||||
search results which is concatenated into a string
|
||||
"""
|
||||
# results = self._batch_search(queries)['result']
|
||||
|
||||
# return [self._passages2string(result) for result in results]
|
||||
response = requests.get(self.config.search_url, params={"queries": queries})
|
||||
response_json = response.json()
|
||||
return response_json
|
||||
|
||||
def _batch_search(self, queries):
|
||||
|
||||
payload = {
|
||||
"queries": queries,
|
||||
"topk": self.config.topk,
|
||||
"return_scores": True
|
||||
}
|
||||
|
||||
return requests.post(self.config.search_url, json=payload).json()
|
||||
|
||||
def _passages2string(self, retrieval_result):
|
||||
format_reference = ''
|
||||
for idx, doc_item in enumerate(retrieval_result):
|
||||
|
||||
content = doc_item['document']['contents']
|
||||
title = content.split("\n")[0]
|
||||
text = "\n".join(content.split("\n")[1:])
|
||||
format_reference += f"Doc {idx+1}(Title: {title}) {text}\n"
|
||||
|
||||
return format_reference
|
||||
@@ -0,0 +1,60 @@
|
||||
import os
|
||||
os.environ["QWEN_SEARCH_ENABLE_CSI"] = "false"
|
||||
os.environ["QWEN_IDP_ENABLE_CSI"] = "false"
|
||||
os.environ["SPECIAL_CODE_MODE"] = "false"
|
||||
os.environ["QWEN_DOC_PARSER_USE_IDP"] = "false"
|
||||
import copy
|
||||
import json
|
||||
from typing import Dict, Iterator, List, Literal, Optional, Union
|
||||
from qwen_agent import Agent
|
||||
from qwen_agent.llm import BaseChatModel
|
||||
from qwen_agent.llm.schema import DEFAULT_SYSTEM_MESSAGE, FUNCTION, Message
|
||||
from qwen_agent.memory import Memory
|
||||
from qwen_agent.settings import MAX_LLM_CALL_PER_RUN
|
||||
from qwen_agent.tools import BaseTool
|
||||
from qwen_agent.utils.utils import extract_files_from_messages
|
||||
|
||||
|
||||
class Qwen_agent(Agent):
|
||||
"""This is a widely applicable function call agent integrated with llm and tool use ability."""
|
||||
|
||||
def __init__(self,
|
||||
function_list: Optional[List[Union[str, Dict, BaseTool]]] = None,
|
||||
llm: Optional[Union[Dict, BaseChatModel]] = None,
|
||||
system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
|
||||
name: Optional[str] = None,
|
||||
description: Optional[str] = None,
|
||||
files: Optional[List[str]] = None,
|
||||
**kwargs):
|
||||
super().__init__(function_list=function_list,
|
||||
llm=llm,
|
||||
system_message=system_message,
|
||||
name=name,
|
||||
description=description)
|
||||
|
||||
|
||||
def _run(self, messages: List[Message], lang: Literal['en', 'zh'] = 'en', **kwargs):
|
||||
return
|
||||
|
||||
def _call_tool(self, tool_name: str, tool_args: Union[str, dict] = '{}', **kwargs) -> str:
|
||||
if tool_name not in self.function_map:
|
||||
return f'Tool {tool_name} does not exists.'
|
||||
# Temporary plan: Check if it is necessary to transfer files to the tool
|
||||
# Todo: This should be changed to parameter passing, and the file URL should be determined by the model
|
||||
if self.function_map[tool_name].file_access:
|
||||
assert 'messages' in kwargs
|
||||
files = extract_files_from_messages(kwargs['messages'], include_images=True) + self.mem.system_files
|
||||
return super()._call_tool(tool_name, tool_args, files=files, **kwargs)
|
||||
else:
|
||||
return super()._call_tool(tool_name, tool_args, **kwargs)
|
||||
|
||||
if __name__ == '__main__':
|
||||
agent = Qwen_agent(function_list=['web_search','VLSearchImage','visit',"code_interpreter"]) #,"PythonInterpreter","google_scholar","google_search"
|
||||
# result = agent._call_tool('web_search', '{"queries": ["What is the meaning of life?"]}')
|
||||
# result = agent._call_tool('google_search', '{"queries": ["What is the meaning of life?"]}')
|
||||
result = agent._call_tool('VLSearchImage', {"images": ["https://mitalinlp.oss-cn-hangzhou.aliyuncs.com/rallm/deep_research_vl_image/hle_image/10.jpg"]},user_query="The image is a sample program from the Piet programming language. What does it intend to print? Write your final answer backwards, and convert it to all lowercase characters (even if the print will contain uppercase letters).\n\nFor example \"Cat\" would be:\ntac")
|
||||
# result = agent._call_tool('visit', '{"url": "https://en.wikipedia.org/wiki/Japanese_submarine_I-19", "goal": "What is the meaning of life?"}')
|
||||
# result = agent._call_tool('code_interpreter', {"code": "import numpy as np\nA = np.array([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [-2, -4, -3, -5]])\nB = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0]])\ncontrollability_matrix = np.hstack((B, np.dot(A, B), np.dot(A**2, B)))\nprint(np.linalg.matrix_rank(controllability_matrix))"})
|
||||
# result = agent._call_tool('PythonInterpreter', {"code": "import numpy as np\nA = np.array([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [-2, -4, -3, -5]])\nB = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0]])\ncontrollability_matrix = np.hstack((B, np.dot(A, B), np.dot(A**2, B)))\nprint(np.linalg.matrix_rank(controllability_matrix))"})
|
||||
# result = agent._call_tool('google_scholar', '{"query": ["What is the meaning of life?"]}')
|
||||
print(result)
|
||||
@@ -0,0 +1,75 @@
|
||||
import torch
|
||||
from typing import Dict, Tuple, List
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class TensorConfig:
|
||||
pad_token_id: int
|
||||
max_prompt_length: int
|
||||
max_obs_length: int
|
||||
max_start_length: int
|
||||
|
||||
class TensorHelper:
|
||||
def __init__(self, config: TensorConfig):
|
||||
self.config = config
|
||||
|
||||
def cut_to_effective_len(self, tensor_dict: Dict[str, torch.Tensor],
|
||||
keys: List[str], cut_left: bool = True) -> Dict[str, torch.Tensor]:
|
||||
"""Cut tensors to their effective length based on attention mask."""
|
||||
effective_len = tensor_dict['attention_mask'].sum(dim=1).max()
|
||||
result = tensor_dict.copy()
|
||||
|
||||
for key in keys:
|
||||
if cut_left:
|
||||
result[key] = tensor_dict[key][:, -effective_len:]
|
||||
else:
|
||||
result[key] = tensor_dict[key][:, :effective_len]
|
||||
return result
|
||||
|
||||
def convert_pad_structure(self, tensor: torch.Tensor, pad_to_left: bool = True) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Convert padding structure and return sorted tensor with indices."""
|
||||
mask = tensor != self.config.pad_token_id if pad_to_left else tensor == self.config.pad_token_id
|
||||
sorted_indices = mask.to(torch.int64).argsort(dim=1, stable=True)
|
||||
return tensor.gather(1, sorted_indices), sorted_indices
|
||||
|
||||
def create_attention_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Create attention mask from input ids."""
|
||||
return torch.where(input_ids != self.config.pad_token_id, 1, 0)
|
||||
|
||||
def create_position_ids(self, attention_mask: torch.Tensor) -> torch.Tensor:
|
||||
"""Create position ids from attention mask."""
|
||||
return (torch.cumsum(attention_mask, dim=1) - 1) * attention_mask
|
||||
|
||||
def concatenate_with_padding(self, tensors: List[torch.Tensor],
|
||||
pad_to_left: bool = True) -> torch.Tensor:
|
||||
"""Concatenate tensors and handle padding."""
|
||||
concatenated = torch.cat(tensors, dim=1)
|
||||
padded_tensor, _ = self.convert_pad_structure(concatenated, pad_to_left)
|
||||
return padded_tensor
|
||||
|
||||
def _example_level_pad(self, responses: torch.Tensor,
|
||||
responses_str: List[str],
|
||||
active_mask: torch.Tensor) -> Tuple[torch.Tensor, List[str]]:
|
||||
"""
|
||||
Pad responses for non-active examples with pad tokens.
|
||||
"""
|
||||
assert active_mask.sum() == responses.shape[0]
|
||||
# Create masked responses tensor
|
||||
batch_size = active_mask.shape[0]
|
||||
seq_len = responses.shape[1]
|
||||
padded_responses = torch.full(
|
||||
(batch_size, seq_len), self.config.pad_token_id,
|
||||
dtype=responses.dtype, device=responses.device
|
||||
)
|
||||
padded_responses[active_mask] = responses
|
||||
|
||||
# Create masked response strings
|
||||
padded_responses_str = [""] * batch_size
|
||||
|
||||
s = 0
|
||||
for i, is_active in enumerate(active_mask):
|
||||
if is_active:
|
||||
padded_responses_str[i] = responses_str[s]
|
||||
s += 1
|
||||
|
||||
return padded_responses, padded_responses_str
|
||||
@@ -0,0 +1,148 @@
|
||||
date=$(date +%Y%m%d)
|
||||
|
||||
######################################
|
||||
### 1. 启动 server （后台） ###
|
||||
######################################
|
||||
|
||||
|
||||
PROJECT_NAME=${date}
|
||||
# benchmark='hle'
|
||||
# EXPERIMENT_NAME='1107'
|
||||
# MODEL_PATH=pretrain_model/webwatcher7b
|
||||
# SUMMERY_MODEL_PATH=pretrain_model/qwen2.5_vl_72b
|
||||
benchmark=$1
|
||||
EXPERIMENT_NAME=$2
|
||||
MODEL_PATH=$3
|
||||
SUMMERY_MODEL_PATH=$4
|
||||
|
||||
export IMG_SEARCH_KEY=$5
|
||||
export JINA_API_KEY=$6
|
||||
export TEXT_SEARCH_KEY=$7
|
||||
export ALIBABA_CLOUD_ACCESS_KEY_ID=$8
|
||||
export ALIBABA_CLOUD_ACCESS_KEY_SECRET=$9
|
||||
|
||||
SAVE_PATH=scripts_eval/results/${PROJECT_NAME}_${benchmark}
|
||||
SAVE_FILE=scripts_eval/results/${PROJECT_NAME}_${benchmark}/${EXPERIMENT_NAME}.jsonl
|
||||
|
||||
if [ ! -d "$SAVE_PATH" ]; then
|
||||
echo "目录 $SAVE_PATH 不存在，正在创建..."
|
||||
mkdir -p "$SAVE_PATH"
|
||||
fi
|
||||
|
||||
# search config
|
||||
echo "==== 启动模型 vllm (端口8001)... ===="
|
||||
# vllm serve $MODEL_PATH --port 8001 --host 0.0.0.0 --limit-mm-per-prompt '{"image": 100}' --served-model-name $MODEL_PATH --max-num-batched-tokens 32768 --max-num-seqs 128 --tensor-parallel-size 1 > ${SAVE_PATH}/${EXPERIMENT_NAME}_vllm.log 2>&1 & vllm_pid=$!
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve $MODEL_PATH --port 8001 --host 0.0.0.0 --limit-mm-per-prompt '{"image": 100}' --served-model-name $MODEL_PATH --max-num-batched-tokens 32768 --max-num-seqs 128 --tensor-parallel-size 1 > ${SAVE_PATH}/${EXPERIMENT_NAME}_vllm.log 2>&1 & vllm_pid=$!
|
||||
|
||||
echo "==== 启动summery model vllm (端口6002)... ===="
|
||||
CUDA_VISIBLE_DEVICES=4,5,6,7 vllm serve $SUMMERY_MODEL_PATH --port 6002 --host 0.0.0.0 --served-model-name $SUMMERY_MODEL_PATH --max-num-batched-tokens 32768 --max-num-seqs 128 --tensor-parallel-size 1 & summery_pid=$!
|
||||
|
||||
#####################################
|
||||
### 2. 等待 server 端口 ready ###
|
||||
#####################################
|
||||
|
||||
timeout=120000
|
||||
start_time=$(date +%s)
|
||||
server1_ready=false
|
||||
server2_ready=false
|
||||
|
||||
while true; do
|
||||
if ! $server1_ready && curl -s http://localhost:8001/v1/chat/completions > /dev/null; then
|
||||
echo -e "\nLocal model (port 8001) is ready!"
|
||||
server1_ready=true
|
||||
fi
|
||||
|
||||
# Check Summary Model
|
||||
if ! $server2_ready && curl -s http://localhost:6002/v1/chat/completions > /dev/null; then
|
||||
echo -e "\nSummary model (port 6002) is ready!"
|
||||
server2_ready=true
|
||||
fi
|
||||
|
||||
# If both servers are ready, exit loop
|
||||
if $server1_ready && $server2_ready; then
|
||||
echo "Both servers are ready for inference!"
|
||||
break
|
||||
fi
|
||||
|
||||
current_time=$(date +%s)
|
||||
elapsed=$((current_time - start_time))
|
||||
if [ $elapsed -gt $timeout ]; then
|
||||
echo -e "Warning: Server startup timeout after ${timeout} seconds"
|
||||
if ! $server1_ready; then
|
||||
echo "Vllm server failed to start"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
printf 'Waiting for servers to start .....'
|
||||
sleep 10
|
||||
done
|
||||
|
||||
#####################################
|
||||
### 3. 启动 infer ####
|
||||
#####################################
|
||||
|
||||
echo "==== 启动 infer... ===="
|
||||
|
||||
export VLLM_MODEL=$MODEL_PATH
|
||||
|
||||
if [ "$benchmark" = "mmsearch" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/mmsearch
|
||||
echo "已设置 IMAGE_DIR 为 mmsearch 路径"
|
||||
elif [ "$benchmark" = "hle" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/hle
|
||||
echo "已设置 IMAGE_DIR 为 hle 路径"
|
||||
elif [ "$benchmark" = "livevqa" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/livevqa
|
||||
echo "已设置 IMAGE_DIR 为 livevqa 路径"
|
||||
elif [ "$benchmark" = "infoseek" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/infoseek
|
||||
echo "已设置 IMAGE_DIR 为 infoseek 路径"
|
||||
elif [ "$benchmark" = "simplevqa" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/simplevqa
|
||||
echo "已设置 IMAGE_DIR 为 simplevqa 路径"
|
||||
elif [ "$benchmark" = "gaia" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/gaia
|
||||
echo "已设置 IMAGE_DIR 为 gaia 路径"
|
||||
elif [ "$benchmark" = "bc_vl_v1" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/bc_vl_v1
|
||||
echo "已设置 IMAGE_DIR 为 bc_vl_v1 路径"
|
||||
elif [ "$benchmark" = "bc_vl_v2" ]; then
|
||||
export IMAGE_DIR=scripts_eval/images/bc_vl_v2
|
||||
echo "已设置 IMAGE_DIR 为 bc-vl-v2 路径"
|
||||
else
|
||||
echo "警告: 未知的 benchmark 值 '$benchmark'. 未设置 IMAGE_DIR."
|
||||
fi
|
||||
|
||||
|
||||
pip uninstall qwen-agent
|
||||
pip install -e vl_search_r1/qwen-agent-o1_search --no-deps
|
||||
pip install "qwen-agent[code_interpreter]"
|
||||
|
||||
|
||||
# for i in 1 2 3
|
||||
# do
|
||||
# SAVE_FILE=${SAVE_PATH}/${EXPERIMENT_NAME}_round${i}.jsonl
|
||||
# [ -s "$SAVE_FILE" ] && > "$SAVE_FILE"
|
||||
# python scripts_eval/agent_eval.py \
|
||||
# --output_file $SAVE_FILE \
|
||||
# --eval_data $benchmark
|
||||
# done
|
||||
|
||||
SAVE_FILE=${SAVE_PATH}/${EXPERIMENT_NAME}.jsonl
|
||||
python scripts_eval/agent_eval.py \
|
||||
--output_file $SAVE_FILE \
|
||||
--eval_data $benchmark
|
||||
|
||||
# echo "==== 关闭服务... ===="
|
||||
if kill ${vllm_pid}; then
|
||||
echo "成功关闭VLLM服务 (PID: ${vllm_pid})"
|
||||
else
|
||||
echo "警告：未能关闭VLLM服务 (PID: ${vllm_pid})，可能已被关闭或不存在。"
|
||||
fi
|
||||
|
||||
if kill ${summery_pid}; then
|
||||
echo "成功关闭VLLM服务 (PID: ${summery_pid})"
|
||||
else
|
||||
echo "警告：未能关闭VLLM服务 (PID: ${summery_pid})，可能已被关闭或不存在。"
|
||||
fi
|
||||
Reference in New Issue
Block a user