Files
wehub-resource-sync 2114b14ee0
Sync main into demo / sync (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:35:26 +08:00

556 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Clock app task definitions.
"""
# -- Task Index (auto-generated, do not edit) --
# 18 tasks | L1×2 L2×9 L3×4 L4×3
#
# [L1] ToggleAlarm {toggle}{time}的闹钟
# [L2] CountAlarms 时钟里一共有几个闹钟
# [L2] AddAlarm 帮我设一个{time}的闹钟
# [L2] DeleteAlarm 帮我把{time}的闹钟删掉
# [L2] SetAlarmRepeat 把{time}的闹钟改成{repeat}
# [L2] AddWorldCity 在世界时钟里添加{city}
# [L2] RemoveWorldCity 把{city}从世界时钟里删掉
# [L2] CheckAlarmNote 时钟里{time}的闹钟备注写的什么
# [L4] AddAlarmWithSettings 设一个{time}的闹钟,重复模式{repeat},备注写“{note}”
# [L4] EnableAllAlarms 帮我把时钟里所有闹钟都打开
# [L3] CheckCityTime 帮我看看世界时钟里{city}现在几点
# [L2] CompareCityTimeDiff {city1}和{city2}现在差几个小时
# [L3] CityLocalTimeDiff 世界时钟里{city}比咱们这儿快还是慢,差多久
# [L3] LatestTimezoneCity 世界时钟里的城市,哪个现在时间最晚
# [L2] AddCityAndCheckTime 在世界时钟里加上{city},然后告诉我那边现在几点
# [L1] AddCityAndCompareTimeDiff 把{new_city}加到世界时钟,然后告诉我{new_city}和{existing_city}差几个小时
# [L4] ReorganizeWorldClock 把世界时钟里的{remove_city}删掉,换成{add_city}
# [L3] SetupMorningAlarms 帮我设两个起床闹钟:{time1}的设成{repeat1}{time2}的设成{repeat2}
# -- End Task Index --
from __future__ import annotations
from typing import Any
from bench_env.task.base import BaseTask
from bench_env.task.common_tasks import AnswerTask, CriteriaTask, build_answer_checks
from bench_env.task.judge import JudgeInput
from bench_env.task.clock.app import CLOCK_NOTE_VALUES, CLOCK_REPEAT_VALUES, Clock
class ToggleAlarm(CriteriaTask):
templates = ["{toggle}{time}的闹钟"]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "atomic"
difficulty = "L1"
capabilities = ["edit"]
parameters = {
"alarm_id": {"type": "string", "default": "a1", "description": "闹钟 id"},
"time": {"type": "string", "default": "04:30", "description": "闹钟时间"},
"toggle": {
"type": "bool",
"values": {"打开": True, "关闭": False},
"default": False,
},
"_alarm": {
"sampler": Clock.sample_existing_alarm,
"fields": {"alarm_id": "alarm_id", "time": "time"},
},
}
expected_changes = ["alarms[id={alarm_id}]"]
criteria = {"alarms[id={alarm_id}].enabled": "{toggle}"}
async def _post_sample(self, env):
await env.set_state({"apps": {"clock": {
f"alarms[id={self.p.alarm_id}]": {"enabled": not bool(self.p.toggle)},
}}}, deep=True, reload=False)
class CountAlarms(AnswerTask):
templates = ["时钟里一共有几个闹钟"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "atomic"
difficulty = "L3"
max_steps = 15
capabilities = ["extract"]
answer = (".alarms", len)
answer_fields = [{"type": "number", "label": "闹钟数量"}]
class AddAlarm(BaseTask):
templates = [
"帮我设一个{time}的闹钟",
"Set an alarm for {time}",
]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L2"
capabilities = ["create"]
parameters = {
"time": {"type": "string", "default": "07:10", "description": "闹钟时间"},
"hour": {"type": "int", "default": 7, "description": "小时"},
"minute": {"type": "int", "default": 10, "description": "分钟"},
"_time": {
"sampler": Clock.sample_new_alarm_time,
"fields": {"time": "time", "hour": "hour", "minute": "minute"},
},
}
expected_changes = ["alarms[+1]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [clock.check_created_alarm(self.p.hour, self.p.minute)]
class DeleteAlarm(BaseTask):
templates = ["帮我把{time}的闹钟删掉"]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L2"
capabilities = ["edit"]
parameters = {
"alarm_id": {"type": "string", "default": "a1", "description": "闹钟 id"},
"time": {"type": "string", "default": "04:30", "description": "闹钟时间"},
"_alarm": {
"sampler": Clock.sample_existing_alarm,
"fields": {"alarm_id": "alarm_id", "time": "time"},
},
}
expected_changes = ["alarms[id={alarm_id}]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [clock.check_deleted_alarm(self.p.alarm_id)]
class SetAlarmRepeat(CriteriaTask):
templates = ["把{time}的闹钟改成{repeat}"]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L2"
capabilities = ["edit"]
parameters = {
"alarm_id": {"type": "string", "default": "a1", "description": "闹钟 id"},
"time": {"type": "string", "default": "04:30", "description": "闹钟时间"},
"repeat": {
"type": "enum",
"values": CLOCK_REPEAT_VALUES,
"default": "daily",
},
"_alarm": {
"sampler": Clock.sample_existing_alarm,
"fields": {"alarm_id": "alarm_id", "time": "time"},
},
}
expected_changes = ["alarms[id={alarm_id}]"]
criteria = {"alarms[id={alarm_id}].repeat": "{repeat}"}
async def _post_sample(self, env):
replacement = next(value for value in CLOCK_REPEAT_VALUES.values() if value != self.p.repeat)
await env.set_state({"apps": {"clock": {
f"alarms[id={self.p.alarm_id}]": {"repeat": replacement},
}}}, deep=True, reload=False)
class AddWorldCity(BaseTask):
templates = [
"在世界时钟里添加{city}",
"Add {city} to the world clock",
]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L2"
capabilities = ["nav", "search"]
parameters = {
"city": {"type": "string", "default": "北京", "description": "城市名称"},
"_city": {
"sampler": Clock.sample_addable_city,
"fields": {"city": "city"},
},
}
expected_changes = ["selectedCityIds", "selectedCities[name={city}]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [{
"field": "selectedCityIds",
"expected": f"contains {self.p.city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": clock.selected_city_matches(self.p.city) and not clock.init.selected_city_matches(self.p.city),
}]
class RemoveWorldCity(BaseTask):
templates = [
"把{city}从世界时钟里删掉",
"Remove {city} from the world clock",
]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L2"
capabilities = ["edit"]
parameters = {
"city": {"type": "string", "default": "伦敦", "description": "城市名称"},
"_city": {
"sampler": Clock.sample_selected_city,
"fields": {"city": "city"},
},
}
expected_changes = ["selectedCityIds", "selectedCities[name={city}]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [{
"field": "selectedCityIds",
"expected": f"not contains {self.p.city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": not clock.selected_city_matches(self.p.city) and clock.init.selected_city_matches(self.p.city),
}]
class CheckAlarmNote(AnswerTask):
templates = ["时钟里{time}的闹钟备注写的什么"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "atomic"
difficulty = "L2"
capabilities = ["extract"]
answer_fields = [{"type": "text", "label": "闹钟备注"}]
parameters = {
"alarm_id": {"type": "string", "default": "a1", "description": "闹钟 id"},
"time": {"type": "string", "default": "04:30", "description": "闹钟时间"},
"_alarm": {
"sampler": Clock.sample_noted_alarm,
"fields": {"alarm_id": "alarm_id", "time": "time"},
},
}
def get_answer(self, input: JudgeInput) -> Any:
return Clock(input.apps_init["clock"]).find_alarm_by_id(self.p.alarm_id)["note"]
class AddAlarmWithSettings(BaseTask):
templates = ["设一个{time}的闹钟,重复模式{repeat},备注写“{note}”"]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L4"
max_steps = 45
capabilities = ["create"]
parameters = {
"time": {"type": "string", "default": "07:10", "description": "闹钟时间"},
"hour": {"type": "int", "default": 7, "description": "小时"},
"minute": {"type": "int", "default": 10, "description": "分钟"},
"repeat": {
"type": "enum",
"values": CLOCK_REPEAT_VALUES,
"default": "daily",
},
"note": {
"type": "enum",
"values": CLOCK_NOTE_VALUES,
"default": "晨练",
},
"_time": {
"sampler": Clock.sample_new_alarm_time,
"fields": {"time": "time", "hour": "hour", "minute": "minute"},
},
}
expected_changes = ["alarms[+1]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [clock.check_created_alarm(
self.p.hour, self.p.minute, repeat=self.p.repeat, note=self.p.note,
)]
class EnableAllAlarms(BaseTask):
templates = [
"帮我把时钟里所有闹钟都打开",
"Turn on all alarms in Clock",
]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L4"
max_steps = 30
capabilities = ["edit"]
expected_changes = ["alarms"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"])
return [{
"field": "alarms",
"expected": "全部 enabled=True",
"actual": [bool(alarm.get("enabled")) for alarm in clock.alarms],
"passed": bool(clock.alarms) and all(bool(alarm.get("enabled")) for alarm in clock.alarms),
}]
class CheckCityTime(AnswerTask):
templates = ["帮我看看世界时钟里{city}现在几点", "世界时钟里{city}现在是几点钟"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "sequential"
difficulty = "L3"
capabilities = ["extract"]
answer_fields = [{"type": "text", "label": "{city}现在几点", "hint": "如:14:30", "matcher": "time"}]
parameters = {
"city": {"type": "string", "default": "巴黎", "description": "城市名称"},
"_city": {
"sampler": Clock.sample_selected_city,
"fields": {"city": "city"},
},
}
def get_answer(self, input: JudgeInput) -> Any:
return Clock(input.apps_init["clock"]).city_time(self.p.city, input.os_init)
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps_init["clock"])
return [clock.check_city_time_answer(self.p.city, input.os_init, input.answer)]
class CompareCityTimeDiff(AnswerTask):
templates = ["{city1}{city2}现在差几个小时", "世界时钟里{city1}{city2}差多少个小时"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "deep_dive"
difficulty = "L2"
capabilities = ["extract", "reasoning"]
answer_fields = [{"type": "number", "label": "时差(小时)"}]
parameters = {
"city1": {"type": "string", "default": "巴黎", "description": "城市 1"},
"city2": {"type": "string", "default": "纽约", "description": "城市 2"},
"_cities": {
"sampler": Clock.sample_selected_city_pair,
"fields": {"city1": "city1", "city2": "city2"},
},
}
def get_answer(self, input: JudgeInput) -> Any:
return Clock(input.apps_init["clock"]).time_diff_hours(self.p.city1, self.p.city2)
class CityLocalTimeDiff(AnswerTask):
templates = ["世界时钟里{city}比咱们这儿快还是慢,差多久"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "sequential"
difficulty = "L3"
capabilities = ["extract", "reasoning"]
answer_fields = [{"type": "text", "label": "{city}与本地时差", "hint": "如:快3小时"}]
parameters = {
"city": {"type": "string", "default": "巴黎", "description": "城市名称"},
"_city": {
"sampler": Clock.sample_city_not_local_offset,
"fields": {"city": "city"},
},
}
def get_answer(self, input: JudgeInput) -> Any:
return Clock(input.apps_init["clock"]).city_local_diff_text(self.p.city, input.os_init)
def get_expected_response(self, input: JudgeInput) -> list:
clock = Clock(input.apps_init["clock"])
diff_minutes = int(clock.find_city(self.p.city)["gmtOffsetMinutes"]) - Clock._local_offset_minutes(input.os_init)
if diff_minutes == 0:
return ["一样"]
sign = "快" if diff_minutes > 0 else "慢"
abs_minutes = abs(diff_minutes)
hours = abs_minutes // 60
minutes = abs_minutes % 60
parts = []
if hours > 0:
parts.append(f"{hours}小时")
if minutes > 0:
parts.append(f"{minutes}分钟")
return [f"{sign}{''.join(parts)}"]
class LatestTimezoneCity(AnswerTask):
templates = ["世界时钟里的城市,哪个现在时间最晚", "帮我看看世界时钟里哪个城市现在最晚"]
apps = ["clock"]
scope = "S1"
objective = "query"
composition = "deep_dive"
difficulty = "L3"
capabilities = ["extract", "reasoning"]
answer_fields = [{"type": "text", "label": "时间最晚的城市", "hint": "如:纽约"}]
def get_answer(self, input: JudgeInput) -> Any:
return Clock(input.apps_init["clock"]).latest_city_name()
def get_expected_response(self, input: JudgeInput) -> list:
result = Clock(input.apps_init["clock"]).latest_city_name()
if isinstance(result, str):
return [result]
# regex(多城市并列)→ 返回第一个城市名
cities = Clock(input.apps_init["clock"]).selected_cities
max_off = max(int(c["gmtOffsetMinutes"]) for c in cities)
winners = [str(c["name"]) for c in cities if int(c["gmtOffsetMinutes"]) == max_off]
return [winners[0]]
class AddCityAndCheckTime(BaseTask):
templates = ["在世界时钟里加上{city},然后告诉我那边现在几点"]
apps = ["clock"]
scope = "S1"
objective = "hybrid"
composition = "sequential"
difficulty = "L2"
capabilities = ["nav", "search", "extract"]
parameters = {
"city": {"type": "string", "default": "北京", "description": "城市名称"},
"_city": {
"sampler": Clock.sample_addable_city,
"fields": {"city": "city"},
},
}
expected_changes = ["selectedCityIds", "selectedCities[name={city}]"]
answer_fields = [{"type": "text", "label": "{city}现在几点", "hint": "如:14:30", "matcher": "time"}]
def get_expected_response(self, input: JudgeInput) -> list:
return [Clock(input.apps["clock"]).city_time(self.p.city, input.os_init)]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [
{
"field": "selectedCityIds",
"expected": f"contains {self.p.city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": clock.selected_city_matches(self.p.city) and not clock.init.selected_city_matches(self.p.city),
},
clock.check_city_time_answer(self.p.city, input.os_init, input.answer),
]
class AddCityAndCompareTimeDiff(BaseTask):
templates = ["把{new_city}加到世界时钟,然后告诉我{new_city}{existing_city}差几个小时"]
apps = ["clock"]
scope = "S1"
objective = "hybrid"
composition = "deep_dive"
difficulty = "L2"
capabilities = ["nav", "search", "extract", "reasoning"]
parameters = {
"new_city": {"type": "string", "default": "北京", "description": "新城市"},
"existing_city": {"type": "string", "default": "巴黎", "description": "已有城市"},
"_cities": {
"sampler": Clock.sample_compare_city_pair_with_new,
"fields": {"new_city": "new_city", "existing_city": "existing_city"},
},
}
expected_changes = ["selectedCityIds", "selectedCities[name={new_city}]"]
answer_fields = [{"type": "number", "label": "时差(小时)"}]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
checks = [{
"field": "selectedCityIds",
"expected": f"contains {self.p.new_city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": clock.selected_city_matches(self.p.new_city) and not clock.init.selected_city_matches(self.p.new_city),
}]
checks.extend(build_answer_checks(clock.time_diff_hours(self.p.new_city, self.p.existing_city), input.answer))
return checks
class ReorganizeWorldClock(BaseTask):
templates = [
"把世界时钟里的{remove_city}删掉,换成{add_city}",
"Remove {remove_city} from the world clock and replace it with {add_city}",
]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L4"
capabilities = ["edit", "search"]
parameters = {
"remove_city": {"type": "string", "default": "伦敦", "description": "待删除城市"},
"add_city": {"type": "string", "default": "北京", "description": "待添加城市"},
"_cities": {
"sampler": Clock.sample_remove_add_city,
"fields": {"remove_city": "remove_city", "add_city": "add_city"},
},
}
expected_changes = ["selectedCityIds", "selectedCities[name={remove_city}]", "selectedCities[name={add_city}]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [
{
"field": "remove_city",
"expected": f"not contains {self.p.remove_city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": not clock.selected_city_matches(self.p.remove_city),
},
{
"field": "add_city",
"expected": f"contains {self.p.add_city}",
"actual": [str(city["name"]) for city in clock.selected_cities],
"passed": clock.selected_city_matches(self.p.add_city) and not clock.init.selected_city_matches(self.p.add_city),
},
]
class SetupMorningAlarms(BaseTask):
templates = ["帮我设两个起床闹钟:{time1}的设成{repeat1}{time2}的设成{repeat2}"]
apps = ["clock"]
scope = "S1"
objective = "operate"
composition = "sequential"
difficulty = "L3"
max_steps = 60
capabilities = ["create"]
parameters = {
"time1": {"type": "string", "default": "07:10", "description": "第一个闹钟时间"},
"h1": {"type": "int", "default": 7, "description": "第一个闹钟小时"},
"m1": {"type": "int", "default": 10, "description": "第一个闹钟分钟"},
"time2": {"type": "string", "default": "07:20", "description": "第二个闹钟时间"},
"h2": {"type": "int", "default": 7, "description": "第二个闹钟小时"},
"m2": {"type": "int", "default": 20, "description": "第二个闹钟分钟"},
"repeat1": {
"type": "enum",
"values": CLOCK_REPEAT_VALUES,
"default": "daily",
},
"repeat2": {
"type": "enum",
"values": CLOCK_REPEAT_VALUES,
"default": "weekday",
},
"_times": {
"sampler": Clock.sample_two_new_alarm_times,
"fields": {
"time1": "time1",
"h1": "h1",
"m1": "m1",
"time2": "time2",
"h2": "h2",
"m2": "m2",
},
},
}
expected_changes = ["alarms[+2]"]
def check_goals(self, input: JudgeInput) -> list[dict[str, Any]]:
clock = Clock(input.apps["clock"], init=input.apps_init["clock"])
return [
clock.check_created_alarm(self.p.h1, self.p.m1, field="alarm1", repeat=self.p.repeat1),
clock.check_created_alarm(self.p.h2, self.p.m2, field="alarm2", repeat=self.p.repeat2),
]