wehub-resource-sync
118 lines
3.9 KiB
YAML
118 lines
3.9 KiB
YAML
# Auto-generated OpenCompass Config for RD-Agent Benchmark
|
|
# DO NOT EDIT MANUALLY - Generated by benchmark.py
|
|
|
|
template: |-
|
|
from mmengine.config import read_base
|
|
from opencompass.models import VLLMwithChatTemplate
|
|
|
|
# ==================== Dataset Import ====================
|
|
# Use explicit imports (not `import *`) to avoid leaking non-serializable
|
|
# objects from dataset configs into the namespace.
|
|
with read_base():
|
|
{% for imp in dataset_imports %}
|
|
{% if imp.names %}
|
|
from {{ imp.module }} import {{ imp.names | join(', ') }}
|
|
{% else %}
|
|
from {{ imp.module }} import *
|
|
{% endif %}
|
|
{% endfor %}
|
|
|
|
# Aggregate all dataset variables
|
|
datasets = sum([v for k, v in locals().items() if (k == 'datasets' or k.endswith('_datasets')) and isinstance(v, list)], [])
|
|
|
|
# Apply dataset modifications
|
|
for ds in datasets:
|
|
{% if test_range %}
|
|
# Apply dataset range (e.g., "[:100]" for validation, "[-100:]" for test)
|
|
if 'reader_cfg' not in ds:
|
|
ds['reader_cfg'] = {}
|
|
ds['reader_cfg']['test_range'] = '{{ test_range }}'
|
|
|
|
# Sync to evaluator's dataset_cfg
|
|
if 'eval_cfg' in ds and 'evaluator' in ds['eval_cfg']:
|
|
evaluator = ds['eval_cfg']['evaluator']
|
|
if isinstance(evaluator, dict) and 'dataset_cfg' in evaluator:
|
|
if 'reader_cfg' not in evaluator['dataset_cfg']:
|
|
evaluator['dataset_cfg']['reader_cfg'] = {}
|
|
evaluator['dataset_cfg']['reader_cfg']['test_range'] = '{{ test_range }}'
|
|
{% endif %}
|
|
{% if num_runs and num_runs > 1 %}
|
|
# Multiple runs (repeat each sample n times for averaging or pass@k)
|
|
ds['n'] = {{ num_runs }}
|
|
{% endif %}
|
|
{% if pass_k %}
|
|
# Pass@k evaluation
|
|
ds['k'] = {{ pass_k }}
|
|
{% endif %}
|
|
pass
|
|
|
|
# ==================== Model Configuration ====================
|
|
models = [
|
|
dict(
|
|
type=VLLMwithChatTemplate,
|
|
abbr='{{ model_abbr }}',
|
|
path='{{ model_path }}',
|
|
model_kwargs=dict(
|
|
tensor_parallel_size={{ tensor_parallel_size }},
|
|
gpu_memory_utilization={{ gpu_memory_utilization }},
|
|
trust_remote_code=True,
|
|
dtype='{{ dtype }}',
|
|
max_model_len={{ max_seq_len }},
|
|
enforce_eager=True,
|
|
),
|
|
max_seq_len={{ max_seq_len }},
|
|
max_out_len={{ max_out_len }},
|
|
batch_size={{ batch_size }},
|
|
generation_kwargs=dict(
|
|
temperature={{ temperature }},
|
|
top_p={{ top_p }},
|
|
top_k={{ top_k }},
|
|
{% if repetition_penalty != 1.0 %}
|
|
repetition_penalty={{ repetition_penalty }},
|
|
{% endif %}
|
|
),
|
|
{% if enable_thinking %}
|
|
chat_template_kwargs=dict(enable_thinking=True),
|
|
{% endif %}
|
|
{% if enable_thinking or use_cot_postprocessor %}
|
|
pred_postprocessor=dict(type='extract-non-reasoning-content'),
|
|
{% endif %}
|
|
run_cfg=dict(
|
|
num_gpus={{ tensor_parallel_size }},
|
|
num_procs=1,
|
|
),
|
|
),
|
|
]
|
|
|
|
# ==================== Inference Configuration ====================
|
|
infer = dict(
|
|
partitioner=dict(
|
|
type='NaivePartitioner',
|
|
),
|
|
runner=dict(
|
|
type='LocalRunner',
|
|
max_num_workers=16,
|
|
task=dict(
|
|
type='OpenICLInferTask',
|
|
),
|
|
),
|
|
)
|
|
|
|
# ==================== Evaluation Configuration ====================
|
|
eval = dict(
|
|
partitioner=dict(
|
|
type='NaivePartitioner',
|
|
),
|
|
runner=dict(
|
|
type='LocalRunner',
|
|
max_num_workers=16,
|
|
task=dict(
|
|
type='OpenICLEvalTask',
|
|
dump_details=True,
|
|
),
|
|
),
|
|
)
|
|
|
|
# ==================== Work Directory ====================
|
|
work_dir = '{{ work_dir }}'
|