wehub-resource-sync
a06f331eb8
install-script / powershell-syntax (push) Waiting to run
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
install-script / install (macos-14) (push) Waiting to run
install-script / install (ubuntu-latest) (push) Waiting to run
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
83 lines
2.2 KiB
Python
83 lines
2.2 KiB
Python
"""Prompt template loader for the eval framework.
|
|
|
|
Loads Jinja2 system and instance prompt templates per evaluation mode
|
|
from the ``eval/prompts/`` directory.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Tuple
|
|
|
|
import jinja2
|
|
|
|
# Valid evaluation modes
|
|
VALID_MODES = ("baseline", "native", "native_augment")
|
|
|
|
# Prompts directory lives alongside this module
|
|
_PROMPTS_DIR = Path(__file__).resolve().parent / "prompts"
|
|
|
|
|
|
def _make_env() -> jinja2.Environment:
|
|
"""Create a Jinja2 environment rooted at the prompts directory."""
|
|
return jinja2.Environment(
|
|
loader=jinja2.FileSystemLoader(str(_PROMPTS_DIR)),
|
|
keep_trailing_newline=True,
|
|
)
|
|
|
|
|
|
def load_templates(mode: str) -> Tuple[jinja2.Template, jinja2.Template]:
|
|
"""Load system and instance Jinja2 templates for *mode*.
|
|
|
|
Parameters
|
|
----------
|
|
mode:
|
|
One of ``baseline``, ``native``, or ``native_augment``.
|
|
|
|
Returns
|
|
-------
|
|
tuple[jinja2.Template, jinja2.Template]
|
|
``(system_template, instance_template)``
|
|
|
|
Raises
|
|
------
|
|
FileNotFoundError
|
|
If either template file does not exist for the given mode.
|
|
"""
|
|
env = _make_env()
|
|
|
|
system_name = f"system_{mode}.jinja"
|
|
instance_name = f"instance_{mode}.jinja"
|
|
|
|
system_path = _PROMPTS_DIR / system_name
|
|
instance_path = _PROMPTS_DIR / instance_name
|
|
|
|
if not system_path.is_file():
|
|
raise FileNotFoundError(f"System template not found: {system_path}")
|
|
if not instance_path.is_file():
|
|
raise FileNotFoundError(f"Instance template not found: {instance_path}")
|
|
|
|
system_template = env.get_template(system_name)
|
|
instance_template = env.get_template(instance_name)
|
|
|
|
return system_template, instance_template
|
|
|
|
|
|
def render_instance_prompt(template: jinja2.Template, task: str) -> str:
|
|
"""Render an instance prompt template with the given *task*.
|
|
|
|
Parameters
|
|
----------
|
|
template:
|
|
A Jinja2 ``Template`` object (typically the instance template
|
|
returned by :func:`load_templates`).
|
|
task:
|
|
The SWE-bench issue description to inject into the template.
|
|
|
|
Returns
|
|
-------
|
|
str
|
|
The fully rendered prompt string.
|
|
"""
|
|
return template.render(task=task)
|