wehub-resource-sync
85742ab165
Deploy Documentation / deploy (push) Has been cancelled
CPU Test / Test (Utilities, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (LLM proxy, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (Others, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (Store, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (Utilities, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (Weave, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (AgentOps, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (LLM proxy, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (Others, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (Weave, latest, Python 3.13) (push) Has been cancelled
Dashboard / Chromatic (push) Has been cancelled
CPU Test / Lint - fast (push) Has been cancelled
CPU Test / Lint - next (push) Has been cancelled
CPU Test / Lint - slow (push) Has been cancelled
CPU Test / Lint - JavaScript (push) Has been cancelled
CPU Test / Build documentation (push) Has been cancelled
CPU Test / Test (AgentOps, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (LLM proxy, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (Others, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (Store, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (Weave, legacy, Python 3.10) (push) Has been cancelled
CPU Test / Test (AgentOps, stable, Python 3.11) (push) Has been cancelled
CPU Test / Test (Store, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (Utilities, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (Weave, stable, Python 3.12) (push) Has been cancelled
CPU Test / Test (AgentOps, latest, Python 3.13) (push) Has been cancelled
CPU Test / Test (LLM proxy, latest, Python 3.13) (push) Has been cancelled
CPU Test / Test (Others, latest, Python 3.13) (push) Has been cancelled
CPU Test / Test (Store, latest, Python 3.13) (push) Has been cancelled
CPU Test / Test (Utilities, latest, Python 3.13) (push) Has been cancelled
CPU Test / Test (JavaScript) (push) Has been cancelled
45 lines
1.2 KiB
Python
45 lines
1.2 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
|
|
# type: ignore
|
|
|
|
import torch
|
|
from datasets import Dataset as HuggingFaceDataset
|
|
from omegaconf import DictConfig
|
|
from verl.utils.dataset.rl_dataset import RLHFDataset
|
|
|
|
from agentlightning.types import Dataset
|
|
|
|
__all__ = [
|
|
"AgentDataset",
|
|
"LoadedDataset",
|
|
]
|
|
|
|
|
|
class AgentDataset(RLHFDataset):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
|
|
self.filter_overlong_prompts = False
|
|
|
|
def __getitem__(self, item):
|
|
row_dict: dict = self.dataframe[item]
|
|
|
|
# add index for each prompt
|
|
index = row_dict.get("extra_info", {}).get("index", 0)
|
|
row_dict["index"] = index
|
|
# Workaround for data proto. At least one tensor is needed.
|
|
row_dict["fake_ids"] = torch.ones(1, dtype=torch.int)
|
|
return row_dict
|
|
|
|
|
|
class LoadedDataset(AgentDataset):
|
|
|
|
def __init__(self, dataset: Dataset):
|
|
super().__init__([], None, DictConfig({})) # type: ignore
|
|
dataset_copy = [dataset[i] for i in range(len(dataset))]
|
|
self.dataframe = HuggingFaceDataset.from_list(dataset_copy)
|
|
|
|
def _read_files_and_tokenize(self):
|
|
pass
|