chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,355 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
import logging
|
||||
import datasets
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
from datetime import timedelta
|
||||
from typing import List, Optional
|
||||
from accelerate import Accelerator, InitProcessGroupKwargs
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
from transformers import DataCollatorWithPadding
|
||||
|
||||
from src.lm import LM, LMArgs, GenerationArgs
|
||||
from src.retrieval import RetrievalArgs
|
||||
from src.utils.util import makedirs, load_json, FileLogger
|
||||
from .eval_retrieval import main as retrieval_main
|
||||
from .icl_utils import flat_options, perplexity_to_choice, compute_scores, _llm_generation_func, _llm_perplexity_func
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CQA = {
|
||||
"arc_c":{'method':'perplexity', 'metric':'acc'},
|
||||
"arc_e":{'method':'perplexity', 'metric':'acc'},
|
||||
"natural_questions":{'method':'generation', 'metric':'em'},
|
||||
"cate_name":'CQA'
|
||||
}
|
||||
Commonsense = {
|
||||
"copa":{'method':'perplexity', 'metric':'acc'},
|
||||
"hellaswag":{'method':'perplexity', 'metric':'acc'},
|
||||
"piqa":{'method':'perplexity', 'metric':'acc'},
|
||||
'cate_name': 'Commonsense'
|
||||
}
|
||||
Coreference = {
|
||||
"winogrande":{'method':'perplexity', 'metric':'acc'},
|
||||
"wsc":{'method':'perplexity', 'metric':'acc'},
|
||||
"wsc273":{'method':'perplexity', 'metric':'acc'},
|
||||
'cate_name': 'Coreference'
|
||||
}
|
||||
Paraphrase = {
|
||||
"mrpc":{'method':'perplexity', 'metric':'acc'},
|
||||
"paws":{'method':'perplexity', 'metric':'acc'},
|
||||
"qqp":{'method':'perplexity', 'metric':'acc'},
|
||||
'cate_name': 'Paraphrase'
|
||||
}
|
||||
NLI = {
|
||||
"rte":{'method':'perplexity', 'metric':'acc'},
|
||||
"snli":{'method':'perplexity', 'metric':'acc'},
|
||||
"mnli_m":{'method':'perplexity', 'metric':'acc'},
|
||||
"mnli_mm":{'method':'perplexity', 'metric':'acc'},
|
||||
"qnli":{'method':'perplexity', 'metric':'acc'},
|
||||
'cate_name': 'NLI'
|
||||
}
|
||||
ReadingComp = {
|
||||
"multirc":{'method':'perplexity', 'metric':'f1'},
|
||||
"openbookqa":{'method':'perplexity', 'metric':'acc'},
|
||||
"boolq":{'method':'perplexity', 'metric':'acc'},
|
||||
"squad_v1":{'method':'generation', 'metric':'em'},
|
||||
'cate_name': 'ReadingComp'
|
||||
}
|
||||
Sentiment = {
|
||||
"sentiment140":{'method':'perplexity', 'metric':'acc'},
|
||||
"sst2":{'method':'perplexity', 'metric':'acc'},
|
||||
"yelp":{'method':'perplexity', 'metric':'acc'},
|
||||
'cate_name': 'Sentiment'
|
||||
}
|
||||
Data2Text = {
|
||||
"common_gen":{'method':'generation', 'metric':'rl'},
|
||||
"e2e_nlg":{'method':'generation', 'metric':'rl'},
|
||||
"dart":{'method':'generation', 'metric':'rl'},
|
||||
'cate_name': 'Data2Text'
|
||||
}
|
||||
Summarize = {
|
||||
"aeslc":{'method':'generation', 'metric':'rl'},
|
||||
"ag_news":{'method':'perplexity', 'metric':'acc'},
|
||||
"gigaword":{'method':'generation', 'metric':'rl'},
|
||||
'cate_name': 'Summarize'
|
||||
}
|
||||
TASK_LIST = [CQA, Commonsense, Coreference, Paraphrase, NLI, ReadingComp, Sentiment, Data2Text, Summarize]
|
||||
task2cat = {}
|
||||
for category in TASK_LIST:
|
||||
cat_name = category["cate_name"]
|
||||
for key, value in category.items():
|
||||
if key == "cate_name":
|
||||
continue
|
||||
task2cat[key] = cat_name
|
||||
|
||||
|
||||
@dataclass
|
||||
class ICLArgs(LMArgs, RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/icl/",
|
||||
metadata={'help': 'Path to the file for saving embeddings and results.'}
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:icl/icl/test.json",
|
||||
metadata={'help': 'Path to the file containing both retrieved keys and answers.'}
|
||||
)
|
||||
task_names: Optional[List[str]] = field(
|
||||
default=None,
|
||||
metadata={'help': 'List of tasks to evaluate.'}
|
||||
)
|
||||
load_prev_result: bool = field(
|
||||
default=False,
|
||||
metadata={'help': 'Load existing results in output_dir?'}
|
||||
)
|
||||
|
||||
context_max_length: int = field(
|
||||
default=1024,
|
||||
metadata={'help': 'Evaluation json file.'},
|
||||
)
|
||||
few_shot: int = field(
|
||||
default=8,
|
||||
metadata={'help': 'How many few shot train samples?'},
|
||||
)
|
||||
|
||||
corpus: str = field(
|
||||
default="llm-embedder:icl/icl/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{contents}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
metrics: List[str] = field(
|
||||
default_factory=lambda: [],
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/icl/icl.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerationArgs(GenerationArgs):
|
||||
max_new_tokens: int = field(
|
||||
default=64,
|
||||
metadata={'help': 'Maximum new tokens to generate.'}
|
||||
)
|
||||
|
||||
|
||||
def remove_double_space(string):
|
||||
return re.sub("[ ]{2,}", " ", string)
|
||||
|
||||
|
||||
def load_test_data(knn_inxs,
|
||||
test_data,
|
||||
corpus_data,
|
||||
filter_diff_task: bool=False,
|
||||
example_num=8,
|
||||
same_task_random=False,
|
||||
):
|
||||
dataset = datasets.load_dataset('json', data_files=test_data)['train']
|
||||
passage_dataset = datasets.load_dataset('json', data_files=corpus_data)['train']
|
||||
|
||||
task_data = defaultdict(list)
|
||||
for i, e in enumerate(tqdm(dataset, desc="Organizing Data")):
|
||||
query = remove_double_space(e['query'])
|
||||
answers = [remove_double_space(x) for x in e['answers']]
|
||||
if knn_inxs is not None:
|
||||
if filter_diff_task:
|
||||
few_shot = []
|
||||
rest_passage = []
|
||||
for x in knn_inxs[i]:
|
||||
icl_e = passage_dataset[int(x)]
|
||||
# print(icl_e['task_name'], e['task_name'])
|
||||
if icl_e['task_name'][:4] == e['task_name'][:4]:
|
||||
few_shot.append(remove_double_space(icl_e['contents']))
|
||||
if len(few_shot) > example_num: break
|
||||
else:
|
||||
if len(rest_passage) < example_num:
|
||||
rest_passage.append(remove_double_space(icl_e['contents']))
|
||||
|
||||
if len(few_shot) < example_num:
|
||||
few_shot.extend(rest_passage)
|
||||
few_shot = few_shot[:example_num]
|
||||
|
||||
else:
|
||||
# if task2cat[e['task_name']] == 'Coreference':
|
||||
# candidates = random.sample(knn_inxs[i][:20], example_num)
|
||||
# else:
|
||||
# candidates = knn_inxs[i][:example_num]
|
||||
candidates = knn_inxs[i][:example_num]
|
||||
few_shot = [remove_double_space(passage_dataset[int(x)]['contents']) for x in candidates]
|
||||
else:
|
||||
few_shot = []
|
||||
data = {"query":query, "answers":answers, "few_shot":few_shot}
|
||||
if 'options' in e:
|
||||
data['options'] = e['options']
|
||||
task_data[e['task_name']].append(data)
|
||||
|
||||
if same_task_random:
|
||||
task_name_2_idx = defaultdict(list)
|
||||
for i, example in enumerate(tqdm(passage_dataset, "Collecting Task Indices")):
|
||||
task_name_2_idx[example["task_name"]].append(i)
|
||||
|
||||
for task_name, task_examples in tqdm(task_data.items(), desc="Collecting Same-Task-Random Examples"):
|
||||
if task_name in ["mnli_m", "mnli_mm"]:
|
||||
corpus_task_name = "mnli"
|
||||
else:
|
||||
corpus_task_name = task_name
|
||||
|
||||
for i, _ in enumerate(task_examples):
|
||||
task_indices = task_name_2_idx[corpus_task_name]
|
||||
example_num = min(example_num, len(task_indices))
|
||||
# get examples of the same task
|
||||
few_shot = [remove_double_space(content) for content in passage_dataset[random.sample(task_indices, example_num)]["contents"]]
|
||||
task_data[task_name][i]["few_shot"] = few_shot
|
||||
|
||||
return task_data
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([ICLArgs, GenerationArgs])
|
||||
args, generation_args = parser.parse_args_into_dataclasses()
|
||||
accelerator = Accelerator(cpu=args.cpu, kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=100000))])
|
||||
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
args.output_dir = output_dir
|
||||
|
||||
if args.retrieval_method != "no":
|
||||
_, preds, _ = retrieval_main(args=args, accelerator=accelerator, log=False)
|
||||
else:
|
||||
preds = None
|
||||
|
||||
llm = LM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
accelerator=accelerator,
|
||||
generation_args=asdict(generation_args)
|
||||
)
|
||||
|
||||
tokenizer = llm.tokenizer
|
||||
|
||||
args.output_dir = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--"))
|
||||
|
||||
task_data = load_test_data(preds, test_data=args.eval_data, corpus_data=args.corpus, example_num=args.few_shot, same_task_random=args.retrieval_method == "same-task-random")
|
||||
|
||||
all_results = []
|
||||
metrics = {}
|
||||
for task_cate in [CQA, Commonsense, Coreference, Paraphrase, NLI, ReadingComp, Sentiment, Data2Text, Summarize]:
|
||||
task_results = []
|
||||
for task_name, setting in task_cate.items():
|
||||
if task_name == 'cate_name':
|
||||
continue
|
||||
# skip tasks that are not specified
|
||||
if args.task_names is not None and task_name not in args.task_names:
|
||||
continue
|
||||
|
||||
save_path = os.path.join(args.output_dir, f'{task_name}.json')
|
||||
|
||||
if args.load_prev_result and os.path.exists(save_path):
|
||||
# the first line is the metric
|
||||
result = load_json(save_path, lines=True)[0]
|
||||
task_results.append(result['metric_value'][setting['metric']])
|
||||
all_results.append(result['metric_value'][setting['metric']])
|
||||
if accelerator.process_index == 0:
|
||||
logger.info(f"loading existing results from {save_path}...")
|
||||
print(result)
|
||||
continue
|
||||
|
||||
test_data = task_data[task_name]
|
||||
if accelerator.process_index == 0:
|
||||
print(f"------{task_name} ({len(all_results) + 1} / {30})------")
|
||||
|
||||
if setting['metric'] == 'acc':
|
||||
assert setting['method'] == 'perplexity'
|
||||
if setting['method'] == 'perplexity':
|
||||
flat_data = flat_options(test_data)
|
||||
dataset = datasets.Dataset.from_list(flat_data)
|
||||
dataset.set_transform(
|
||||
partial(
|
||||
_llm_perplexity_func,
|
||||
tokenizer=tokenizer,
|
||||
example_num=args.few_shot,
|
||||
max_input_tokens=args.context_max_length,
|
||||
add_llama_inst=args.add_llama_inst,
|
||||
)
|
||||
)
|
||||
else:
|
||||
dataset = datasets.Dataset.from_list(test_data)
|
||||
dataset.set_transform(
|
||||
partial(
|
||||
_llm_generation_func,
|
||||
tokenizer=tokenizer,
|
||||
example_num=args.few_shot,
|
||||
max_input_tokens=args.context_max_length,
|
||||
add_llama_inst=args.add_llama_inst,
|
||||
)
|
||||
)
|
||||
|
||||
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
if setting['method'] == 'perplexity':
|
||||
predictions = llm.compute_nlls(dataloader)
|
||||
predictions = perplexity_to_choice(test_data, predictions)
|
||||
else:
|
||||
if args.add_llama_inst:
|
||||
eos_token_id = tokenizer.eos_token_id
|
||||
else:
|
||||
eos_token_id = tokenizer.encode("\n", add_special_tokens=False)[-1]
|
||||
|
||||
predictions = llm.generate(dataloader, eos_token_id=eos_token_id)
|
||||
predictions = [x.strip() for x in predictions]
|
||||
|
||||
if setting['metric'] in ['em']:
|
||||
labels = [x['answers'] for x in test_data]
|
||||
else:
|
||||
labels = [x['answers'][0] for x in test_data]
|
||||
|
||||
metric_value = compute_scores(setting['metric'], predictions, labels)
|
||||
|
||||
result = {'task_name':task_name, 'setting':setting, 'metric_value':metric_value}
|
||||
if accelerator.process_index == 0:
|
||||
print(result)
|
||||
with open(makedirs(save_path), 'w') as f:
|
||||
f.write(json.dumps(result, ensure_ascii=False) + "\n")
|
||||
for i, sample in enumerate(test_data):
|
||||
sample["output"] = predictions[i]
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
|
||||
task_results.append(result['metric_value'][setting['metric']])
|
||||
all_results.append(result['metric_value'][setting['metric']])
|
||||
|
||||
if len(task_results):
|
||||
metrics[task_cate['cate_name']] = np.mean(task_results)
|
||||
|
||||
metrics['avg'] = np.mean(all_results)
|
||||
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
if accelerator.process_index == 0:
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
import os
|
||||
import logging
|
||||
import datasets
|
||||
|
||||
from copy import deepcopy
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.lm import SRLMArgs, SelfRetrievalLM
|
||||
from src.retrieval import Retriever, RetrievalArgs, TASK_CONFIG
|
||||
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
import transformers
|
||||
# disable too long input warning
|
||||
transformers.logging.set_verbosity_error()
|
||||
|
||||
|
||||
# merge two args to get unified arguments
|
||||
@dataclass
|
||||
class LRLMArgs(RetrievalArgs, SRLMArgs):
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:lrlm/books3/test.json",
|
||||
metadata={'help': 'Evaluation json file.'},
|
||||
)
|
||||
lm_batch_size: int = field(
|
||||
default=1,
|
||||
metadata={'help': 'Evaluation json file.'},
|
||||
)
|
||||
|
||||
context_max_length: int = field(
|
||||
default=32768,
|
||||
metadata={'help': 'Evaluation json file.'},
|
||||
)
|
||||
anchor_length: int = field(
|
||||
default=160000,
|
||||
metadata={'help': 'Evaluation file containing long texts.'}
|
||||
)
|
||||
chunk_size: int = field(
|
||||
default=128,
|
||||
metadata={'help': 'How many tokens in a chunk?'}
|
||||
)
|
||||
key_num: int = field(
|
||||
default=8,
|
||||
metadata={'help': 'How many chunks to retrieve at a time?'}
|
||||
)
|
||||
chunk_batch_size: int = field(
|
||||
default=1,
|
||||
metadata={'help': 'How many retrieval & generation to execute in parallel?'}
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/lrlm",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
debug_retrieval: bool = field(
|
||||
default=False,
|
||||
metadata={'help': 'Check retrieval queries and values?'}
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.retrieval_method == "bm25":
|
||||
# NOTE: we can only use naive bm25 for self retrieval
|
||||
self.retrieval_method = "naive-bm25"
|
||||
|
||||
|
||||
def process_lrlm(tokenizer, context_max_length=4096, target_length=1024, anchor_length=160000):
|
||||
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
|
||||
has_eos = False
|
||||
if test[-1] == 1:
|
||||
has_eos = True
|
||||
|
||||
left_truncation_tokenizer = deepcopy(tokenizer)
|
||||
left_truncation_tokenizer.truncation_side = "left"
|
||||
|
||||
@DatasetProcessFn()
|
||||
def _process(text, **kwds):
|
||||
output = {}
|
||||
text = text[:anchor_length]
|
||||
|
||||
inputs = left_truncation_tokenizer(text, max_length=context_max_length, truncation=True, return_token_type_ids=False, add_special_tokens=False)
|
||||
|
||||
if len(inputs.input_ids) < target_length:
|
||||
return None
|
||||
|
||||
labels = inputs["input_ids"].copy()
|
||||
inputs_length = len(labels)
|
||||
labels[:-target_length] = [-100 for _ in range(inputs_length - target_length)]
|
||||
inputs["labels"] = labels
|
||||
|
||||
for k, v in inputs.items():
|
||||
output[k] = v
|
||||
return output
|
||||
return _process
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([LRLMArgs])
|
||||
args, = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
retriever = Retriever(
|
||||
retrieval_method=args.retrieval_method,
|
||||
# for dense retriever
|
||||
query_encoder=args.query_encoder,
|
||||
key_encoder=args.key_encoder,
|
||||
pooling_method=args.pooling_method,
|
||||
dense_metric=args.dense_metric,
|
||||
query_max_length=args.query_max_length,
|
||||
key_max_length=args.key_max_length,
|
||||
tie_encoders=args.tie_encoders,
|
||||
truncation_side=args.truncation_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
dtype=args.dtype,
|
||||
accelerator=accelerator,
|
||||
# for bm25 retriever
|
||||
anserini_dir=args.anserini_dir,
|
||||
k1=args.k1,
|
||||
b=args.b
|
||||
)
|
||||
|
||||
if args.add_instruction:
|
||||
instruction = TASK_CONFIG[args.version]["instruction"]["lrlm"]
|
||||
else:
|
||||
instruction = None
|
||||
|
||||
srlm = SelfRetrievalLM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
retriever=retriever,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
context_window_size=args.context_window_size,
|
||||
chunk_size=args.chunk_size,
|
||||
key_num=args.key_num,
|
||||
chunk_batch_size=args.chunk_batch_size,
|
||||
add_key_continuation=args.add_key_continuation,
|
||||
retrieval_method=args.retrieval_method,
|
||||
order_method=args.order_method,
|
||||
integrate_method=args.integrate_method,
|
||||
instruction=instruction,
|
||||
debug_retrieval=args.debug_retrieval,
|
||||
add_sep=args.add_sep,
|
||||
accelerator=accelerator,
|
||||
)
|
||||
|
||||
tokenizer = srlm.tokenizer
|
||||
|
||||
logging.info(f"Loading data from {args.eval_data}...")
|
||||
|
||||
if args.retrieval_method == "no" and args.context_max_length != args.context_window_size:
|
||||
logger.warning(f"Found retrieval_method is 'no', setting context_max_length to the same as context_window_size ({args.context_window_size})!")
|
||||
args.context_max_length = args.context_window_size
|
||||
|
||||
with accelerator.main_process_first():
|
||||
dataset = datasets.load_dataset("json", data_files=args.eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
dataset = dataset.map(process_lrlm(
|
||||
tokenizer,
|
||||
context_max_length=args.context_max_length,
|
||||
target_length=args.target_length,
|
||||
anchor_length=args.anchor_length,
|
||||
), remove_columns=dataset.column_names, batched=True, batch_size=50, num_proc=64)
|
||||
|
||||
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
perplexity = srlm.compute_perplexity(dataloader)
|
||||
metrics = {"perplexity": perplexity}
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
dataset = os.path.normpath(args.eval_data).split(os.sep)[-2]
|
||||
log_path = os.path.join(args.log_path, f"{dataset}.log")
|
||||
|
||||
file_logger = FileLogger(makedirs(log_path))
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,319 @@
|
||||
import os
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import datasets
|
||||
from typing import List
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from collections import defaultdict
|
||||
|
||||
from src.lm import (
|
||||
LM,
|
||||
LMArgs
|
||||
)
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
RetrievalMetric,
|
||||
)
|
||||
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
|
||||
from .eval_retrieval import main as retrieval_main
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
import transformers
|
||||
transformers.logging.set_verbosity_error()
|
||||
|
||||
SUBJECT_2_CATEGORY={"abstract_algebra": "STEM", "anatomy": "others", "astronomy": "STEM", "business_ethics": "others", "clinical_knowledge": "others", "college_biology": "STEM", "college_chemistry": "STEM", "college_computer_science": "STEM", "college_mathematics": "STEM", "college_medicine": "others", "college_physics": "STEM", "computer_security": "STEM", "conceptual_physics": "STEM", "econometrics": "Social Sciences", "electrical_engineering": "STEM", "elementary_mathematics": "STEM", "formal_logic": "Humanities", "global_facts": "others", "high_school_biology": "STEM", "high_school_chemistry": "STEM", "high_school_computer_science": "STEM", "high_school_european_history": "Humanities", "high_school_geography": "Social Sciences", "high_school_government_and_politics": "Social Sciences", "high_school_macroeconomics": "Social Sciences", "high_school_mathematics": "STEM", "high_school_microeconomics": "Social Sciences", "high_school_physics": "STEM", "high_school_psychology": "Social Sciences", "high_school_statistics": "STEM", "high_school_us_history": "Humanities", "high_school_world_history": "Humanities", "human_aging": "others", "human_sexuality": "Social Sciences", "international_law": "Humanities", "jurisprudence": "Humanities", "logical_fallacies": "Humanities", "machine_learning": "STEM", "management": "others", "marketing": "others", "medical_genetics": "others", "miscellaneous": "others", "moral_disputes": "Humanities", "moral_scenarios": "Humanities", "nutrition": "others", "philosophy": "Humanities", "prehistory": "Humanities", "professional_accounting": "others", "professional_law": "Humanities", "professional_medicine": "others", "professional_psychology": "Social Sciences", "public_relations": "Social Sciences", "security_studies": "Social Sciences", "sociology": "Social Sciences", "us_foreign_policy": "Social Sciences", "virology": "others", "world_religions": "Humanities"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MMLUArgs(LMArgs, RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/mmlu",
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:qa/mmlu/test.json",
|
||||
metadata={'help': 'Path to the test file.'}
|
||||
)
|
||||
lm_batch_size: int = field(
|
||||
default=2,
|
||||
metadata={'help': 'Evaluation batch size.'},
|
||||
)
|
||||
|
||||
few_shot: int = field(
|
||||
default=0,
|
||||
metadata={'help': 'How many few shot train samples?'},
|
||||
)
|
||||
train_data: str = field(
|
||||
default="llm-embedder:qa/mmlu/dev.json",
|
||||
metadata={'help': 'Path to the file containing training examples.'}
|
||||
)
|
||||
|
||||
corpus: str = field(
|
||||
default="llm-embedder:qa/msmarco/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{title} {text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
key_max_length: int = field(
|
||||
default=128,
|
||||
metadata={'help': 'How many tokens at maximum in a key.'}
|
||||
)
|
||||
hits: int = field(
|
||||
default=10,
|
||||
metadata={'help': 'How many hits per query?'},
|
||||
)
|
||||
key_num: int = field(
|
||||
default=3,
|
||||
metadata={'help': 'How many docs to provide in prompt?'},
|
||||
)
|
||||
metrics: List[str] = field(
|
||||
default_factory=lambda: ["collate_key"],
|
||||
)
|
||||
save_to_output: bool = field(
|
||||
default=True,
|
||||
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/mmlu/mmlu.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
def process_mmlu(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False, add_llama_inst=False):
|
||||
tokenizer.truncation_side = 'right'
|
||||
left_truncation_tokenizer = copy.deepcopy(tokenizer)
|
||||
left_truncation_tokenizer.truncation_side = 'left'
|
||||
|
||||
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
|
||||
|
||||
has_bos = has_eos = False
|
||||
if test[0] == 1:
|
||||
has_bos = True
|
||||
if test[-1] == 1:
|
||||
has_eos = True
|
||||
|
||||
if few_shot > 0:
|
||||
assert train_data is not None
|
||||
train_data = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
|
||||
train_df = train_data.to_pandas()
|
||||
# transform the dataframe into dict of dataframes
|
||||
train_df = {k: v[:few_shot] for k, v in train_df.groupby("subject")}
|
||||
|
||||
options = ['A', 'B', 'C', 'D']
|
||||
|
||||
def _prepare_sample(query, choices, answer):
|
||||
"""
|
||||
<Question>
|
||||
A. <Choices 1>
|
||||
B. <Choices 2>
|
||||
C. <Choices 3>
|
||||
D. <Choices 4>
|
||||
Answer: <Answer>
|
||||
"""
|
||||
# answer maybe int or numpy int64
|
||||
if not isinstance(answer, str):
|
||||
answer = options[answer]
|
||||
|
||||
sample = f"{query}\n{chr(10).join([f'{option}. {choice}' for option, choice in zip(options, choices)])}\nAnswer: {answer}"
|
||||
return sample
|
||||
|
||||
def _prepare_knowledge(key, max_length=None):
|
||||
if key is not None:
|
||||
key = key[:key_num]
|
||||
key = "\n".join(key)
|
||||
key = f"Knowledge:\n{key}"
|
||||
if max_length is not None:
|
||||
# truncate key if necessary
|
||||
key = tokenizer.decode(tokenizer.encode(key, add_special_tokens=False, truncation=True, max_length=max_length))
|
||||
else:
|
||||
key = ""
|
||||
return key
|
||||
|
||||
@DatasetProcessFn(augment=True)
|
||||
def _process(query, choices, query_id, subject, answer, key=None, **kwds):
|
||||
"""Yield key and query with a prompt template"""
|
||||
output = defaultdict(list)
|
||||
query = query.strip()
|
||||
|
||||
head = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
|
||||
|
||||
if few_shot > 0:
|
||||
train_samples = ""
|
||||
for i in range(few_shot):
|
||||
if i >= len(train_df[subject]):
|
||||
break
|
||||
train_sample = train_df[subject].iloc[i][['query', 'choices', 'answer']]
|
||||
train_sample = _prepare_sample(**train_sample) + "\n\n"
|
||||
train_samples += train_sample
|
||||
else:
|
||||
train_samples = ""
|
||||
|
||||
knowledge_max_length = context_max_length - len(tokenizer.encode(head + train_samples + _prepare_sample(query, choices, 'A'))) - int(has_bos) - int(has_eos)
|
||||
if knowledge_max_length < 0:
|
||||
knowledge = ""
|
||||
else:
|
||||
knowledge = _prepare_knowledge(key, knowledge_max_length)
|
||||
|
||||
for option in options:
|
||||
left = knowledge
|
||||
right = head + train_samples + _prepare_sample(query, choices, option)
|
||||
# \n\n to split knowledge and prompts
|
||||
if len(left):
|
||||
right = "\n\n" + right
|
||||
|
||||
# TODO: add llama instruction
|
||||
# if add_llama_inst:
|
||||
# left = "[INST]" + left
|
||||
# right = right + "[/INST]"
|
||||
|
||||
inputs = left_truncation_tokenizer(left + right, truncation=True, max_length=context_max_length, return_token_type_ids=False)
|
||||
|
||||
if has_eos and not is_encoder_decoder:
|
||||
inputs = remove_eos(inputs, tokenizer.eos_token_id)
|
||||
|
||||
# find answer length
|
||||
option_seq = tokenizer.encode("Answer: " + option, add_special_tokens=False)
|
||||
option_length = len(option_seq) - len(tokenizer.encode("Answer:", add_special_tokens=False))
|
||||
|
||||
if is_encoder_decoder:
|
||||
labels = inputs["input_ids"].copy()[-option_length:]
|
||||
for k, v in inputs.items():
|
||||
inputs[k] = v[:-option_length]
|
||||
inputs["labels"] = labels
|
||||
|
||||
else:
|
||||
# take care of padded tokens
|
||||
labels = inputs["input_ids"].copy()
|
||||
labels = [x if inputs["attention_mask"][i] == 1 else -100 for i, x in enumerate(labels)]
|
||||
labels[:-option_length] = [-100] * (len(labels) - option_length)
|
||||
inputs["labels"] = labels
|
||||
|
||||
inputs["query_id"] = query_id
|
||||
for k, v in inputs.items():
|
||||
output[k].append(v)
|
||||
return output
|
||||
return _process
|
||||
|
||||
|
||||
def evaluate_mmlu(eval_data, save_path, **kwds):
|
||||
def compute_metric(eval_preds):
|
||||
makedirs(save_path)
|
||||
|
||||
tasks = defaultdict(list)
|
||||
results = defaultdict(list)
|
||||
samples = {}
|
||||
|
||||
with open(eval_data) as f:
|
||||
for line in f:
|
||||
sample = json.loads(line.strip())
|
||||
samples[sample["query_id"]] = sample
|
||||
|
||||
# nll must comes in the order of A, B, C, and D
|
||||
for query_id, nll in zip(*eval_preds):
|
||||
# store log likelihood
|
||||
results[query_id].append(-nll)
|
||||
|
||||
with open(makedirs(save_path), "w") as f:
|
||||
for k, v in results.items():
|
||||
output = max(enumerate(v), key=lambda x: x[1])[0]
|
||||
sample = samples[k]
|
||||
sample["output"] = output
|
||||
tasks[sample["subject"]].append((output, sample["answer"]))
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
|
||||
metrics = defaultdict(list)
|
||||
for task_name, task_eval_preds in tasks.items():
|
||||
accuracy = 0
|
||||
for pred, label in task_eval_preds:
|
||||
accuracy += int(pred == label)
|
||||
accuracy /= len(task_eval_preds)
|
||||
|
||||
category = SUBJECT_2_CATEGORY[task_name]
|
||||
metrics[f"{category}"].append(accuracy)
|
||||
metrics["all"].append(accuracy)
|
||||
|
||||
for k, v in metrics.items():
|
||||
metrics[k] = sum(v) / len(v)
|
||||
|
||||
metrics = {
|
||||
"STEM": metrics["STEM"],
|
||||
"Social Sciences": metrics["Social Sciences"],
|
||||
"Humanities": metrics["Humanities"],
|
||||
"Others": metrics["others"],
|
||||
"All": metrics["all"],
|
||||
}
|
||||
|
||||
return dict(metrics)
|
||||
return compute_metric
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([MMLUArgs])
|
||||
args, = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
# modify the output_dir for retrieval
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
args.output_dir = output_dir
|
||||
|
||||
if args.retrieval_method != "no":
|
||||
retrieval_main(args=args, accelerator=accelerator, log=False)
|
||||
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
|
||||
else:
|
||||
eval_data = args.eval_data
|
||||
|
||||
lm = LM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
accelerator=accelerator
|
||||
)
|
||||
|
||||
tokenizer = lm.tokenizer
|
||||
|
||||
with accelerator.main_process_first():
|
||||
logging.info(f"Loading data from {eval_data}...")
|
||||
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
dataset = dataset.map(process_mmlu(
|
||||
tokenizer,
|
||||
context_max_length=args.context_max_length,
|
||||
key_num=args.key_num,
|
||||
few_shot=args.few_shot,
|
||||
train_data=args.train_data,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
is_encoder_decoder=lm.model.config.is_encoder_decoder,
|
||||
add_llama_inst=args.add_llama_inst
|
||||
), remove_columns=dataset.column_names, batched=True, num_proc=32)
|
||||
|
||||
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
results = lm.compute_nlls(dataloader)
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
|
||||
metrics = evaluate_mmlu(eval_data, result_path)(results)
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
import os
|
||||
import logging
|
||||
import datasets
|
||||
import torch
|
||||
import numpy as np
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.lm import SRLMArgs, SelfRetrievalLM
|
||||
from src.retrieval import Retriever, RetrievalArgs, TASK_CONFIG
|
||||
from src.utils.util import makedirs, pad_nested_lists, get_max_length_in_nested_lists, FileLogger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
import transformers
|
||||
# disable too long input warning
|
||||
transformers.logging.set_verbosity_error()
|
||||
|
||||
|
||||
# merge two args to get unified arguments
|
||||
@dataclass
|
||||
class LRLMArgs(RetrievalArgs, SRLMArgs):
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:chat/msc/test.json",
|
||||
metadata={'help': 'Evaluation file containing long texts.'}
|
||||
)
|
||||
lm_batch_size: int = field(
|
||||
default=1,
|
||||
metadata={'help': 'Evaluation batch size.'},
|
||||
)
|
||||
add_position_ids: bool = field(
|
||||
default=False,
|
||||
metadata={'help': 'Create position ids based on attention masks? Useful when training left-padded models with absolute position embeddings.'}
|
||||
)
|
||||
key_num: int = field(
|
||||
default=1,
|
||||
metadata={'help': 'How many chunks to retrieve at a time?'}
|
||||
)
|
||||
log_path: str = field(
|
||||
default="data/results/msc/msc.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
debug_retrieval: bool = field(
|
||||
default=False,
|
||||
metadata={'help': 'Check retrieval queries and values?'}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HistoryCollator:
|
||||
"""Collate histories, pad them, and return masks"""
|
||||
def __call__(self, batch_elem):
|
||||
first_elem = batch_elem[0]
|
||||
return_batch = {}
|
||||
|
||||
for key, value in first_elem.items():
|
||||
batch_value = [elem[key] for elem in batch_elem]
|
||||
if key == "history":
|
||||
longest = get_max_length_in_nested_lists(batch_value)
|
||||
batch_value, history_mask = pad_nested_lists(batch_value, longest, "", "right")
|
||||
history_mask = torch.tensor(history_mask, dtype=torch.bool)
|
||||
return_batch["history_mask"] = history_mask
|
||||
|
||||
elif key == "answers":
|
||||
# there is only one answer
|
||||
key = "answer"
|
||||
batch_value = [elem[0] for elem in batch_value]
|
||||
|
||||
elif key in ["query_id", "task"]:
|
||||
continue
|
||||
|
||||
# strip here for convenience
|
||||
return_batch[key] = np.char.strip(np.array(batch_value))
|
||||
return return_batch
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([LRLMArgs])
|
||||
args, = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
retriever = Retriever(
|
||||
retrieval_method=args.retrieval_method,
|
||||
# for dense retriever
|
||||
query_encoder=args.query_encoder,
|
||||
key_encoder=args.key_encoder,
|
||||
pooling_method=args.pooling_method,
|
||||
dense_metric=args.dense_metric,
|
||||
query_max_length=args.query_max_length,
|
||||
key_max_length=args.key_max_length,
|
||||
tie_encoders=args.tie_encoders,
|
||||
truncation_side=args.truncation_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
dtype=args.dtype,
|
||||
accelerator=accelerator,
|
||||
# for bm25 retriever
|
||||
anserini_dir=args.anserini_dir,
|
||||
k1=args.k1,
|
||||
b=args.b
|
||||
)
|
||||
|
||||
if args.add_instruction:
|
||||
instruction = TASK_CONFIG[args.version]["instruction"]["chat"]
|
||||
else:
|
||||
instruction = None
|
||||
|
||||
lm = SelfRetrievalLM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
retriever=retriever,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
context_window_size=args.context_window_size,
|
||||
chunk_size=args.chunk_size,
|
||||
key_num=args.key_num,
|
||||
chunk_batch_size=args.chunk_batch_size,
|
||||
retrieval_method=args.retrieval_method,
|
||||
order_method=args.order_method,
|
||||
integrate_method=args.integrate_method,
|
||||
instruction=instruction,
|
||||
debug_retrieval=args.debug_retrieval,
|
||||
add_sep=args.add_sep,
|
||||
accelerator=accelerator,
|
||||
)
|
||||
|
||||
logging.info(f"Loading data from {args.eval_data}...")
|
||||
|
||||
with accelerator.main_process_first():
|
||||
dataset = datasets.load_dataset("json", data_files=args.eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
|
||||
data_collator = HistoryCollator()
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
perplexity = lm.compute_perplexity(dataloader)
|
||||
metrics = {"perplexity": perplexity}
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
log_path = os.path.join(args.log_path)
|
||||
|
||||
file_logger = FileLogger(makedirs(log_path))
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,281 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import datasets
|
||||
from typing import List
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.lm import (
|
||||
LM,
|
||||
LMArgs,
|
||||
GenerationArgs
|
||||
)
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
RetrievalMetric,
|
||||
)
|
||||
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
|
||||
from .eval_retrieval import main as retrieval_main
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
PROPID_2_TEMPLATE = {
|
||||
22: "What is {}'s occupation?",
|
||||
218: "In what city was {} born?",
|
||||
91: "What genre is {}?",
|
||||
257: "Who is the father of {}?",
|
||||
182: "In what country is {}?",
|
||||
164: "Who was the producer of {}?",
|
||||
526: "Who was the director of {}?",
|
||||
97: "What is {} the capital of?",
|
||||
533: "Who was the screenwriter for {}?",
|
||||
639: "Who was the composer of {}?",
|
||||
472: "What color is {}?",
|
||||
106: "What is the religion of {}?",
|
||||
560: "What sport does {} play?",
|
||||
484: "Who is the author of {}?",
|
||||
292: "Who is the mother of {}?",
|
||||
422: "What is the capital of {}?"
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PopQAArgs(LMArgs, RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/popqa",
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:qa/popqa/test.json",
|
||||
metadata={'help': 'Path to the test file.'}
|
||||
)
|
||||
|
||||
few_shot: int = field(
|
||||
default=15,
|
||||
metadata={'help': 'How many few shot train samples?'},
|
||||
)
|
||||
|
||||
hits: int = field(
|
||||
default=10,
|
||||
metadata={'help': 'How many hits per query?'},
|
||||
)
|
||||
key_num: int = field(
|
||||
default=3,
|
||||
metadata={'help': 'How many docs to provide in prompt?'},
|
||||
)
|
||||
corpus: str = field(
|
||||
default="llm-embedder:qa/nq/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{title} {text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
key_max_length: int = field(
|
||||
default=128,
|
||||
metadata={'help': 'How many tokens at maximum in a key.'}
|
||||
)
|
||||
metrics: List[str] = field(
|
||||
default_factory=lambda: ["collate_key"],
|
||||
)
|
||||
save_to_output: bool = field(
|
||||
default=True,
|
||||
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/popqa/popqa.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerationArgs(GenerationArgs):
|
||||
max_new_tokens: int = field(
|
||||
default=16,
|
||||
metadata={'help': 'Maximum new tokens to generate.'}
|
||||
)
|
||||
eos_token_id: int = 13
|
||||
|
||||
|
||||
def process_popqa(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False):
|
||||
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
|
||||
has_bos = has_eos = False
|
||||
if test[0] == 1:
|
||||
has_bos = True
|
||||
if test[-1] == 1:
|
||||
has_eos = True
|
||||
|
||||
if few_shot > 0:
|
||||
assert train_data is not None
|
||||
assert few_shot // (len(PROPID_2_TEMPLATE) - 1), f"Make sure the number of few shot examples is a multiple of the template number!"
|
||||
train_dataset = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
|
||||
train_df = train_dataset.to_pandas()
|
||||
train_df = {k: v[:few_shot] for k, v in train_df.groupby("prop_id")}
|
||||
nshot_per_template = few_shot // (len(PROPID_2_TEMPLATE) - 1)
|
||||
|
||||
def _prepare_sample(query, obj=None, **kwds):
|
||||
sample = f"Q: {query} A:"
|
||||
if obj is not None:
|
||||
sample = sample + " " + obj
|
||||
return sample
|
||||
|
||||
def _prepare_retrieval(keys):
|
||||
if keys is not None:
|
||||
keys = keys[:key_num]
|
||||
keys = "\n".join(keys)
|
||||
keys = f"Knowledge: {keys}"
|
||||
else:
|
||||
keys = ""
|
||||
return keys
|
||||
|
||||
@DatasetProcessFn()
|
||||
def _process(query, query_id, prop_id, key=None, _index=None, **kwds):
|
||||
"""Yield keys and query with a prompt template"""
|
||||
output = {}
|
||||
query = query.strip()
|
||||
|
||||
knowledge = _prepare_retrieval(key)
|
||||
|
||||
train_samples_max_length = context_max_length - len(tokenizer.encode("\n\n" if len(knowledge) else "" + _prepare_sample(query), add_special_tokens=False)) - int(has_bos)
|
||||
|
||||
if few_shot > 0:
|
||||
train_samples = ""
|
||||
train_samples_length = 0
|
||||
|
||||
for k, df in train_df.items():
|
||||
# avoid contamination
|
||||
if k == prop_id:
|
||||
continue
|
||||
for sample in df.sample(nshot_per_template).iloc:
|
||||
train_sample = _prepare_sample(**sample) + "\n\n"
|
||||
# make sure the length of training samples does not exceed maximum length
|
||||
if train_samples_length + len(tokenizer.encode(train_sample)) > train_samples_max_length:
|
||||
break
|
||||
else:
|
||||
train_samples += train_sample
|
||||
train_samples_length += len(tokenizer.encode(train_sample))
|
||||
else:
|
||||
train_samples = ""
|
||||
|
||||
left = knowledge
|
||||
# \n\n to split retrieved knowledge
|
||||
right = "\n\n" + train_samples + _prepare_sample(query)
|
||||
|
||||
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
|
||||
|
||||
# strip spaces and \n in the head (when there is no retrieved passage)
|
||||
seq = tokenizer.decode(pair).strip()
|
||||
inputs = tokenizer(seq, return_token_type_ids=False)
|
||||
|
||||
if has_eos and not is_encoder_decoder:
|
||||
inputs = remove_eos(inputs, tokenizer.eos_token_id)
|
||||
|
||||
inputs["query_id"] = query_id
|
||||
|
||||
for k, v in inputs.items():
|
||||
output[k] = v
|
||||
return output
|
||||
return _process
|
||||
|
||||
|
||||
def evaluate_popqa(eval_data, save_path, **kwds):
|
||||
def compute_metric(eval_preds):
|
||||
makedirs(save_path)
|
||||
|
||||
samples = {}
|
||||
with open(eval_data) as f:
|
||||
for line in f:
|
||||
sample = json.loads(line.strip())
|
||||
samples[sample["query_id"]] = sample
|
||||
|
||||
accuracy = 0
|
||||
with open(save_path, "w") as f:
|
||||
for query_id, generation in zip(*eval_preds):
|
||||
sample = samples[query_id]
|
||||
answers = sample['possible_answers']
|
||||
correct = False
|
||||
for answer in answers:
|
||||
# if any answer matches
|
||||
if answer in generation or answer.lower() in generation or answer.capitalize() in generation:
|
||||
correct = True
|
||||
break
|
||||
|
||||
accuracy += int(correct)
|
||||
|
||||
sample["output"] = generation
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
|
||||
accuracy /= len(eval_preds[0])
|
||||
return {"accuracy": accuracy}
|
||||
return compute_metric
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([PopQAArgs, GenerationArgs])
|
||||
args, generation_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
# modify the output_dir for retrieval
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
args.output_dir = output_dir
|
||||
|
||||
if args.retrieval_method != "no":
|
||||
retrieval_main(args=args, accelerator=accelerator, log=False)
|
||||
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
|
||||
else:
|
||||
eval_data = args.eval_data
|
||||
|
||||
llm = LM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
accelerator=accelerator,
|
||||
generation_args=asdict(generation_args)
|
||||
)
|
||||
|
||||
tokenizer = llm.tokenizer
|
||||
|
||||
logging.info(f"Loading data from {eval_data}...")
|
||||
|
||||
with accelerator.main_process_first():
|
||||
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
dataset = dataset.map(process_popqa(
|
||||
tokenizer,
|
||||
context_max_length=args.context_max_length,
|
||||
key_num=args.key_num,
|
||||
few_shot=args.few_shot,
|
||||
# popqa extracts few-shot examples from test data
|
||||
train_data=args.eval_data,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
is_encoder_decoder=llm.model.config.is_encoder_decoder
|
||||
), remove_columns=dataset.column_names, batched=True, num_proc=32)
|
||||
|
||||
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
results = llm.generate(dataloader)
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
|
||||
metrics = evaluate_popqa(eval_data, result_path)(results)
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,261 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import datasets
|
||||
import random
|
||||
from typing import List
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.lm import (
|
||||
LM,
|
||||
LMArgs,
|
||||
GenerationArgs
|
||||
)
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
RetrievalMetric,
|
||||
)
|
||||
from src.utils.util import makedirs, remove_eos, normalize_text, DefaultDataCollator, DatasetProcessFn, FileLogger
|
||||
from .eval_retrieval import main as retrieval_main
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QAArgs(LMArgs, RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/qa/",
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:qa/nq/test.json",
|
||||
metadata={'help': 'Path to the test file.'}
|
||||
)
|
||||
lm_batch_size: int = field(
|
||||
default=4,
|
||||
metadata={'help': 'Evaluation batch size.'},
|
||||
)
|
||||
|
||||
few_shot: int = field(
|
||||
default=10,
|
||||
metadata={'help': 'How many few shot train samples?'},
|
||||
)
|
||||
train_data: str = field(
|
||||
default="llm-embedder:qa/nq/dev.json",
|
||||
metadata={'help': 'Path to the file containing training examples.'}
|
||||
)
|
||||
|
||||
hits: int = field(
|
||||
default=10,
|
||||
metadata={'help': 'How many hits per query?'},
|
||||
)
|
||||
key_num: int = field(
|
||||
default=3,
|
||||
metadata={'help': 'How many docs to provide in prompt?'},
|
||||
)
|
||||
corpus: str = field(
|
||||
default="llm-embedder:qa/nq/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{title} {text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
query_max_length: int = field(
|
||||
default=32,
|
||||
metadata={'help': 'How many tokens at maximum in a query.'}
|
||||
)
|
||||
key_max_length: int = field(
|
||||
default=128,
|
||||
metadata={'help': 'How many tokens at maximum in a key.'}
|
||||
)
|
||||
metrics: List[str] = field(
|
||||
default_factory=lambda: ["collate_key"],
|
||||
)
|
||||
save_to_output: bool = field(
|
||||
default=True,
|
||||
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/qa/qa.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerationArgs(GenerationArgs):
|
||||
max_new_tokens: int = field(
|
||||
default=32,
|
||||
metadata={'help': 'Maximum new tokens to generate.'}
|
||||
)
|
||||
eos_token_id: int = 13
|
||||
|
||||
|
||||
def process_qa(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False):
|
||||
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
|
||||
has_bos = has_eos = False
|
||||
if test[0] == 1:
|
||||
has_bos = True
|
||||
if test[-1] == 1:
|
||||
has_eos = True
|
||||
|
||||
if few_shot > 0:
|
||||
assert train_data is not None
|
||||
train_dataset = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
|
||||
sample_indices = random.sample(range(len(train_dataset)), few_shot)
|
||||
train_dataset = train_dataset.select(sample_indices)
|
||||
|
||||
def _prepare_sample(query, answers=None, **kwds):
|
||||
sample = f"Question: {query}\nAnswer:"
|
||||
if answers is not None:
|
||||
sample = sample + " " + random.choice(answers)
|
||||
return sample
|
||||
|
||||
def _prepare_retrieval(keys):
|
||||
if keys is not None:
|
||||
keys = keys[:key_num]
|
||||
keys = "\n".join(keys)
|
||||
keys = f"Knowledge: {keys}"
|
||||
else:
|
||||
keys = ""
|
||||
return keys
|
||||
|
||||
@DatasetProcessFn()
|
||||
def _process(query, query_id, key=None, **kwds):
|
||||
"""Yield keys and query with a prompt template"""
|
||||
output = {}
|
||||
query = query.strip()
|
||||
|
||||
knowledge = _prepare_retrieval(key)
|
||||
|
||||
train_samples_max_length = context_max_length - len(tokenizer.encode("\n\n" if len(knowledge) else "" + _prepare_sample(query), add_special_tokens=False)) - int(has_bos)
|
||||
|
||||
if few_shot > 0:
|
||||
train_samples = ""
|
||||
train_samples_length = 0
|
||||
|
||||
for i in range(few_shot):
|
||||
train_sample = train_dataset[i]
|
||||
train_sample = _prepare_sample(**train_sample) + "\n\n"
|
||||
if train_samples_length + len(tokenizer.encode(train_sample)) > train_samples_max_length:
|
||||
break
|
||||
else:
|
||||
train_samples += train_sample
|
||||
train_samples_length += len(tokenizer.encode(train_sample))
|
||||
else:
|
||||
train_samples = ""
|
||||
|
||||
left = knowledge
|
||||
# \n\n to split retrieved knowledge
|
||||
right = "\n\n" + train_samples + _prepare_sample(query)
|
||||
|
||||
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
|
||||
|
||||
# strip spaces and \n in the head (when there is no retrieved passage)
|
||||
seq = tokenizer.decode(pair).strip()
|
||||
inputs = tokenizer(seq, return_token_type_ids=False)
|
||||
|
||||
if has_eos and not is_encoder_decoder:
|
||||
inputs = remove_eos(inputs, tokenizer.eos_token_id)
|
||||
|
||||
inputs["query_id"] = query_id
|
||||
|
||||
for k, v in inputs.items():
|
||||
output[k] = v
|
||||
return output
|
||||
return _process
|
||||
|
||||
|
||||
def evaluate_qa(eval_data, save_path, **kwds):
|
||||
def compute_metric(eval_preds):
|
||||
makedirs(save_path)
|
||||
|
||||
samples = {}
|
||||
with open(eval_data) as f:
|
||||
for line in f:
|
||||
sample = json.loads(line.strip())
|
||||
samples[sample["query_id"]] = sample
|
||||
|
||||
exact_match = 0
|
||||
with open(save_path, "w") as f:
|
||||
for query_id, generation in zip(*eval_preds):
|
||||
sample = samples[query_id]
|
||||
em = max(normalize_text(generation) == normalize_text(answer) for answer in sample["answers"])
|
||||
exact_match += int(em)
|
||||
|
||||
sample["output"] = generation
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
|
||||
exact_match /= len(eval_preds[0])
|
||||
return {"exact_match": exact_match}
|
||||
return compute_metric
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([QAArgs, GenerationArgs])
|
||||
args, generation_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
# modify the output_dir for retrieval
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
args.output_dir = output_dir
|
||||
|
||||
if args.retrieval_method != "no":
|
||||
retrieval_main(args=args, accelerator=accelerator, log=False)
|
||||
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
|
||||
else:
|
||||
eval_data = args.eval_data
|
||||
|
||||
llm = LM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
accelerator=accelerator,
|
||||
generation_args=asdict(generation_args)
|
||||
)
|
||||
|
||||
tokenizer = llm.tokenizer
|
||||
|
||||
logging.info(f"Loading data from {eval_data}...")
|
||||
|
||||
with accelerator.main_process_first():
|
||||
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
dataset = dataset.map(process_qa(
|
||||
tokenizer,
|
||||
context_max_length=args.context_max_length,
|
||||
key_num=args.key_num,
|
||||
few_shot=args.few_shot,
|
||||
train_data=args.train_data,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
is_encoder_decoder=llm.model.config.is_encoder_decoder
|
||||
), remove_columns=dataset.column_names, batched=True, num_proc=32)
|
||||
|
||||
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
results = llm.generate(dataloader)
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
|
||||
metrics = evaluate_qa(eval_data, result_path)(results)
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,235 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import datasets
|
||||
import random
|
||||
from typing import List
|
||||
from accelerate import Accelerator
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.lm import (
|
||||
LM,
|
||||
LMArgs,
|
||||
GenerationArgs
|
||||
)
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
RetrievalMetric,
|
||||
)
|
||||
from src.utils.util import makedirs, remove_eos, normalize_text, DefaultDataCollator, DatasetProcessFn, FileLogger
|
||||
from .eval_retrieval import main as retrieval_main
|
||||
from .icl_utils import compute_metrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QRECCArgs(LMArgs, RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/qrecc",
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:convsearch/qrecc/test.concat.json",
|
||||
metadata={'help': 'Query jsonl.'}
|
||||
)
|
||||
corpus: str = field(
|
||||
default="llm-embedder:convsearch/qrecc/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
do_generate: bool = field(
|
||||
default=False,
|
||||
metadata={'help': 'Generate for computing qa metrics?'}
|
||||
)
|
||||
|
||||
hits: int = field(
|
||||
default=100,
|
||||
metadata={'help': 'How many hits per query?'},
|
||||
)
|
||||
key_num: int = field(
|
||||
default=3,
|
||||
metadata={'help': 'How many docs to provide in prompt?'},
|
||||
)
|
||||
metrics: List[str] = field(
|
||||
default_factory=lambda: ["ndcg", "recall", "collate_key"],
|
||||
)
|
||||
cutoffs: List[int] = field(
|
||||
default_factory=lambda: [3, 10, 100],
|
||||
metadata={'help': 'Cutoffs to evaluate retrieval metrics.'}
|
||||
)
|
||||
max_neg_num: int = field(
|
||||
default=32,
|
||||
metadata={'help': 'Maximum negative number to mine.'}
|
||||
)
|
||||
save_to_output: bool = field(
|
||||
default=True,
|
||||
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
|
||||
)
|
||||
|
||||
log_path: str = field(
|
||||
default="data/results/qrecc/qrecc.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerationArgs(GenerationArgs):
|
||||
max_new_tokens: int = field(
|
||||
default=128,
|
||||
metadata={'help': 'Maximum new tokens to generate.'}
|
||||
)
|
||||
eos_token_id: int = 13
|
||||
|
||||
|
||||
def process_qrecc(tokenizer, context_max_length=2048, key_num=3, is_encoder_decoder=False):
|
||||
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
|
||||
has_bos = has_eos = False
|
||||
if test[0] == 1:
|
||||
has_bos = True
|
||||
if test[-1] == 1:
|
||||
has_eos = True
|
||||
|
||||
def _prepare_sample(query, answers=None, **kwds):
|
||||
sample = f"Context and Question: {query}\nAnswer:"
|
||||
if answers is not None:
|
||||
sample = sample + " " + random.choice(answers)
|
||||
return sample
|
||||
|
||||
def _prepare_retrieval(keys):
|
||||
if keys is not None:
|
||||
keys = keys[:key_num]
|
||||
keys = "\n".join(keys)
|
||||
knowledge = f"Knowledge: {keys}"
|
||||
else:
|
||||
knowledge = ""
|
||||
return knowledge
|
||||
|
||||
@DatasetProcessFn()
|
||||
def _process(query, query_id, key=None, **kwds):
|
||||
"""Yield keys and query with a prompt template"""
|
||||
output = {}
|
||||
query = query.strip()
|
||||
knowledge = _prepare_retrieval(key)
|
||||
|
||||
left = knowledge
|
||||
# \n\n to split retrieved knowledge
|
||||
right = "\n\n" + _prepare_sample(query)
|
||||
|
||||
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
|
||||
|
||||
# strip spaces and \n in the head (when there is no retrieved passage)
|
||||
seq = tokenizer.decode(pair).strip()
|
||||
inputs = tokenizer(seq, return_token_type_ids=False)
|
||||
|
||||
if has_eos and not is_encoder_decoder:
|
||||
inputs = remove_eos(inputs, tokenizer.eos_token_id)
|
||||
|
||||
inputs["query_id"] = query_id
|
||||
|
||||
for k, v in inputs.items():
|
||||
output[k] = v
|
||||
return output
|
||||
return _process
|
||||
|
||||
|
||||
def evaluate_qrecc(eval_data, save_path, **kwds):
|
||||
def compute_metric(eval_preds):
|
||||
makedirs(save_path)
|
||||
|
||||
samples = {}
|
||||
with open(eval_data) as f:
|
||||
for line in f:
|
||||
sample = json.loads(line.strip())
|
||||
samples[sample["query_id"]] = sample["answers"][0]
|
||||
|
||||
preds = []
|
||||
answers = []
|
||||
with open(save_path, "w") as f:
|
||||
for query_id, generation in zip(*eval_preds):
|
||||
answer = samples[query_id]
|
||||
preds.append(generation)
|
||||
answers.append(answer)
|
||||
|
||||
sample["output"] = generation
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
|
||||
rouge_l = compute_metrics("rl", labels=answers, preds=preds)
|
||||
return rouge_l
|
||||
return compute_metric
|
||||
|
||||
|
||||
def main():
|
||||
parser = HfArgumentParser([QRECCArgs, GenerationArgs])
|
||||
args, generation_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
# modify the output_dir for retrieval
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
args.output_dir = output_dir
|
||||
|
||||
if args.retrieval_method != "no":
|
||||
# retrieval metrics computes ndcg and recall
|
||||
_, _, metrics = retrieval_main(args=args, accelerator=accelerator, log=False)
|
||||
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
|
||||
else:
|
||||
eval_data = args.eval_data
|
||||
metrics = {}
|
||||
|
||||
if args.do_generate:
|
||||
llm = LM(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
dtype=args.lm_dtype,
|
||||
device_map=args.lm_device_map,
|
||||
padding_side=args.padding_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
accelerator=accelerator,
|
||||
generation_args=asdict(generation_args)
|
||||
)
|
||||
|
||||
tokenizer = llm.tokenizer
|
||||
|
||||
logging.info(f"Loading data from {eval_data}...")
|
||||
|
||||
with accelerator.main_process_first():
|
||||
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
|
||||
dataset = dataset.map(process_qrecc(
|
||||
tokenizer,
|
||||
context_max_length=args.context_max_length,
|
||||
key_num=args.key_num,
|
||||
is_encoder_decoder=llm.model.config.is_encoder_decoder
|
||||
), remove_columns=dataset.column_names, batched=True, num_proc=32)
|
||||
|
||||
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=args.lm_batch_size,
|
||||
collate_fn=data_collator,
|
||||
pin_memory=True,
|
||||
)
|
||||
dataloader = accelerator.prepare(dataloader)
|
||||
|
||||
results = llm.generate(dataloader)
|
||||
if accelerator.process_index == 0:
|
||||
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
|
||||
lm_metrics = evaluate_qrecc(eval_data, result_path)(results)
|
||||
|
||||
else:
|
||||
lm_metrics = {}
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
metrics.update(lm_metrics)
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,163 @@
|
||||
import os
|
||||
import torch
|
||||
import logging
|
||||
import datasets
|
||||
from typing import List
|
||||
from accelerate import Accelerator
|
||||
from transformers import HfArgumentParser
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
Retriever,
|
||||
RetrievalDataset,
|
||||
RetrievalMetric,
|
||||
TASK_CONFIG,
|
||||
)
|
||||
from src.utils.util import makedirs, FileLogger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Args(RetrievalArgs):
|
||||
eval_data: str = field(
|
||||
default=None,
|
||||
metadata={'help': 'Query jsonl.'}
|
||||
)
|
||||
output_dir: str = field(
|
||||
default="data/outputs/",
|
||||
)
|
||||
corpus: str = field(
|
||||
default=None,
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{title} {text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
log_path: str = field(
|
||||
default="data/results/performance.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
def main(args, accelerator=None, log=True):
|
||||
if accelerator is None:
|
||||
accelerator = Accelerator(cpu=args.cpu)
|
||||
|
||||
with accelerator.main_process_first():
|
||||
config = TASK_CONFIG[args.version]
|
||||
instruction = config["instruction"]
|
||||
|
||||
# we should get the evaluation task before specifying instruction
|
||||
# NOTE: only dense retrieval needs instruction
|
||||
if args.eval_data is not None and args.add_instruction and args.retrieval_method == "dense":
|
||||
raw_eval_dataset = datasets.load_dataset('json', data_files=args.eval_data, split='train', cache_dir=args.dataset_cache_dir)
|
||||
eval_task = raw_eval_dataset[0]["task"]
|
||||
else:
|
||||
eval_task = None
|
||||
|
||||
eval_dataset = RetrievalDataset.prepare_eval_dataset(
|
||||
data_file=args.eval_data,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
instruction=instruction[eval_task] if eval_task is not None else None,
|
||||
)
|
||||
corpus = RetrievalDataset.prepare_corpus(
|
||||
data_file=args.corpus,
|
||||
key_template=args.key_template,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
instruction=instruction[eval_task] if eval_task is not None else None
|
||||
)
|
||||
|
||||
result_path = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="result", save_name=args.save_name)
|
||||
|
||||
if args.load_result:
|
||||
query_ids, preds = RetrievalMetric._load_result(result_path)
|
||||
|
||||
else:
|
||||
retriever = Retriever(
|
||||
retrieval_method=args.retrieval_method,
|
||||
# for dense retriever
|
||||
query_encoder=args.query_encoder,
|
||||
key_encoder=args.key_encoder,
|
||||
pooling_method=args.pooling_method,
|
||||
dense_metric=args.dense_metric,
|
||||
query_max_length=args.query_max_length,
|
||||
key_max_length=args.key_max_length,
|
||||
tie_encoders=args.tie_encoders,
|
||||
truncation_side=args.truncation_side,
|
||||
cache_dir=args.model_cache_dir,
|
||||
dtype=args.dtype,
|
||||
accelerator=accelerator,
|
||||
# for bm25 retriever
|
||||
anserini_dir=args.anserini_dir,
|
||||
k1=args.k1,
|
||||
b=args.b
|
||||
)
|
||||
|
||||
retriever.index(
|
||||
corpus,
|
||||
output_dir=args.output_dir,
|
||||
# for dense retriever
|
||||
embedding_name=args.embedding_name,
|
||||
index_factory=args.faiss_index_factory,
|
||||
load_encode=args.load_encode,
|
||||
save_encode=args.save_encode,
|
||||
load_index=args.load_index,
|
||||
save_index=args.save_index,
|
||||
batch_size=args.batch_size,
|
||||
# for bm25 retriever
|
||||
threads=args.threads,
|
||||
language=args.language,
|
||||
storeDocvectors=args.storeDocvectors,
|
||||
load_collection=args.load_collection,
|
||||
)
|
||||
|
||||
query_ids, preds = retriever.search(
|
||||
eval_dataset=eval_dataset,
|
||||
hits=args.hits,
|
||||
# for dense retriever
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
|
||||
del retriever
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if args.save_result and accelerator.process_index == 0:
|
||||
RetrievalMetric._save_result(query_ids, preds, result_path)
|
||||
|
||||
if accelerator.process_index == 0:
|
||||
# NOTE: this corpus is for computing metrics, where no instruction is given
|
||||
no_instruction_corpus = RetrievalDataset.prepare_corpus(
|
||||
data_file=args.corpus,
|
||||
key_template=args.key_template,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
)
|
||||
|
||||
metrics = RetrievalMetric.get_metric_fn(
|
||||
args.metrics,
|
||||
cutoffs=args.cutoffs,
|
||||
eval_data=args.eval_data,
|
||||
corpus=no_instruction_corpus,
|
||||
save_name=args.save_name,
|
||||
output_dir=args.output_dir,
|
||||
save_to_output=args.save_to_output,
|
||||
max_neg_num=args.max_neg_num,
|
||||
cache_dir=args.dataset_cache_dir,
|
||||
filter_answers=args.filter_answers,
|
||||
)(query_ids, preds)
|
||||
|
||||
if log:
|
||||
file_logger = FileLogger(makedirs(args.log_path))
|
||||
file_logger.log(metrics, Args=asdict(args))
|
||||
else:
|
||||
metrics = {}
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
return query_ids, preds, metrics
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = HfArgumentParser([Args])
|
||||
args, = parser.parse_args_into_dataclasses()
|
||||
main(args)
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
import logging
|
||||
from typing import List
|
||||
from dataclasses import dataclass, field
|
||||
from transformers import HfArgumentParser
|
||||
from src.retrieval import (
|
||||
RetrievalArgs,
|
||||
)
|
||||
from .eval_retrieval import main
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolArgs(RetrievalArgs):
|
||||
output_dir: str = field(
|
||||
default="data/results/tool",
|
||||
)
|
||||
eval_data: str = field(
|
||||
default="llm-embedder:tool/toolbench/test.json",
|
||||
metadata={'help': 'Query jsonl.'}
|
||||
)
|
||||
corpus: str = field(
|
||||
default="llm-embedder:tool/toolbench/corpus.json",
|
||||
metadata={'help': 'Corpus path for retrieval.'}
|
||||
)
|
||||
key_template: str = field(
|
||||
default="{text}",
|
||||
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
|
||||
)
|
||||
|
||||
cutoffs: List[int] = field(
|
||||
default_factory=lambda: [1,3,5],
|
||||
metadata={'help': 'Cutoffs to evaluate retrieval metrics.'}
|
||||
)
|
||||
max_neg_num: int = field(
|
||||
default=32,
|
||||
metadata={'help': 'Maximum negative number to mine.'}
|
||||
)
|
||||
log_path: str = field(
|
||||
default="data/results/tool/toolbench.log",
|
||||
metadata={'help': 'Path to the file for logging.'}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = HfArgumentParser([ToolArgs])
|
||||
args, = parser.parse_args_into_dataclasses()
|
||||
if args.retrieval_method == "dense":
|
||||
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
|
||||
args.output_dir = output_dir
|
||||
else:
|
||||
output_dir = os.path.join(args.output_dir, args.retrieval_method)
|
||||
main(args)
|
||||
@@ -0,0 +1,296 @@
|
||||
import collections
|
||||
import re
|
||||
import string
|
||||
import copy
|
||||
import logging
|
||||
import numpy as np
|
||||
from sklearn.metrics import f1_score
|
||||
from typing import List, Dict
|
||||
from rouge import Rouge
|
||||
from transformers.tokenization_utils import PreTrainedTokenizer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_answer(text, punc_chars, punc_repl):
|
||||
"""Lower text and remove punctuation, articles and extra whitespace."""
|
||||
|
||||
def remove_articles(s):
|
||||
return re.sub(r"\b(a|an|the)\b", " ", s)
|
||||
|
||||
def replace_punctuation(s):
|
||||
to_replace = set(punc_chars)
|
||||
return "".join(punc_repl if ch in to_replace else ch for ch in s)
|
||||
|
||||
def white_space_fix(s):
|
||||
return " ".join(s.split())
|
||||
|
||||
text = text.lower()
|
||||
text = replace_punctuation(text)
|
||||
text = remove_articles(text)
|
||||
text = white_space_fix(text)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_squad(answer):
|
||||
"""Normalization used in official SQuAD evaluation script."""
|
||||
return _normalize_answer(answer, punc_chars=string.punctuation, punc_repl="")
|
||||
|
||||
|
||||
def _metric_max_over_ground_truths(metric_fn, ground_truths, prediction):
|
||||
"""Computes the maximum of the metric over all ground truths."""
|
||||
return max(
|
||||
metric_fn(ground_truth, prediction) for ground_truth in ground_truths
|
||||
)
|
||||
|
||||
|
||||
def _exact_match_score(target, prediction):
|
||||
return target == prediction
|
||||
|
||||
|
||||
def _f1_score(target, prediction):
|
||||
"""Computes token f1 score for a single target and prediction."""
|
||||
prediction_tokens = prediction.split()
|
||||
target_tokens = target.split()
|
||||
common = (collections.Counter(prediction_tokens) &
|
||||
collections.Counter(target_tokens))
|
||||
num_same = sum(common.values())
|
||||
if num_same == 0:
|
||||
return 0
|
||||
precision = 1.0 * num_same / len(prediction_tokens)
|
||||
recall = 1.0 * num_same / len(target_tokens)
|
||||
f1 = (2 * precision * recall) / (precision + recall)
|
||||
return f1
|
||||
|
||||
def qa_metrics(targets, predictions, return_list=False):
|
||||
"""Computes exact match and f1 QA scores, expecting pre-normalized text."""
|
||||
if len(targets) != len(predictions):
|
||||
raise ValueError("Number of targets and predictions must match.")
|
||||
if return_list:
|
||||
em=[
|
||||
_metric_max_over_ground_truths(_exact_match_score, t, p)
|
||||
for p, t in zip(predictions, targets)
|
||||
]
|
||||
f1=[
|
||||
_metric_max_over_ground_truths(_f1_score, t, p)
|
||||
for p, t in zip(predictions, targets)
|
||||
]
|
||||
return em, f1
|
||||
em = np.mean([
|
||||
_metric_max_over_ground_truths(_exact_match_score, t, p)
|
||||
for p, t in zip(predictions, targets)
|
||||
])
|
||||
f1 = np.mean([
|
||||
_metric_max_over_ground_truths(_f1_score, t, p)
|
||||
for p, t in zip(predictions, targets)
|
||||
])
|
||||
# em *= 100
|
||||
# f1 *= 100
|
||||
logger.info("EM = %.2f, F1 = %.2f", em, f1)
|
||||
#return {"em": em, "f1": f1}
|
||||
return em, f1
|
||||
|
||||
|
||||
class App:
|
||||
def __init__(self):
|
||||
self.functions = {}
|
||||
|
||||
def add(self, key):
|
||||
def adder(func):
|
||||
self.functions[key] = func
|
||||
return func
|
||||
|
||||
return adder
|
||||
|
||||
def __getitem__(self, __name: str):
|
||||
return self.functions[__name]
|
||||
|
||||
|
||||
metric_dict = App()
|
||||
|
||||
|
||||
@metric_dict.add("rouge")
|
||||
def rouge(preds, labels, return_list=False):
|
||||
# https://github.com/pltrdy/rouge
|
||||
r1s, r2s, rls = [], [], []
|
||||
r = Rouge()
|
||||
for i in range(len(labels)):
|
||||
if "\n" not in preds[i]:
|
||||
preds[i] += "\n" # to ensure rouge metrics
|
||||
if "\n" not in labels[i]:
|
||||
labels[i] += "\n"
|
||||
scores = r.get_scores(preds[i], labels[i])[0]
|
||||
r1s.append(scores["rouge-1"]["f"])
|
||||
r2s.append(scores["rouge-2"]["f"])
|
||||
rls.append(scores["rouge-l"]["f"])
|
||||
if return_list: # used for scoring data
|
||||
return r1s
|
||||
r1 = sum(r1s) / len(r1s)
|
||||
r2 = sum(r2s) / len(r2s)
|
||||
rl = sum(rls) / len(rls)
|
||||
return r1, r2, rl
|
||||
|
||||
|
||||
@metric_dict.add("squad")
|
||||
def squad(labels, preds, return_list=False):
|
||||
"""Computes SQuAD metrics, maximizing over answers per question.
|
||||
Args:
|
||||
labels: list of lists of strings
|
||||
preds: list of strings
|
||||
Returns:
|
||||
dict with score_key: squad score across all labels and predictions
|
||||
"""
|
||||
labels = [[normalize_squad(t) for t in u] for u in labels]
|
||||
preds = [normalize_squad(p) for p in preds]
|
||||
if return_list: # used for scoring data
|
||||
em, f1 = qa_metrics(labels, preds, return_list=True)
|
||||
return f1
|
||||
em, f1 = qa_metrics(labels, preds) # em,f1
|
||||
return em, f1
|
||||
|
||||
|
||||
|
||||
@metric_dict.add("simple_accuracy")
|
||||
def simple_accuracy(preds, labels, return_list=False):
|
||||
if isinstance(preds[0], str):
|
||||
labels = [label.strip() for label in labels]
|
||||
preds = [pred.strip() for pred in preds]
|
||||
res = [int(preds[i] == labels[i]) for i in range(len(preds))]
|
||||
if return_list:
|
||||
return res
|
||||
acc = sum(res) / len(res)
|
||||
return acc
|
||||
|
||||
|
||||
def compute_metrics(metric, labels, preds):
|
||||
assert len(preds) == len(labels)
|
||||
if metric == "acc":
|
||||
return {"acc": simple_accuracy(preds, labels)}
|
||||
elif metric == "rl":
|
||||
r1, r2, rl = rouge(preds, labels)
|
||||
# return {"r1": r1, "r2": r2, "rl": rl}
|
||||
return {"rl": rl}
|
||||
elif metric == "f1":
|
||||
f1 = f1_score(y_true=labels, y_pred=preds, pos_label='1')
|
||||
return {"f1": f1}
|
||||
elif metric == "em":
|
||||
em, f1 = squad(labels=labels, preds=preds)
|
||||
# return {"em": em, "f1": f1}
|
||||
return {"em": em}
|
||||
|
||||
def compute_scores(metric, preds, labels):
|
||||
if not isinstance(preds[0], str):
|
||||
preds = np.array(preds)
|
||||
labels = np.array(labels)
|
||||
scores = compute_metrics(metric, labels=labels, preds=preds)
|
||||
return scores
|
||||
|
||||
def flat_options(data):
|
||||
flat_data = []
|
||||
for e in data:
|
||||
for option in e['options']:
|
||||
flat_data.append({"query":e['query'], "few_shot":e['few_shot'], 'input_answer':option})
|
||||
return flat_data
|
||||
|
||||
def perplexity_to_choice(data, perplexity):
|
||||
inx = 0
|
||||
results = []
|
||||
for e in data:
|
||||
cur_perplexity = []
|
||||
for _ in e['options']:
|
||||
cur_perplexity.append(perplexity[inx])
|
||||
inx += 1
|
||||
ans = np.argmin(cur_perplexity)
|
||||
results.append(str(ans))
|
||||
return results
|
||||
|
||||
|
||||
def get_length(tokenizer, text):
|
||||
tokenized_example = tokenizer.encode_plus(text,truncation=False, return_tensors='pt')
|
||||
shape = tokenized_example.input_ids.squeeze().shape
|
||||
if len(shape)==0:
|
||||
return 1
|
||||
else:
|
||||
return int(shape[0])
|
||||
|
||||
|
||||
def get_prompt_length(tokenizer, prompts_list, question, n_tokens_in_prompt: int=1024):
|
||||
lengths_list = [get_length(tokenizer, prompt) for prompt in prompts_list]
|
||||
q_length = get_length(tokenizer, question)
|
||||
max_prompts = np.searchsorted(np.cumsum(lengths_list), n_tokens_in_prompt - q_length)
|
||||
return max_prompts
|
||||
|
||||
|
||||
def _llm_generation_func(examples: Dict[str, List],
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
example_num: int=8,
|
||||
max_input_tokens: int=1024,
|
||||
add_llama_inst: bool=False):
|
||||
texts = []
|
||||
n_tokens_in_prompt = max_input_tokens
|
||||
if add_llama_inst:
|
||||
n_tokens_in_prompt -= 8
|
||||
|
||||
for i in range(len(examples['query'])):
|
||||
prompts_list = examples['few_shot'][i][::-1]
|
||||
max_prompts = get_prompt_length(
|
||||
tokenizer=tokenizer,
|
||||
prompts_list=prompts_list,
|
||||
question=examples['query'][i],
|
||||
n_tokens_in_prompt=n_tokens_in_prompt
|
||||
)
|
||||
example_num = min(example_num, max_prompts)
|
||||
|
||||
inputs = prompts_list[:example_num]
|
||||
|
||||
inputs.append(examples['query'][i])
|
||||
|
||||
if add_llama_inst:
|
||||
inputs = "[INST] " + "\n".join(inputs) + " [/INST]"
|
||||
else:
|
||||
inputs = "\n".join(inputs)+'\n'
|
||||
|
||||
texts.append(inputs)
|
||||
return tokenizer(texts, return_tensors="pt", padding=True, max_length=1024, return_token_type_ids=False)
|
||||
|
||||
|
||||
def _llm_perplexity_func(examples: Dict[str, List],
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
example_num: int=8,
|
||||
max_input_tokens: int=1024,
|
||||
add_llama_inst: bool=False):
|
||||
texts = []
|
||||
answers = []
|
||||
n_tokens_in_prompt = max_input_tokens
|
||||
if add_llama_inst:
|
||||
n_tokens_in_prompt -= 8
|
||||
|
||||
for i in range(len(examples['query'])):
|
||||
prompts_list = examples['few_shot'][i][::-1]
|
||||
max_prompts = get_prompt_length(tokenizer=tokenizer,
|
||||
prompts_list=prompts_list,
|
||||
question=examples['query'][i],
|
||||
n_tokens_in_prompt=n_tokens_in_prompt)
|
||||
example_num = min(example_num, max_prompts)
|
||||
|
||||
inputs = prompts_list[:example_num]
|
||||
|
||||
inputs.append(examples['query'][i])
|
||||
if add_llama_inst:
|
||||
# NOTE: two more spaces after [/INST]
|
||||
inputs = "[INST] " + "\n".join(inputs) + " [/INST] " + examples['input_answer'][i].lstrip()
|
||||
else:
|
||||
inputs = "\n".join(inputs)+'\n ' + examples['input_answer'][i] # add a space after \n to split input and answer
|
||||
|
||||
texts.append(inputs)
|
||||
answers.append(examples['input_answer'][i])
|
||||
|
||||
inputs = tokenizer(texts, return_tensors="pt", padding=True, return_token_type_ids=False)
|
||||
|
||||
labels = copy.deepcopy(inputs['input_ids'])
|
||||
for i, ans in enumerate(answers):
|
||||
ans_ids = tokenizer.encode(ans, add_special_tokens=False)
|
||||
labels[i][:-len(ans_ids)] = -100
|
||||
|
||||
inputs['labels'] = labels
|
||||
return inputs
|
||||
Reference in New Issue
Block a user