chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,574 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Reranker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Reranker is disigned in cross-encoder architecture that takes the query and text at the same time and directly output their score of similarity. It is more capable on scoring the query-text relevance, but with the tradeoff of slower speed. Thus, complete retrieval system usually contains retrievers in the first stage to do a large scope retrieval, and then follows by rerankers to rerank the results more precisely.\n",
|
||||
"\n",
|
||||
"In this tutorial, we will go through text retrieval pipeline with reranker and evaluate the results before and after reranking.\n",
|
||||
"\n",
|
||||
"Note: Step 1-4 are identical to the tutorial of [evaluation](https://github.com/FlagOpen/FlagEmbedding/tree/master/Tutorials/4_Evaluation). We suggest to first go through that if you are not familiar with retrieval."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 0. Setup"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Install the dependencies in the environment."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U FlagEmbedding faiss-cpu"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Download and preprocess the MS Marco dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datasets import load_dataset\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"data = load_dataset(\"namespace-Pt/msmarco\", split=\"dev\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"queries = np.array(data[:100][\"query\"])\n",
|
||||
"corpus = sum(data[:5000][\"positive\"], [])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Embedding"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Inference Embeddings: 100%|██████████| 21/21 [01:59<00:00, 5.68s/it]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"shape of the corpus embeddings: (5331, 768)\n",
|
||||
"data type of the embeddings: float32\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import FlagModel\n",
|
||||
"\n",
|
||||
"# get the BGE embedding model\n",
|
||||
"model = FlagModel('BAAI/bge-base-en-v1.5',\n",
|
||||
" query_instruction_for_retrieval=\"Represent this sentence for searching relevant passages:\",\n",
|
||||
" use_fp16=True)\n",
|
||||
"\n",
|
||||
"# get the embedding of the corpus\n",
|
||||
"corpus_embeddings = model.encode(corpus)\n",
|
||||
"\n",
|
||||
"print(\"shape of the corpus embeddings:\", corpus_embeddings.shape)\n",
|
||||
"print(\"data type of the embeddings: \", corpus_embeddings.dtype)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Indexing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"total number of vectors: 5331\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import faiss\n",
|
||||
"\n",
|
||||
"# get the length of our embedding vectors, vectors by bge-base-en-v1.5 have length 768\n",
|
||||
"dim = corpus_embeddings.shape[-1]\n",
|
||||
"\n",
|
||||
"# create the faiss index and store the corpus embeddings into the vector space\n",
|
||||
"index = faiss.index_factory(dim, 'Flat', faiss.METRIC_INNER_PRODUCT)\n",
|
||||
"corpus_embeddings = corpus_embeddings.astype(np.float32)\n",
|
||||
"index.train(corpus_embeddings)\n",
|
||||
"index.add(corpus_embeddings)\n",
|
||||
"\n",
|
||||
"print(f\"total number of vectors: {index.ntotal}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Retrieval"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_embeddings = model.encode_queries(queries)\n",
|
||||
"ground_truths = [d[\"positive\"] for d in data]\n",
|
||||
"corpus = np.asarray(corpus)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Searching: 100%|██████████| 1/1 [00:00<00:00, 22.35it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from tqdm import tqdm\n",
|
||||
"\n",
|
||||
"res_scores, res_ids, res_text = [], [], []\n",
|
||||
"query_size = len(query_embeddings)\n",
|
||||
"batch_size = 256\n",
|
||||
"# The cutoffs we will use during evaluation, and set k to be the maximum of the cutoffs.\n",
|
||||
"cut_offs = [1, 10]\n",
|
||||
"k = max(cut_offs)\n",
|
||||
"\n",
|
||||
"for i in tqdm(range(0, query_size, batch_size), desc=\"Searching\"):\n",
|
||||
" q_embedding = query_embeddings[i: min(i+batch_size, query_size)].astype(np.float32)\n",
|
||||
" # search the top k answers for each of the queries\n",
|
||||
" score, idx = index.search(q_embedding, k=k)\n",
|
||||
" res_scores += list(score)\n",
|
||||
" res_ids += list(idx)\n",
|
||||
" res_text += list(corpus[idx])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Reranking"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now we will use a reranker to rerank the list of answers we retrieved using our index. Hopefully, this will lead to better results."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The following table lists the available BGE rerankers. Feel free to try out to see their differences!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"| Model | Language | Parameters | Description | Base Model |\n",
|
||||
"|:-------|:--------:|:----:|:-----------------:|:--------------------------------------:|\n",
|
||||
"| [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) | Multilingual | 568M | a lightweight cross-encoder model, possesses strong multilingual capabilities, easy to deploy, with fast inference. | XLM-RoBERTa-Large |\n",
|
||||
"| [BAAI/bge-reranker-v2-gemma](https://huggingface.co/BAAI/bge-reranker-v2-gemma) | Multilingual | 2.51B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English proficiency and multilingual capabilities. | Gemma2-2B |\n",
|
||||
"| [BAAI/bge-reranker-v2-minicpm-layerwise](https://huggingface.co/BAAI/bge-reranker-v2-minicpm-layerwise) | Multilingual | 2.72B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English and Chinese proficiency, allows freedom to select layers for output, facilitating accelerated inference. | MiniCPM |\n",
|
||||
"| [BAAI/bge-reranker-v2.5-gemma2-lightweight](https://huggingface.co/BAAI/bge-reranker-v2.5-gemma2-lightweight) | Multilingual | 9.24B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English and Chinese proficiency, allows freedom to select layers, compress ratio and compress layers for output, facilitating accelerated inference. | Gemma2-9B |\n",
|
||||
"| [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | Chinese and English | 560M | a cross-encoder model which is more accurate but less efficient | XLM-RoBERTa-Large |\n",
|
||||
"| [BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base) | Chinese and English | 278M | a cross-encoder model which is more accurate but less efficient | XLM-RoBERTa-Base |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"First, let's use a small example to see how reranker works:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[-9.474676132202148, -2.823843240737915, 5.76226806640625]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import FlagReranker\n",
|
||||
"\n",
|
||||
"reranker = FlagReranker('BAAI/bge-reranker-large', use_fp16=True) \n",
|
||||
"# Setting use_fp16 to True speeds up computation with a slight performance degradation\n",
|
||||
"\n",
|
||||
"# use the compute_score() function to calculate scores for each input sentence pair\n",
|
||||
"scores = reranker.compute_score([\n",
|
||||
" ['what is panda?', 'Today is a sunny day'], \n",
|
||||
" ['what is panda?', 'The tiger (Panthera tigris) is a member of the genus Panthera and the largest living cat species native to Asia.'],\n",
|
||||
" ['what is panda?', 'The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.']\n",
|
||||
" ])\n",
|
||||
"print(scores)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, let's use the reranker to rerank our previously retrieved results:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"new_ids, new_scores, new_text = [], [], []\n",
|
||||
"for i in range(len(queries)):\n",
|
||||
" # get the new scores of the previously retrieved results\n",
|
||||
" new_score = reranker.compute_score([[queries[i], text] for text in res_text[i]])\n",
|
||||
" # sort the lists of ids and scores by the new scores\n",
|
||||
" new_id = [tup[1] for tup in sorted(list(zip(new_score, res_ids[i])), reverse=True)]\n",
|
||||
" new_scores.append(sorted(new_score, reverse=True))\n",
|
||||
" new_ids.append(new_id)\n",
|
||||
" new_text.append(corpus[new_id])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Evaluate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"For details of these metrics, please checkout the tutorial of [evaluation](https://github.com/FlagOpen/FlagEmbedding/tree/master/Tutorials/4_Evaluation)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 6.1 Recall"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 26,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def calc_recall(preds, truths, cutoffs):\n",
|
||||
" recalls = np.zeros(len(cutoffs))\n",
|
||||
" for text, truth in zip(preds, truths):\n",
|
||||
" for i, c in enumerate(cutoffs):\n",
|
||||
" recall = np.intersect1d(truth, text[:c])\n",
|
||||
" recalls[i] += len(recall) / max(min(len(recall), len(truth)), 1)\n",
|
||||
" recalls /= len(preds)\n",
|
||||
" return recalls"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Before reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"recall@1:\t0.97\n",
|
||||
"recall@10:\t1.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"recalls_init = calc_recall(res_text, ground_truths, cut_offs)\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" print(f\"recall@{c}:\\t{recalls_init[i]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"recall@1:\t0.99\n",
|
||||
"recall@10:\t1.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"recalls_rerank = calc_recall(new_text, ground_truths, cut_offs)\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" print(f\"recall@{c}:\\t{recalls_rerank[i]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 6.2 MRR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def MRR(preds, truth, cutoffs):\n",
|
||||
" mrr = [0 for _ in range(len(cutoffs))]\n",
|
||||
" for pred, t in zip(preds, truth):\n",
|
||||
" for i, c in enumerate(cutoffs):\n",
|
||||
" for j, p in enumerate(pred):\n",
|
||||
" if j < c and p in t:\n",
|
||||
" mrr[i] += 1/(j+1)\n",
|
||||
" break\n",
|
||||
" mrr = [k/len(preds) for k in mrr]\n",
|
||||
" return mrr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Before reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"MRR@1:\t0.97\n",
|
||||
"MRR@10:\t0.9825\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mrr_init = MRR(res_text, ground_truths, cut_offs)\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" print(f\"MRR@{c}:\\t{mrr_init[i]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"MRR@1:\t0.99\n",
|
||||
"MRR@10:\t0.995\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mrr_rerank = MRR(new_text, ground_truths, cut_offs)\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" print(f\"MRR@{c}:\\t{mrr_rerank[i]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 6.3 nDCG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Before reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"nDCG@1: 0.97\n",
|
||||
"nDCG@10: 0.9869253606521631\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.metrics import ndcg_score\n",
|
||||
"\n",
|
||||
"pred_hard_encodings = []\n",
|
||||
"for pred, label in zip(res_text, ground_truths):\n",
|
||||
" pred_hard_encoding = list(np.isin(pred, label).astype(int))\n",
|
||||
" pred_hard_encodings.append(pred_hard_encoding)\n",
|
||||
"\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" nDCG = ndcg_score(pred_hard_encodings, res_scores, k=c)\n",
|
||||
" print(f\"nDCG@{c}: {nDCG}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After reranking:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"nDCG@1: 0.99\n",
|
||||
"nDCG@10: 0.9963092975357145\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pred_hard_encodings_rerank = []\n",
|
||||
"for pred, label in zip(new_text, ground_truths):\n",
|
||||
" pred_hard_encoding = list(np.isin(pred, label).astype(int))\n",
|
||||
" pred_hard_encodings_rerank.append(pred_hard_encoding)\n",
|
||||
"\n",
|
||||
"for i, c in enumerate(cut_offs):\n",
|
||||
" nDCG = ndcg_score(pred_hard_encodings_rerank, new_scores, k=c)\n",
|
||||
" print(f\"nDCG@{c}: {nDCG}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "base",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.13"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# BGE Reranker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Like embedding models, BGE has a group of rerankers with various sizes and functionalities. In this tutorial, we will introduce the BGE rerankers series."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 0. Installation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Install the dependencies in the environment."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U FlagEmbedding"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. bge-reranker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The first generation of BGE reranker contains two models:\n",
|
||||
"\n",
|
||||
"| Model | Language | Parameters | Description | Base Model |\n",
|
||||
"|:-------|:--------:|:----:|:-----------------:|:--------------------------------------:|\n",
|
||||
"| [BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base) | Chinese and English | 278M | a cross-encoder model which is more accurate but less efficient | XLM-RoBERTa-Base |\n",
|
||||
"| [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | Chinese and English | 560M | a cross-encoder model which is more accurate but less efficient | XLM-RoBERTa-Large |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n",
|
||||
"You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[7.984375, -6.84375, -7.15234375, 5.44921875]"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import FlagReranker\n",
|
||||
"\n",
|
||||
"model = FlagReranker(\n",
|
||||
" 'BAAI/bge-reranker-large',\n",
|
||||
" use_fp16=True,\n",
|
||||
" devices=[\"cuda:0\"], # if you don't have GPUs, you can use \"cpu\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"pairs = [\n",
|
||||
" [\"What is the capital of France?\", \"Paris is the capital of France.\"],\n",
|
||||
" [\"What is the capital of France?\", \"The population of China is over 1.4 billion people.\"],\n",
|
||||
" [\"What is the population of China?\", \"Paris is the capital of France.\"],\n",
|
||||
" [\"What is the population of China?\", \"The population of China is over 1.4 billion people.\"]\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"scores = model.compute_score(pairs)\n",
|
||||
"scores"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. bge-reranker v2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"| Model | Language | Parameters | Description | Base Model |\n",
|
||||
"|:-------|:--------:|:----:|:-----------------:|:--------------------------------------:|\n",
|
||||
"| [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) | Multilingual | 568M | a lightweight cross-encoder model, possesses strong multilingual capabilities, easy to deploy, with fast inference. | XLM-RoBERTa-Large |\n",
|
||||
"| [BAAI/bge-reranker-v2-gemma](https://huggingface.co/BAAI/bge-reranker-v2-gemma) | Multilingual | 2.51B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English proficiency and multilingual capabilities. | Gemma2-2B |\n",
|
||||
"| [BAAI/bge-reranker-v2-minicpm-layerwise](https://huggingface.co/BAAI/bge-reranker-v2-minicpm-layerwise) | Multilingual | 2.72B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English and Chinese proficiency, allows freedom to select layers for output, facilitating accelerated inference. | MiniCPM |\n",
|
||||
"| [BAAI/bge-reranker-v2.5-gemma2-lightweight](https://huggingface.co/BAAI/bge-reranker-v2.5-gemma2-lightweight) | Multilingual | 9.24B | a cross-encoder model which is suitable for multilingual contexts, performs well in both English and Chinese proficiency, allows freedom to select layers, compress ratio and compress layers for output, facilitating accelerated inference. | Gemma2-9B |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### bge-reranker-v2-m3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"bge-reranker-v2-m3 is trained based on bge-m3, introducing great multi-lingual capability as keeping a slim model size."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[0.003483424193080668]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import FlagReranker\n",
|
||||
"\n",
|
||||
"# Setting use_fp16 to True speeds up computation with a slight performance degradation (if using gpu)\n",
|
||||
"reranker = FlagReranker('BAAI/bge-reranker-v2-m3', devices=[\"cuda:0\"], use_fp16=True)\n",
|
||||
"\n",
|
||||
"score = reranker.compute_score(['query', 'passage'])\n",
|
||||
"# or set \"normalize=True\" to apply a sigmoid function to the score for 0-1 range\n",
|
||||
"score = reranker.compute_score(['query', 'passage'], normalize=True)\n",
|
||||
"\n",
|
||||
"print(score)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### bge-reranker-v2-gemma"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"bge-reranker-v2-gemma is trained based on gemma-2b. It has excellent performances with both English proficiency and multilingual capabilities."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading checkpoint shards: 100%|██████████| 3/3 [00:00<00:00, 5.29it/s]\n",
|
||||
"You're using a GemmaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 1/1 [00:00<00:00, 45.99it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[1.974609375]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import FlagLLMReranker\n",
|
||||
"\n",
|
||||
"reranker = FlagLLMReranker('BAAI/bge-reranker-v2-gemma', devices=[\"cuda:0\"], use_fp16=True)\n",
|
||||
"\n",
|
||||
"score = reranker.compute_score(['query', 'passage'])\n",
|
||||
"print(score)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### bge-reranker-v2-minicpm-layerwise"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"bge-reranker-v2-minicpm-layerwise is trained based on minicpm-2b-dpo-bf16. It's suitable for multi-lingual contexts, performs well in Both English and Chinese proficiency.\n",
|
||||
"\n",
|
||||
"Another special functionality is the layerwise design gives user freedom to select layers for output, facilitating accelerated inference."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading checkpoint shards: 100%|██████████| 3/3 [00:00<00:00, 3.85it/s]\n",
|
||||
"You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 1/1 [00:00<00:00, 24.51it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[-7.06640625]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import LayerWiseFlagLLMReranker\n",
|
||||
"\n",
|
||||
"reranker = LayerWiseFlagLLMReranker('BAAI/bge-reranker-v2-minicpm-layerwise', devices=[\"cuda:0\"], use_fp16=True)\n",
|
||||
"\n",
|
||||
"# Adjusting 'cutoff_layers' to pick which layers are used for computing the score.\n",
|
||||
"score = reranker.compute_score(['query', 'passage'], cutoff_layers=[28])\n",
|
||||
"print(score)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### bge-reranker-v2.5-gemma2-lightweight"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"bge-reranker-v2.5-gemma2-lightweight is trained based on gemma2-9b. It's also suitable for multi-lingual contexts.\n",
|
||||
"\n",
|
||||
"Besides the layerwise reduction functionality, bge-reranker-v2.5-gemma2-lightweight integrates token compression capabilities to further save more resources while maintaining outstanding performances."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading checkpoint shards: 100%|██████████| 4/4 [00:01<00:00, 3.60it/s]\n",
|
||||
"You're using a GemmaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 1/1 [00:00<00:00, 23.95it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[14.734375]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from FlagEmbedding import LightWeightFlagLLMReranker\n",
|
||||
"\n",
|
||||
"reranker = LightWeightFlagLLMReranker('BAAI/bge-reranker-v2.5-gemma2-lightweight', devices=[\"cuda:0\"], use_fp16=True)\n",
|
||||
"\n",
|
||||
"# Adjusting 'cutoff_layers' to pick which layers are used for computing the score.\n",
|
||||
"score = reranker.compute_score(['query', 'passage'], cutoff_layers=[28], compress_ratio=2, compress_layers=[24, 40])\n",
|
||||
"print(score)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Comparison"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"BGE reranker series provides a great number of choices for all kinds of functionalities. You can select the model according your senario and resource:\n",
|
||||
"\n",
|
||||
"- For multilingual, utilize `BAAI/bge-reranker-v2-m3`, `BAAI/bge-reranker-v2-gemma` and `BAAI/bge-reranker-v2.5-gemma2-lightweight`.\n",
|
||||
"\n",
|
||||
"- For Chinese or English, utilize `BAAI/bge-reranker-v2-m3` and `BAAI/bge-reranker-v2-minicpm-layerwise`.\n",
|
||||
"\n",
|
||||
"- For efficiency, utilize `BAAI/bge-reranker-v2-m3` and the low layer of `BAAI/bge-reranker-v2-minicpm-layerwise`.\n",
|
||||
"\n",
|
||||
"- For saving resources and extreme efficiency, utilize `BAAI/bge-reranker-base` and `BAAI/bge-reranker-large`.\n",
|
||||
"\n",
|
||||
"- For better performance, recommand `BAAI/bge-reranker-v2-minicpm-layerwise` and B`AAI/bge-reranker-v2-gemma`.\n",
|
||||
"\n",
|
||||
"Make sure always test on your real use case and choose the one with best speed-quality balance!"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "ft",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Evaluate Reranker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Reranker usually better captures the latent semantic meanings between sentences. But comparing to using an embedding model, it will take quadratic $O(N^2)$ running time for the whole dataset. Thus the most common use cases of rerankers in information retrieval or RAG is reranking the top k answers retrieved according to the embedding similarities.\n",
|
||||
"\n",
|
||||
"The evaluation of reranker has the similar idea. We compare how much better the rerankers can rerank the candidates searched by a same embedder. In this tutorial, we will evaluate two rerankers' performances on BEIR benchmark, with bge-large-en-v1.5 as the base embedding model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note: We highly recommend to run this notebook with GPU. The whole pipeline is very time consuming. For simplicity, we only use a single task FiQA in BEIR."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 0. Installation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"First install the required dependency"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install FlagEmbedding"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. bge-reranker-large"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The first model is bge-reranker-large, a BERT like reranker with about 560M parameters.\n",
|
||||
"\n",
|
||||
"We can use the evaluation pipeline of FlagEmbedding to directly run the whole process:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Split 'dev' not found in the dataset. Removing it from the list.\n",
|
||||
"ignore_identical_ids is set to True. This means that the search results will not contain identical ids. Note: Dataset such as MIRACL should NOT set this to True.\n",
|
||||
"pre tokenize: 100%|██████████| 57/57 [00:03<00:00, 14.68it/s]\n",
|
||||
"You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
|
||||
" warnings.warn(\n",
|
||||
"Inference Embeddings: 100%|██████████| 57/57 [00:44<00:00, 1.28it/s]\n",
|
||||
"pre tokenize: 100%|██████████| 1/1 [00:00<00:00, 61.59it/s]\n",
|
||||
"Inference Embeddings: 100%|██████████| 1/1 [00:00<00:00, 6.22it/s]\n",
|
||||
"Searching: 100%|██████████| 21/21 [00:00<00:00, 68.26it/s]\n",
|
||||
"pre tokenize: 0%| | 0/64 [00:00<?, ?it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 100%|██████████| 64/64 [00:08<00:00, 7.15it/s]\n",
|
||||
"Compute Scores: 100%|██████████| 64/64 [01:39<00:00, 1.56s/it]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%bash\n",
|
||||
"python -m FlagEmbedding.evaluation.beir \\\n",
|
||||
"--eval_name beir \\\n",
|
||||
"--dataset_dir ./beir/data \\\n",
|
||||
"--dataset_names fiqa \\\n",
|
||||
"--splits test dev \\\n",
|
||||
"--corpus_embd_save_dir ./beir/corpus_embd \\\n",
|
||||
"--output_dir ./beir/search_results \\\n",
|
||||
"--search_top_k 1000 \\\n",
|
||||
"--rerank_top_k 100 \\\n",
|
||||
"--cache_path /root/.cache/huggingface/hub \\\n",
|
||||
"--overwrite True \\\n",
|
||||
"--k_values 10 100 \\\n",
|
||||
"--eval_output_method markdown \\\n",
|
||||
"--eval_output_path ./beir/beir_eval_results.md \\\n",
|
||||
"--eval_metrics ndcg_at_10 recall_at_100 \\\n",
|
||||
"--ignore_identical_ids True \\\n",
|
||||
"--embedder_name_or_path BAAI/bge-large-en-v1.5 \\\n",
|
||||
"--reranker_name_or_path BAAI/bge-reranker-large \\\n",
|
||||
"--embedder_batch_size 1024 \\\n",
|
||||
"--reranker_batch_size 1024 \\\n",
|
||||
"--devices cuda:0 \\"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. bge-reranker-v2-gemma"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The second model is bge-reranker-v2-m3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Split 'dev' not found in the dataset. Removing it from the list.\n",
|
||||
"ignore_identical_ids is set to True. This means that the search results will not contain identical ids. Note: Dataset such as MIRACL should NOT set this to True.\n",
|
||||
"initial target device: 100%|██████████| 4/4 [01:14<00:00, 18.51s/it]\n",
|
||||
"pre tokenize: 100%|██████████| 15/15 [00:01<00:00, 11.21it/s]\n",
|
||||
"You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 100%|██████████| 15/15 [00:01<00:00, 11.32it/s]\n",
|
||||
"You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 100%|██████████| 15/15 [00:01<00:00, 10.29it/s]\n",
|
||||
"You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 100%|██████████| 15/15 [00:01<00:00, 13.99it/s]\n",
|
||||
"You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
|
||||
" warnings.warn(\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
|
||||
" warnings.warn(\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
|
||||
" warnings.warn(\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
|
||||
" warnings.warn(\n",
|
||||
"Inference Embeddings: 100%|██████████| 15/15 [00:12<00:00, 1.24it/s]\n",
|
||||
"Inference Embeddings: 100%|██████████| 15/15 [00:12<00:00, 1.23it/s]\n",
|
||||
"Inference Embeddings: 100%|██████████| 15/15 [00:12<00:00, 1.22it/s]\n",
|
||||
"Inference Embeddings: 100%|██████████| 15/15 [00:12<00:00, 1.21it/s]\n",
|
||||
"Chunks: 100%|██████████| 4/4 [00:30<00:00, 7.70s/it]\n",
|
||||
"Chunks: 100%|██████████| 4/4 [00:00<00:00, 47.90it/s]\n",
|
||||
"Searching: 100%|██████████| 21/21 [00:00<00:00, 128.34it/s]\n",
|
||||
"initial target device: 100%|██████████| 4/4 [01:09<00:00, 17.43s/it]\n",
|
||||
"pre tokenize: 0%| | 0/16 [00:00<?, ?it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 12%|█▎ | 2/16 [00:00<00:02, 6.46it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 12%|█▎ | 2/16 [00:00<00:03, 4.60it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 25%|██▌ | 4/16 [00:00<00:02, 4.61it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"pre tokenize: 100%|██████████| 16/16 [00:03<00:00, 4.12it/s]\n",
|
||||
"pre tokenize: 100%|██████████| 16/16 [00:04<00:00, 3.78it/s]\n",
|
||||
"pre tokenize: 100%|██████████| 16/16 [00:04<00:00, 3.95it/s]\n",
|
||||
"pre tokenize: 100%|██████████| 16/16 [00:04<00:00, 3.81it/s]\n",
|
||||
"Compute Scores: 100%|██████████| 67/67 [00:29<00:00, 2.30it/s]\n",
|
||||
"Compute Scores: 100%|██████████| 67/67 [00:29<00:00, 2.27it/s]\n",
|
||||
"Compute Scores: 100%|██████████| 67/67 [00:29<00:00, 2.27it/s]\n",
|
||||
"Compute Scores: 100%|██████████| 67/67 [00:30<00:00, 2.19it/s]\n",
|
||||
"Chunks: 100%|██████████| 4/4 [00:51<00:00, 12.97s/it]\n",
|
||||
"/share/project/xzy/Envs/ft/lib/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 8 leaked semaphore objects to clean up at shutdown\n",
|
||||
" warnings.warn('resource_tracker: There appear to be %d '\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%bash\n",
|
||||
"python -m FlagEmbedding.evaluation.beir \\\n",
|
||||
"--eval_name beir \\\n",
|
||||
"--dataset_dir ./beir/data \\\n",
|
||||
"--dataset_names fiqa \\\n",
|
||||
"--splits test dev \\\n",
|
||||
"--corpus_embd_save_dir ./beir/corpus_embd \\\n",
|
||||
"--output_dir ./beir/search_results \\\n",
|
||||
"--search_top_k 1000 \\\n",
|
||||
"--rerank_top_k 100 \\\n",
|
||||
"--cache_path /root/.cache/huggingface/hub \\\n",
|
||||
"--overwrite True \\\n",
|
||||
"--k_values 10 100 \\\n",
|
||||
"--eval_output_method markdown \\\n",
|
||||
"--eval_output_path ./beir/beir_eval_results.md \\\n",
|
||||
"--eval_metrics ndcg_at_10 recall_at_100 \\\n",
|
||||
"--ignore_identical_ids True \\\n",
|
||||
"--embedder_name_or_path BAAI/bge-large-en-v1.5 \\\n",
|
||||
"--reranker_name_or_path BAAI/bge-reranker-v2-m3 \\\n",
|
||||
"--embedder_batch_size 1024 \\\n",
|
||||
"--reranker_batch_size 1024 \\\n",
|
||||
"--devices cuda:0 cuda:1 cuda:2 cuda:3 \\\n",
|
||||
"--reranker_max_length 1024 \\"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Comparison"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'fiqa-test': {'ndcg_at_10': 0.40991, 'ndcg_at_100': 0.48028, 'map_at_10': 0.32127, 'map_at_100': 0.34227, 'recall_at_10': 0.50963, 'recall_at_100': 0.75987, 'precision_at_10': 0.11821, 'precision_at_100': 0.01932, 'mrr_at_10': 0.47786, 'mrr_at_100': 0.4856}}\n",
|
||||
"{'fiqa-test': {'ndcg_at_10': 0.44828, 'ndcg_at_100': 0.51525, 'map_at_10': 0.36551, 'map_at_100': 0.38578, 'recall_at_10': 0.519, 'recall_at_100': 0.75987, 'precision_at_10': 0.12299, 'precision_at_100': 0.01932, 'mrr_at_10': 0.53382, 'mrr_at_100': 0.54108}}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"with open('beir/search_results/bge-large-en-v1.5/bge-reranker-large/EVAL/eval_results.json') as f:\n",
|
||||
" results_1 = json.load(f)\n",
|
||||
" print(results_1)\n",
|
||||
" \n",
|
||||
"with open('beir/search_results/bge-large-en-v1.5/bge-reranker-v2-m3/EVAL/eval_results.json') as f:\n",
|
||||
" results_2 = json.load(f)\n",
|
||||
" print(results_2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"From the above results we can see that bge-reranker-v2-m3 has advantage on almost all the metrics."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "ft",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user