chore: import upstream snapshot with attribution

FlagEmbedding/evaluation/air_bench/__init__.py

@@ -0,0 +1,8 @@
+from .arguments import AIRBenchEvalModelArgs, AIRBenchEvalArgs
+from .runner import AIRBenchEvalRunner
+
+__all__ = [
+    "AIRBenchEvalModelArgs",
+    "AIRBenchEvalArgs",
+    "AIRBenchEvalRunner"
+]

FlagEmbedding/evaluation/air_bench/__main__.py

@@ -0,0 +1,32 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.air_bench import (
+    AIRBenchEvalArgs, AIRBenchEvalModelArgs,
+    AIRBenchEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        AIRBenchEvalArgs,
+        AIRBenchEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: AIRBenchEvalArgs
+    model_args: AIRBenchEvalModelArgs
+
+    runner = AIRBenchEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()
+    print("==============================================")
+    print("Search results have been generated.")
+    print("For computing metrics, please refer to the official AIR-Bench docs:")
+    print("- https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/submit_to_leaderboard.md")

FlagEmbedding/evaluation/air_bench/arguments.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+from air_benchmark import EvalArgs as AIRBenchEvalArgs
+
+
+@dataclass
+class AIRBenchEvalModelArgs:
+    """
+    Evaluation Model arguments for AIR Bench.
+    """
+    embedder_name_or_path: str = field(
+        metadata={"help": "The embedder name or path.", "required": True}
+    )
+    embedder_model_class: Optional[str] = field(
+        default=None, metadata={"help": "The embedder model class. Available classes: ['encoder-only-base', 'encoder-only-m3', 'decoder-only-base', 'decoder-only-icl']. Default: None. For the custom model, you need to specifiy the model class.", "choices": ["encoder-only-base", "encoder-only-m3", "decoder-only-base", "decoder-only-icl"]}
+    )
+    normalize_embeddings: bool = field(
+        default=True, metadata={"help": "whether to normalize the embeddings"}
+    )
+    pooling_method: str = field(
+        default="cls", metadata={"help": "The pooling method fot the embedder."}
+    )
+    use_fp16: bool = field(
+        default=True, metadata={"help": "whether to use fp16 for inference"}
+    )
+    devices: Optional[str] = field(
+        default=None, metadata={"help": "Devices to use for inference.", "nargs": "+"}
+    )
+    query_instruction_for_retrieval: Optional[str] = field(
+        default=None, metadata={"help": "Instruction for query"}
+    )
+    query_instruction_format_for_retrieval: str = field(
+        default="{}{}", metadata={"help": "Format for query instruction"}
+    )
+    examples_for_task: Optional[str] = field(
+        default=None, metadata={"help": "Examples for task"}
+    )
+    examples_instruction_format: str = field(
+        default="{}{}", metadata={"help": "Format for examples instruction"}
+    )
+    trust_remote_code: bool = field(
+        default=False, metadata={"help": "Trust remote code"}
+    )
+    reranker_name_or_path: Optional[str] = field(
+        default=None, metadata={"help": "The reranker name or path."}
+    )
+    reranker_model_class: Optional[str] = field(
+        default=None, metadata={"help": "The reranker model class. Available classes: ['encoder-only-base', 'decoder-only-base', 'decoder-only-layerwise', 'decoder-only-lightweight']. Default: None. For the custom model, you need to specify the model class.", "choices": ["encoder-only-base", "decoder-only-base", "decoder-only-layerwise", "decoder-only-lightweight"]}
+    )
+    reranker_peft_path: Optional[str] = field(
+        default=None, metadata={"help": "The reranker peft path."}
+    )
+    use_bf16: bool = field(
+        default=False, metadata={"help": "whether to use bf16 for inference"}
+    )
+    query_instruction_for_rerank: Optional[str] = field(
+        default=None, metadata={"help": "Instruction for query"}
+    )
+    query_instruction_format_for_rerank: str = field(
+        default="{}{}", metadata={"help": "Format for query instruction"}
+    )
+    passage_instruction_for_rerank: Optional[str] = field(
+        default=None, metadata={"help": "Instruction for passage"}
+    )
+    passage_instruction_format_for_rerank: str = field(
+        default="{}{}", metadata={"help": "Format for passage instruction"}
+    )
+    model_cache_dir: str = field(
+        default=None, metadata={"help": "Cache directory for models."}
+    )
+    # ================ for inference ===============
+    embedder_batch_size: int = field(
+        default=3000, metadata={"help": "Batch size for inference."}
+    )
+    reranker_batch_size: int = field(
+        default=3000, metadata={"help": "Batch size for inference."}
+    )
+    embedder_query_max_length: int = field(
+        default=512, metadata={"help": "Max length for query."}
+    )
+    embedder_passage_max_length: int = field(
+        default=512, metadata={"help": "Max length for passage."}
+    )
+    truncate_dim: Optional[int] = field(
+        default=None, metadata={"help": "The dimension to truncate embeddings to. Useful for Matryoshka Representation Learning models. If None, no truncation is performed."}
+    )
+    reranker_query_max_length: Optional[int] = field(
+        default=None, metadata={"help": "Max length for reranking."}
+    )
+    reranker_max_length: int = field(
+        default=512, metadata={"help": "Max length for reranking."}
+    )
+    normalize: bool = field(
+        default=False, metadata={"help": "whether to normalize the reranking scores"}
+    )
+    prompt: Optional[str] = field(
+        default=None, metadata={"help": "The prompt for the reranker."}
+    )
+    cutoff_layers: List[int] = field(
+        default=None, metadata={"help": "The output layers of layerwise/lightweight reranker."}
+    )
+    compress_ratio: int = field(
+        default=1, metadata={"help": "The compress ratio of lightweight reranker."}
+    )
+    compress_layers: Optional[int] = field(
+        default=None, metadata={"help": "The compress layers of lightweight reranker.", "nargs": "+"}
+    )
+
+    def __post_init__(self):
+        # replace "\\n" with "\n"
+        if "\\n" in self.query_instruction_format_for_retrieval:
+            self.query_instruction_format_for_retrieval = self.query_instruction_format_for_retrieval.replace("\\n", "\n")
+        if "\\n" in self.examples_instruction_format:
+            self.examples_instruction_format = self.examples_instruction_format.replace("\\n", "\n")
+        if "\\n" in self.query_instruction_format_for_rerank:
+            self.query_instruction_format_for_rerank = self.query_instruction_format_for_rerank.replace("\\n", "\n")
+        if "\\n" in self.passage_instruction_format_for_rerank:
+            self.passage_instruction_format_for_rerank = self.passage_instruction_format_for_rerank.replace("\\n", "\n")

FlagEmbedding/evaluation/air_bench/examples/long-doc/arxiv-gemini.jsonl

@@ -0,0 +1,3 @@
+{"query": "So, which AI model did the best on the MMMU benchmark according to Yue and his team back in 2023?", "pos": "MMMU (val) Gemini Ultra (0-shot) GPT-4V (0-shot)\nMaj@32 pass@1 pass@1\nArt & Design 74.2 70.0 65.8\nBusiness 62.7 56.7 59.3\nScience 49.3 48.0 54.7\nHealth & Medicine 71.3 67.3 64.7\nHumanities & Social Science 78.3 78.3 72.5\nTechnology & Engineering 53.0 47.1 36.7\nOverall 62.4 59.4 56.8\nTable 8|Gemini Ultra performance on the MMMU benchmark (Yue et al., 2023) per discipline."}
+{"query": "The GSPMD partitioner, part of the XLA compiler, is responsible for dividing the training step calculation.", "pos": "The GSPMD partitioner (Xu et al., 2021) in the XLA compiler\npartitions the training step computation, and the MegaScale XLA compiler (XLA, 2019) pass statically\nschedules appropriate collectives so that they maximally overlap with the computation with very little\nvariation in step time.\nMaintaining a high goodput2at this scale would have been impossible using the conventional\napproach of periodic checkpointing of weights to persistent cluster storage. For Gemini models, we\ninstead made use of redundant in-memory copies of the model state, and on any unplanned hardware\nfailures, we rapidly recover directly from an intact model replica."}
+{"query": "What's the impact of where you live and your social status on how well AI image labeling tech works?", "pos": "Thoughwedo\nnot see large discrepancies across different groups, we note that this metric is imperfect as the human\nreference captions could be inherently biased. Additionally, we perform a zero-shot classification style\nevaluation with the Dollarstreet dataset (Rojas et al., 2022) to measure discrepancies in performance\nacross images which come from different geographic locations. As is seen in previous work, we find\nthat models work less effectively for images from lower socioeconomic regions and regions outside\nNorth America and Europe. This is an area where we need further research and work to improve in\nfuture iterations of our models.\nIn addition to comparing performance on tasks across groups, we also consider how people are\ndescribed in captions."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/arxiv-gpt3.jsonl

@@ -0,0 +1,3 @@
+{"query": "What gauges the effects of data contamination?", "pos": "We also undertake a systematic study of “data contamination” – a growing problem when training high capacity models\non datasets such as Common Crawl, which can potentially include content from test datasets simply because such\ncontent often exists on the web. In this paper we develop systematic tools to measure data contamination and quantify\nits distorting effects. Although we find that data contamination has a minimal effect on GPT-3’s performance on most\ndatasets, we do identify a few datasets where it could be inflating results, and we either do not report results on these\ndatasets or we note them with an asterisk, depending on the severity."}
+{"query": "What strategies did the United States employ to convince Pakistan to exercise its influence over the Taliban?", "pos": "Direct\npressure on the Taliban had proved unsuccessful. As one NSC staff note\nput it, \"Under the Taliban, Afghanistan is not so much a state sponsor\nof terrorism as it is a state sponsored by terrorists.\" In early 2000,\nthe United States began a high-level effort to persuade Pakistan to use\nits influence over the Taliban. In January 2000, Assistant Secretary\nof State Karl Inderfurth and the State Department’s counterterrorism\ncoordinator, Michael Sheehan, met with General Musharraf in Islamabad,\ndangling before him the possibility of a presidential visit in March as a\nreward for Pakistani cooperation. Such a visit was coveted by Musharraf,\npartly as a sign of his government’s legitimacy."}
+{"query": "What does carrying rotten potatoes symbolize?", "pos": "The children started complaining about the\ntrouble loudly.\nThen Mrs. Smith told them why she asked them to play the game. She\nsaid,\"This is exactly the situation when you carry your hatred for somebody\ninside your heart. The terrible smell of the hatred will pollute your\nheart and you will carry something unnecessary with you all the time. If\nyou cannot stand the smell of the rotten potatoes for just two weeks, can\nyou imagine how heavy it would be to have the hatred in your heart for your\nlifetime? So throw away any hatred from your heart, and you’ll be really\nhappy.\"\nQ: Which of the following is True according to the passage?\nA: If a kid hated four people,he or she had to carry four potatoes."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/arxiv-llama2.jsonl

@@ -0,0 +1,3 @@
+{"query": "Could you elucidate on the values of temperature and top-p that are utilized for pass@1 scores?", "pos": "8 62.8\n13B 18.3 60.2 30.6 69.0\n34B 22.6 77.2 33.0 76.1\n70B29.9 89.0 45.0 81.4\nTable 21: Code generation results on Human-Eval and MBPP . We report 0-shot and 3-shot results for\nHuman-Eval and MBPP respectively. For pass@100 and pass@80 scores, we use a temperature of 0.8 and\ntop-p=0.95. For pass@1 scores, we use a temperature of 0.1 and top- p=0.95.\n49"}
+{"query": "What do high safety scores and low helpfulness ratings suggest?", "pos": "Here we show more evidence and\nqualitative results to manifest this tension. Figure32 are two scatter plots of helpfulness and safety reward\nmodel scores on the safety test set for safe and unsafe responses. The tension can be observed at the bottom\nright corner (i.e., high safety score but low helpfulness score) in the safe response plot (left) and the top left\ncorner (i.e., low safety score but high helpfulness score) in the unsafe response plot (right). We also list two\nqualitative examples where safety and helpfulness reward models don’t agree with each other in Table 35."}
+{"query": "The process of carefully adjusting precautions relies on using challenging stimuli together with protected displays to make its operation run more smoothly.", "pos": "4.2 Safety Fine-Tuning\nIn this section, we describe our approach to safety fine-tuning, including safety categories, annotation\nguidelines,and the techniques we use to mitigate safety risks. We employ a process similar to the general\nfine-tuning methods as described in Section 3, with some notable differences related to safety concerns.\nSpecifically, we use the following techniques in safety fine-tuning:\n1.Supervised Safety Fine-Tuning : We initialize by gathering adversarial prompts and safe demonstra-\ntions that are then included in the general supervised fine-tuning process (Section 3.1). This teaches\nthe model to align with our safety guidelines even before RLHF,and thus lays the foundation for\nhigh-quality human preference data annotation."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/arxiv-llm-survey.jsonl

@@ -0,0 +1,3 @@
+{"query": "What are the pre-training challenges for large language models?", "pos": "To make this survey more self-contained, we present the\ndetailed formulations for these configurations in Table 6.\nNormalization Methods. Training instability is a challeng-\ning issue for pre-training LLMs. To alleviate this issue,\nnormalization is a widely adopted strategy to stabilize the\ntraining of neural networks. In the vanilla Transformer [22],\nLayerNorm [256] is employed. Recently, several advanced\nnormalization techniques have been proposed as alterna-\ntives to LayerNorm, e.g., RMSNorm, and DeepNorm.\n•LayerNorm. In the early research, BatchNorm [265] is\na commonly used normalization method. However, it is\ndifficult to deal with sequence data of variable lengths and\nsmall-batch data."}
+{"query": "Language learning models seriously struggle to grasp complex symbols when they're thrown in scenarios they don't know jack about.", "pos": "For an example of\nthe out-of-domain test, LLMs could only see the examples\nwith two words in context, but it requires LLMs to concate-\nnate the last letters of three or more words. Typically, the\naccuracy of the generated symbols is adopted to evaluate\nthe performance of LLMs on these tasks. Thus, LLMs need\nto understand the semantic relations among the symbolic\noperations and their composition in complex scenarios.\nHowever, under the out-of-domain setting, as LLMs have\nnot seen the complex compositions of symbolic operations\nand rules ( e.g., twice the number of operations in context\nexamples), it is hard for LLMs to capture their accurate\nmeanings."}
+{"query": "Could you shed some light on the two primary ways that LLMs employ demonstrations as discussed in document 493?", "pos": "How LLMs Perform ICL? At the inference stage, researchers\nfocus on analyzing how the ICL capability operates based\non given demonstrations since no explicit learning or updat-\ning is involved. According to the discussion in [493], there\nare two main ways for LLMs to utilize demonstrations: task\nrecognition and task learning.\n•Task recognition. In the first way, LLMs recognize the\ntask from demonstrations and utilize the prior knowledge\nobtained from pre-training to solve new test tasks. A Proba-\nbly Approximately Correct (PAC) framework [494] has been\nproposed to assess the learnability of ICL."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/book-a-brief-history-of-time_stephen-hawking.jsonl

@@ -0,0 +1,3 @@
+{"query": "Why is it believed the universe began at a particular time?", "pos": "According to a number of earlycosmologies and the Jewish/Christian/Muslim tradition, the universe started at a finite, and not very distant,time in the past. One argument for such a beginning was the feeling that it was necessary to have “First Cause”to explain the existence of the universe. (Within the universe, you always explained one event as being causedby some earlier event, but the existence of the universe itself could be explained in this way only if it had somebeginning.) Another argument was put forward by St. Augustine in his book The City of God. He pointed out\nthat civilization is progressing and we remember who performed this deed or developed that technique."}
+{"query": "Could you elucidate on the intricate procedure of stellar constitution?", "pos": "Andeven then it was a long time before the implications of the theory for massive stars were understood.To understand how a black hole might be formed, we first need an understanding of the life cycle of a star. A star isformed when a large amount of gas (mostly hydrogen) starts to collapse in on itself due to its gravitational attraction. Asit contracts, the atoms of the gas collide with each other more and more frequently and at greater and greater speeds –the gas heats up. Eventually, the gas will be so hot that when the hydrogen atoms collide they no longer bounce offeach other, but instead coalesce to form helium. The heat released in this reaction, which is like a controlled hydrogenbomb explosion, is what makes the star shine."}
+{"query": "Black hole existence evidence?", "pos": "the body that has collapsed must be lost when a black hole is formed, because afterward all we can possibly measureabout the body is its mass and rate of rotation. The significance of this will be seen in the next chapter.Black holes are one of only a fairly small number of cases in the history of science in which a theory was developed ingreat detail as a mathematical model before there was any evidence from observations that it was correct. Indeed, thisused to be the main argument of opponents of black holes: how could one believe in objects for which the onlyevidence was calculations based on the dubious theory of general relativity?"}

FlagEmbedding/evaluation/air_bench/examples/long-doc/book-origin-of-species_darwin.jsonl

@@ -0,0 +1,3 @@
+{"query": "So, like, what's the big deal about the top species from the bigger groups talked about in Chapter 4?", "pos": "In our second and fourth chapters, on Variation and on Natural Selection, I have attempted\nto show that it is the widely ranging, the much diffused and common, that is the dominant species\nbelonging to the larger genera, which vary most.  The varieties, or incipient species, thus produced\nultimately become converted, as I believe, into new and distinct species; and these, on the principle\nof inheritance, tend to produce other new and dominant species.  Consequently the groups which\nare now large, and which generally include many dominant species, tend to go on increasing\nindefinitely in size."}
+{"query": "Identify the unique species in the Chthamalinae subfamily of sessile cirripedes and the location of its fossil discovery.", "pos": "I suspect that but few of\nthe very many animals which live on the beach between high and low watermark are preserved.\nFor instance, the several species of the Chthamalinae (a sub-family of sessile cirripedes) coat the\nrocks all over the world in infinite numbers:  they are all strictly littoral, with the exception of a\nsingle Mediterranean species, which inhabits deep water and has been found fossil in Sicily,\nwhereas not one other species has hitherto been found in any tertiary formation:  yet it is now\nknown that the genus Chthamalus existed during the chalk period.  The molluscan genus Chiton\noffers a partially analogous case."}
+{"query": "Why are there flaws in the geological record?", "pos": "Nor is their rarity surprising, when we remember how large a proportion of the\nbones of tertiary mammals have been discovered either in caves or in lacustrine deposits; and that\nnot a cave or true lacustrine bed is known belonging to the age of our secondary or palaeozoic\nformations.\nBut the imperfection in the geological record mainly results from another and more important cause\nthan any of the foregoing; namely, from the several formations being separated from each other by\nwide intervals of time.  When we see the formations tabulated in written works, or when we follow\nthem in nature, it is difficult to avoid believing that they are closely consecutive."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/healthcare-pubmed_100k-200k_1.jsonl

@@ -0,0 +1,3 @@
+{"query": "What does 'O' represent in peptides?", "pos": "by changing the peptides to be amphiphilic or completely polar, they systematically synthesized several derived peptides. each of them has a different polar uncharged group : p11-8 (423, based on glutamine q, sequence ac-qqrfowofeqq-nh2 ; o represents ornithine), p11-12 (424, based on serine s, sequence ac-ssrfowofess- nh2), p11-16 (427, based on asparagine n, sequence ac-nnrfowofenn- nh2), and p11-18 (428, based on threonine t, sequence ac-ttrfowofett- nh2)."}
+{"query": "Could you elucidate on the system that was demonstrated by Van Esch and his team utilizing 1,3,5-triamide cyclohexane-based hydrogelators 67 for the alignment of nanofibers?", "pos": "used an electrical field to assist the alignment of the nanofibers and demonstrated that the application of a voltage bias, indeed, helps the directional orientation of the fibrils. using the 1,3,5-triamide cyclohexane-based hydrogelators 67, van esch et al. demonstrated an elegant system that forms well-defined nanostructures by the orthogonal self-assembly of hydrogelators and surfactants."}
+{"query": "What environmental factors influence peptide self-assembly?", "pos": "as pointed out by the authors, the hydrophobic effect between 267 molecules favors axial assembly and their electrostatic forces modulate lateral assembly. at a concentration of 0.05 wt %, the peptide self-assembles to form a filament consisting of about 120 molecules of 267. the authors also reported that various environmental factors (e.g., ph, salt, molecular crowding reagents, and   peptides) can regulate the self-assembled filaments in an assembly of predictable manner, which provides useful insights for developing coiled coils as peptide-based materials. it would be interesting to know the proteolytic stability of these self-assembled filaments. besides native peptides acting as hydrogelators, peptide derivatives can also self-assemble in water to form hydrogels."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/healthcare-pubmed_100k-200k_2.jsonl

@@ -0,0 +1,3 @@
+{"query": "Why do we get different parameter sets when looking at how ion and water-oxygen atoms interact?", "pos": "the problem here is which one to chose to obtain a consistent set of parameters. the multiple parameter sets arise because similar aij and bij terms can be obtained between the ion and water-oxygen atoms : for a certain rmin/2 and , there will be a corresponding bigger rmin/2 and smaller , or a smaller rmin/2 and bigger , which yield similar aij and bij terms (see eqs 54 and 55). when two different ion parameter sets, which give similar aij terms between an ion and oxygen in water, are applied to the same biomolecule, they may give quite different aij terms between the same atom type on the biomolecule and the metal ion after applying the combining rules."}
+{"query": "Chemical physicists managed to mock-up ions in common force fields using the 12-6 Lennard-Jones model without any direct bonding, which verified the structure traits of water-based potassium.", "pos": "the 12-6 lj nonbonded model remains a fast and practical way to simulate ions using classical force fields. the blyp functional was used for the system containing a k ion and 59 water molecules. in total, 0.168 ps of equilibration and 1.98 ps of sampling were performed in the nve ensemble. good agreement between the cpmd and classical md simulations was obtained for the structural properties of aqueous k, validating in part the classical representation of the k ion. moreover, it has also shown that it is possible to simultaneously simulate two or more experimental properties for some of the monovalent ions (e.g., na, k, rb, cs) using the 12-6 lj nonbonded model."}
+{"query": "Which concepts are encapsulated within classical models in AMOEBA?", "pos": "they also proposed that the ct effect may need to be included to improve the model. ponder, ren, and co-workers have created the atomic multipole optimized energetics for biomolecular simulation (amoeba) force field. it has bonded terms (bond, angle, dihedral, and improper torsion terms) represented using classical models. the bond and angle parameters are fit on the basis of qm-derived values (e.g., geometries and vibrational frequencies). the electrostatic interaction is represented by permanent monopoles (point charges), dipoles, and quadrupoles derived from the distributed multipole analysis (dma) procedure, along with the polarizable dipoles."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/healthcare-pubmed_100k-200k_3.jsonl

@@ -0,0 +1,3 @@
+{"query": "In what manner does the transference of electric charge impact the process of dimerization?", "pos": "the natural bond orbital analysis suggests that the n(py)  *(r  x) charge transfer plays a key role in the formation of these dimers, while the symmetry-adapted perturbation theory energy decomposition analysis indicates that the xb in r  xpyridine complexes is predominantly inductive in nature. halogen-bonded systems containing one or two xbs were analyzed by using the natural orbitals for chemical valence (nocv) method combined with the extended-transition-state (ets) method."}
+{"query": "What affects XB's susceptibility to steric hindrance?", "pos": "for the reason stated above, xb is, in general, more sensitive to steric hindrance than hb. in the infinite chain formed by 1,4-diiodotetrafluorobenzene with 4,4- and 2,2-bipyridine, the c  in distances are 2.864 and 3.158 , respectively ; when 2,4-bipyridine forms heteromeric crystals with the same xb donor, only the 4-pyridyl nitrogen is halogen-bonded, and trimers are formed wherein the c , we will see how, in the formation of dna base pairs wherein xb substitutes for hb, the most stable pairing was given by bromine as the advantage offered by the greater polarizability of iodine was overwhelmed by the disadvantage resulting from its greater size."}
+{"query": "Are there any haloheteroarenes with iodine atoms?", "pos": "color code : carbon, gray ; nitrogen, blue ; iodine, purple ; fluorine, yellow. the most commonly used classes of haloheteroarenes are those containing nitrogen atom(s) in the ring. both neutral and positively charged haloheteroarenes can function as scaffolds for an xb donor site (figure 55) ; the cationic form is typically obtained by reacting the neutral form with an alkyl halide or a hydrogen halide, and the released anion works as an xb acceptor for the activated xb donor site."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/healthcare-pubmed_30k-40k_10-merged.jsonl

@@ -0,0 +1,3 @@
+{"query": "How do intrinsic and extrinsic factors impact A42 aggregation, and why is drug development challenging for this process?", "pos": "indeed, it has been shown that the dominant mechanism for catalyzing the formation of toxic a42 species is surface-catalyzed secondary nucleation. in other words, once a small but critical concentration of a42 aggregates has been generated through primary nucleation of monomers, surface-catalyzed secondary nucleation becomes the dominant process where the surface of the existing fibrils serve as catalytic sites for the generation of toxic oligomeric species [ 54, 57 ]. furthermore, the role of intrinsic and extrinsic factors on the aggregation process of a42 has been partly unveiled and a great effort has been focused on drug development against a42 aggregation, which has proven to be very difficult [ 100, 101 ]."}
+{"query": "Excitons transfer energy and can jump to a higher state, then lose energy quickly, causing them to vanish.", "pos": "this process occurs at high excitation densities when one exciton transfers its energy to another exciton and brings it to a higher-energy excited state. the higher-energy excited state relaxes rapidly, and overall an exciton is lost. in so far as quenching occurs throughout the volume of the sample, this measurement resembles volume quenching, but with excitons acting as their own quenchers. in these experiments, typically high light intensities are used, far higher than used under solar illumination conditions, and the consequences of this have to be taken into account when analyzing the data."}
+{"query": "Causes of artifacts?", "pos": "however, a limiting step in the measurements of the intrinsic young s modulus of amyloid fibrillar aggregates on a surface is the correct evaluation of the cross-sectional moment of inertia i. recently, it was presented a general approach based on theory of elasticity and an innovative calculation of the polymorphic fibrillar aggregates cross-sectional moment of inertia i in order to evaluate correctly the nanomechanical properties of amyloids. this method enables to calculate bending rigidities b and matching the measured experimental values of young s modulus of amyloid fibrils [ 149, 165 ]. however, fibril imaging by afm requires deposition on a surface and drying, which can potentially lead to artifacts in the evaluation of the persistence length and bending rigidity."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/healthcare-pubmed_40k-50k_5-merged.jsonl

@@ -0,0 +1,3 @@
+{"query": "What is the primary product of photodimerization?", "pos": "sensitization is also the preferred way to promote coumarin and many of its derivatives into the excited state. in the absence of an external olefin, [ 2  +  2 ] photodimerization occurs with the hh cis-anti-cis product (rac-317, figure 15) being the major product. in benzene as the solvent and with benzophenone as triplet sensitizer, yields over 90% were achieved by ding and co-workers. compound rac-317 served as the starting material for the synthesis of new phosphane ligands."}
+{"query": "What's a basic challenge in the [2 + 2] photocycloaddition reactions?", "pos": "the requirement of an aryl enone was a fundamental obstacle in the [ 2  +  2 ] photocycloaddition reactions, which limited the application of this methodology. in order to overcome this problem, the yoon group described a visible-light-induced [ 2  +  2 ] photocycloaddition reaction of,-unsaturated 2-imidazolyl ketones such as 483 (scheme 163, dbu = 1,8-diazabicyclo[5.4.0]undec-7-ene)."}
+{"query": "Dry AMD causes photoreceptors to break down because the retinal pigment epithelium, which supports retinal neurons, isn't working properly.", "pos": "intravitreal anti-vegf therapies have emerged as a standard of care to treat wet amd ; however, there is currently no fda-approved treatment available for the dry form. thus, safe and effective treatment of dry amd remains a critical unmet need. atrophic (dry) form of amd represents a slowly progressing neurodegenerative disorder of the eye in which specialized retinal neurons (rod and cone photoreceptors) degenerate in the central part of the retina called macula. histopathological and clinical data suggest that photoreceptor degeneration in dry amd is triggered by abnormalities in the retinal pigment epithelium (rpe) that lies beneath photoreceptors and provides critical metabolic support to these light-sensing neuronal cells."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/law-lex_files_300k-400k.jsonl

@@ -0,0 +1,3 @@
+{"query": "Where can I get the NIST Standard Reference Materials Catalog?", "pos": "The calibration services, standard reference materials and related measurement services along with changes and fees are published in two Special Publications (SP's) and their supplements. These are SP 250 “Calibration and Related Measurement Services of the National Institute of Standards & Technology” 1\n\n     and SP 260 “NIST Standard Reference Materials Catalog.” 1 A complete catalog of all publications by NIST authors is issued annually as a supplement to SP 305 “Publications of the National Institute of Standards & Technology.” Announcements and listings of recent NIST publications and services are published in each issue of the bimonthly “NIST Journal of Research” 2\n\n     and the NIST monthly magazine, “Dimensions/NIST” 2."}
+{"query": "What is the acceptable tolerance level for cranberries?", "pos": "(1) Having determined the errors on each dimension and given to each its proper sign (see § 241.5), add the errors on the effective diameter of head and the distance between heads algebraically and multiply the result by 1.67 (or 5/3). Then add this result to the error on the circumference of bulge algebraically. If the result obtained is not greater than the tolerance given in the following table for the proper subdivision, then the barrel is within the tolerance allowed; if the result is greater than this tolerance, then the barrel is not within the tolerance allowed.\n\n  \n\n  \n\n    \n\nSize of subdivision\n\nTolerance\n\nFor fruits, vegetables, and other dry commodities (inches)\n\nFor cranberries (inches)"}
+{"query": "What tools and techniques might be listed in solicitation announcements for specific industry sectors?", "pos": "Specific industry sectors to be addressed and sub-categories of tools and techniques may be specified in solicitations. These sectors or sub-categories will be specified in the solicitation announcement. Examples of tools and techniques include, but are not limited to, manufacturing assessment tools, environmental benchmarking tools, training delivery programs, electronically accessible environmental information resources, environmental demonstration facilities, software tools, etc. Projects must be completed within the scope of the effort proposed and should not require on-going federal support.\n\n  (c) Award period. Projects initiated under this category may be carried out over up to three years. Proposals selected for award will receive all funding from currently available funds. If an application is selected for funding, DOC has no obligation to provide any additional future funding in connection with that award."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/law-lex_files_400k-500k.jsonl

@@ -0,0 +1,3 @@
+{"query": "When is the deadline for quarterly returns per § 53.153(a)?", "pos": "[T.D. ATF-308, 56 FR 303, Jan. 3, 1991, as amended by T.D. ATF-330, 57 FR 40325, Sept. 3, 1992. Redesignated in part by T.D. ATF-365, 60 FR 33670, June 28, 1995]\n\n\n\n\n\n§ 53.153\n\nTime for filing returns.\n\n(a) Quarterly returns. Each return required to be made under § 53.151(a) for a return period of one calendar quarter shall be filed on or before the last day of the first calendar month following the close of the period for which it is made."}
+{"query": "How are federal tax liens enforced?", "pos": "The satisfaction of the levy described in paragraph (b) of this section by an insuring organization shall be without prejudice to any civil action for the enforcement of any Federal tax lien with respect to a life insurance or endowment contract. Thus, this levy procedure is not the exclusive means of subjecting the life insurance and endowment contracts of the person against whom a tax is assessed to the collection of the person's unpaid assessment. The United States may choose to foreclose the tax lien in any case where it is appropriate, as, for example, to reach the cash surrender value (as distinguished from cash loan value) of a life insurance or endowment contract.\n\n(e) Cross references."}
+{"query": "Taxpayers must compile detailed lists for each tax jurisdiction, including names, addresses, and tax classifications.", "pos": "(b) Multiple locations and/or classes of tax. A taxpayer subject to special tax for the same period at more than one location or for more than one class of tax must—\n\n(1) File one special tax return, TTB Form 5630.5t, with payment of tax, to cover all such locations and classes of tax; and\n\n(2) Prepare, in duplicate, a list identified with the taxpayer's name, address (as shown on TTB Form 5630.5t), employer identification number, and period covered by the return. The list must show, by State, the name, address, and tax class of each location for which special tax is being paid."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/law-lex_files_500k-600k.jsonl

@@ -0,0 +1,3 @@
+{"query": "Define \"project.\"", "pos": "Private, as applied to an agency, organization, or institution, means that it is not under Federal or public supervision or control.\n\n\n\nProject means the activity described in an application.\n\n\n\nProject component means an activity, strategy, intervention, process, product, practice, or policy included in a project. Evidence may pertain to an individual project component or to a combination of project components (e.g., training teachers on instructional practices for English learners and follow-on coaching for these teachers).\n\n\n\nProject period means the period established in the award document during which Federal sponsorship begins and ends (See, 2 CFR 200.77 Period of performance)."}
+{"query": "How do you file and respond to written motions in legal proceedings?", "pos": "The ALJ may require that oral motions be reduced to writing.\n\n(c) Within 15 days after a written motion is served, or such other time as may be fixed by the ALJ, any party may file a response to the motion.\n\n(d) The ALJ may not grant a written motion before the time for filing responses to the motion has expired, except upon consent of the parties or following a hearing on the motion, but may overrule or deny the motion without awaiting a response.\n\n(e) The ALJ shall make a reasonable effort to dispose of all outstanding motions prior to the beginning of the hearing.\n\n(Authority: 31 U.S.C. 3803(g)(3)(A))"}
+{"query": "What does the Credit Enhancement for Charter School Facilities Program do?", "pos": "(3) Assist charter schools with the predevelopment costs required to assess sites for the purpose of acquiring (by purchase, lease, donation, or otherwise) an interest (including an interest held by a third party for the benefit of a charter school) in improved or unimproved real property or constructing new facilities, or renovating, repairing, or altering existing facilities, and that are necessary to commence or continue the operation of a charter school.\n\n  (c) Grantees may demonstrate innovative credit enhancement initiatives while meeting the program purposes under paragraph (b) of this section.\n\n  (d) For the purposes of these regulations, the Credit Enhancement for Charter School Facilities Program includes grants made under the Charter School Facilities Financing Demonstration Grant Program.\n\n  [70 FR 15003, Mar."}

FlagEmbedding/evaluation/air_bench/examples/long-doc/law-lex_files_600k-700k.jsonl

@@ -0,0 +1,3 @@
+{"query": "What happens to the loss?", "pos": "This loss shall be the measured loss less the net gain of any voice frequency repeaters in the circuit. Testing shall also be conducted to verify that the loss increases gradually as the frequency increases. The loss on H88 loaded loops should be down only slightly at 2.8 kHz but drop rapidly above 2.8 kHz. The loss on D66 loaded loops shall be fairly constant to about 3.4 kHz and there shall be good response at 4.0 kHz. When voice frequency repeaters are in the circuit there will be some frequency weighting in the build-out network and the loss at the higher frequencies will be greater than for nonrepeatered loops."}
+{"query": "We'll let all borrowers know by mail or email whenever there's a new Federal Register document about contract forms.", "pos": "The amendment may change the existing identification of a listed contract form; for example, changing the issuance date of a listed contract form or by identifying a new required contract form. The notice of rulemaking will describe the new standard contract form or the substantive change in the listed contract form, as the case may be, and the issues involved. The standard contract form or relevant portions thereof may be appended to the supplementary information section of the notice of rulemaking. As appropriate, the notice of rulemaking shall provide an opportunity for interested persons to provide comments. A copy of each such Federal Register document shall be sent by regular or electronic mail to all borrowers.\n\n[63 FR 58285, Oct. 30, 1998]"}
+{"query": "Which renewable energy projects can get funding through grant proposals?", "pos": "A grant project is eligible if it improves, or maintains energy services, or reduces the costs of providing energy services to eligible communities. Examples of eligible activities include, but are not limited to, the acquisition, construction, replacement, repair, or improvement of:\n\n(a) Electric generation, transmission, and distribution facilities, equipment, and services serving the eligible community;\n\n(b) Natural gas distribution or storage facilities and associated equipment and activities serving the eligible community;\n\n(c) Petroleum product storage and handling facilities serving residential or community use.\n\n(d) Renewable energy facilities used for on-grid or off-grid electric power generation, water or space heating, or process heating and power for the eligible community;\n\n(e) Backup up or emergency power generation or energy storage equipment, including distributed generation, to serve the eligible community; and"}

FlagEmbedding/evaluation/air_bench/examples/qa/arxiv.jsonl

@@ -0,0 +1,3 @@
+{"query": "How does the amount of energy affect how stuff scatters?", "pos": "it has been suggested that one may construct a lorentz - invariant noncommutative field theory by extending the coordinate algebra to additional, fictitious coordinates that transform nontrivially under the lorentz group. integration over these coordinates in the action produces a four - dimensional effective theory with lorentz invariance intact. previous applications of this approach, in particular to a specific construction of noncommutative qed, have been studied only in a low - momentum approximation. here we discuss lorentz - invariant field theories in which the relevant physics can be studied without requiring an expansion in the inverse scale of noncommutativity. qualitatively, we find that tree - level scattering cross sections are dramatically suppressed as the center - of - mass energy exceeds the scale of noncommutativity, that cross sections that are isotropic in the commutative limit can develop a pronounced angular dependence, and that nonrelativistic potentials (for example, the coloumb potential) become nonsingular at the origin. we consider a number of processes in noncommutative qed that may be studied at a future linear collider. we also give an example of scattering via a four - fermion operator in which the noncommutative modifications of the interaction can unitarize the tree - level amplitude, without requiring any other new physics in the ultraviolet."}
+{"query": "Why go for canonical instead of grand-canonical?", "pos": "the production of hadrons in relativistic heavy ion collisions is studied using a statistical ensemble with thermal and chemical equilibrium. special attention is given to exact conservation laws, i.e. certain charges are treated canonically instead of using the usual grand canonical approach. for small systems, the exact conservation of baryon number, strangeness and electric charge is to be taken into account. we have derived compact, analytical expressions for particle abundances in such ensemble. as an application, the change in @xmath0 ratios in ags experiments with different interaction system sizes is well reproduced. the canonical treatment of three charges becomes impractical very quickly with increasing system size. thus, we draw our attention to exact conservation of strangeness, and treat baryon number and electric charge grand canonically. we present expressions for particle abundances in such ensemble as well, and apply them to reproduce the large variety of particle ratios in gsi sis 2 a gev ni  ni experiments. at the energies considered here, the exact strangeness conservation fully accounts for strange particle suppression, and no extra chemical factor is needed.    [ on the exact conservation laws in thermal models ]"}
+{"query": "How good is the mean-field approximation at guessing the ground state features of molecular stuff?", "pos": "we present a model for molecular materials made up of polar and polarizable molecular units. a simple two state model is adopted for each molecular site and only classical intermolecular interactions are accounted for, neglecting any intermolecular overlap. the complex and interesting physics driven by interactions among polar and polarizable molecules becomes fairly transparent in the adopted model. collective effects are recognized in the large variation of the molecular polarity with supramolecular interactions, and cooperative behavior shows up with the appearance, in attractive lattices, of discontinuous charge crossovers. the mf approximation proves fairly accurate in the description of the gs properties of mm, including static linear and non - linear optical susceptibilities, apart from the region in the close proximity of the discontinuous charge crossover. sizeable deviations from the excitonic description are recognized both in the excitation spectrum and in linear and non - linear optical responses. new and interesting phenomena are recognized near the discontinuous charge crossover for non - centrosymmetric clusters, where the primary photoexcitation event corresponds to a multielectron transfer."}

FlagEmbedding/evaluation/air_bench/examples/qa/finance.jsonl

@@ -0,0 +1,3 @@
+{"query": "What is the effect of the LME Singapore Contract on trade dynamics?", "pos": "The London Metal Exchange's, LME,\ndecision to introduce a dollar-denominated aluminium contract,\nwith the Port of Singapore listed as a delivery point, is a\npositive move, physical traders and LME dealers said.\n    Earlier this week the LME declared that a 99.70 pct minimum\npurity aluminium contract would commence trading on June 1,\n1987, alongside its long-established sterling-based 99.50 pct\ncontract.\n    This is the LME's first dollar contract and non-European\ndelivery point, and the Board and Committee are looking at\nSingapore as a delivery point for other contracts.\n    Trade sources said the LME's new contract will conform with\nexisting industry practice, where 99.70 standard re-melt\nmaterial, priced in dollars, is most commonly traded.\n    The location of a warehouse in Singapore is also a positive\nmove by the LME, given its ideal location for Australian and\nJapanese traders, who would be able to place metal on to\nwarrant speedily and relatively inexpensively, they said.\n    Hedging during the LME ring sessions becomes much simpler\nwith a dollar contract. At present pre-market trading is almost\nexclusively dollar-based, but currency conversions have to be\ndone during the sterling rings, they added.\n    LME ring dealers said the new contract would match more\nclosely trade requirements and possibly alleviate some of the\nrecent wide backwardations.\n    Very little physical business is now done in 99.50 pct\npurity metal, nearly all of which is produced in Eastern Bloc\ncountries, such as Romania.\n    The Soviet Union also produces 99.50 pct, but has declined\nas an exporter recently, they said.\n    Some dealers said the new 99.70 contract may suffer from\nliquidity problems initially, as business may continue to\ncentre on the present good ordinary brand (gob) contract, where\nthere are many holders of large short positions on the LME.\n    But others said the new contract would soon attract trading\ninterest, given that much 99.70 metal has already been\nattracted to the LME's warehouses by backwardations.\n    The LME also has a much more viable liquidity base for a\nnew contract, compared to the Comex market in New York, where\nhigh grade aluminium futures are not particularly active, they\nsaid.\n    Thus, it seems likely that the sterling contract will\neventually lose trading interest and volumes will decline. Like\nstandard zinc, which was superseded by a high grade contract,\ngob aluminium will probably be replaced, although the process\nin this case may take longer, they added.\n    Forming a new contract and establishing a Singapore\nwarehouse are constructive moves by the LME but backwardations,\nwhich make physical trading difficult, would not totally\ndisappear as a result, the trade sources said.\n    These premiums for prompt metal have become a\nsemi-permanent feature over the last year, due to increased\nbusiness and volatility in traded options, and are presently\naround 50 stg.\n    Increasingly large granting of option positions has been\ntaking place. When some of these are declared and exercised at\nthe end of the relevant month, physical tightness and squeezes\naround these dates are commonplace, they said.\n    Listing Singapore as a delivery point allows Far Eastern\noperators to deliver aluminium into a LME warehouse instead of\nhaving to cover.\n    But tightness and backwardations are seen continuing, even\nthough the LME's new option contracts widen the gap between the\ndeclaration and prompt dates.\n    These will be due on the first and third Wednesday of the\nmonth, whereas at present most fall on the 20th and 25th.\n    Backwardations will remain while operators continue to\ngrant options where potential tonnage to be delivered exceeds\naluminium stock levels, an LME option trader said.\n Reuter\n"}
+{"query": "Please provide the estimated quantity of the broad monetary aggregate designated as M-3, which encompasses the extensive range of financial assets held principally by households, as recorded in the month of February.", "pos": "South African year-on-year broadly\ndefined M-3 money supply growth slowed to 8.62 pct in January\nfrom 9.32 pct in December, Reserve Bank figures show.\n    M-3 fell to 77.98 billion rand in January from 79.31\nbillion in December, while preliminary February figures show\nM-3 at 79.42 billion rand for a year-on-year rise of 10.63 pct.\n    M-2 showed a rise of 5.09 pct for January at 55.68 billion\nrand after 4.30 pct in December, M-1 16.72 pct at 5.12 billion\nafter 12.80 pct and M-1A 22.79 pct at 14.30 billion rand after\n20.54 pct.\n REUTER\n"}
+{"query": "When did Reagan impose tariffs?", "pos": "The White House issued a\nlist of Japanese exports to covered by the 100 pct tariffs\nimposed by President Reagan.\n    - Automatic data processing machines (1986 imports worth\n180 mln dlrs), including certain desk and lap models with\nmicroprocessor-based calculating mechanism capable of handling\nwords of at least 16-bits off the microprocessor;\n    - Complete color television sets, with 18, 19 or 20 inch\nscreens (1986 imports 90 mln dlrs);\n    - Power tools, including certain drills, percussion\nhammers, sanders, polishers, grinders.\n Reuter\n"}

FlagEmbedding/evaluation/air_bench/examples/qa/healthcare.jsonl

@@ -0,0 +1,3 @@
+{"query": "Which technique was employed to assess the blood pressure in Wistar rats subjected to various sodium intake regimens?", "pos": "Male Wistar rats were fed on normal- (0.5% Na(+); NS), high- (3.12% Na(+); HS),or low-sodium (0.06% Na(+); LS) diets for 3, 6, and 9 weeks after weaning. Blood pressure (BP) was measured using a computerized tail-cuff system. An intravenous insulin tolerance test (ivITT) was performed in fasted animals. At the end of each period, rats were killed and blood samples were collected for glucose and insulin determinations. The white adipose tissue (WAT) from abdominal and inguinal subcutaneous (SC) and periepididymal (PE) depots were weighed and processed for adipocyte isolation and measurement of in vitro rates of insulin-stimulated 2-deoxy-D-[(3)H]-glucose uptake (2DGU) and conversion of -[U-(14)C]-glucose into (14)CO(2)."}
+{"query": "How long were the kids treated with chemo for their stomach lymphoma?", "pos": "Only two patients, 5 and 12 years old, with primary gastric NHL were found. Upper gastroduodenal endoscopy detected an ulcer in the lesser curvature of the body of the stomach, in both cases. Endoscopy revealed a moderate chronic gastritis in the antrum of both patients that was H. pylori associated in one of them who also suffered from chronic gastritis. Biopsy specimens demonstrated infiltration by Burkitt lymphoma (BL). The two patients received chemotherapy for 6 months. Additionally, one of the two patients received a triple therapy regimen with bismuth, amoxicillin, and metronidazole for H. pylori. Fifteen and six years later they are in complete remission, free of symptoms."}
+{"query": "What are the correlations between the volume of tissue resected and the resulting clinical outcomes?", "pos": "Between May 2011 and April 2013, LSG was performed in 102 consecutive patients undergoing bariatric surgery. Two patients were excluded, and data from the remaining 100 patients were analyzed in this study. Patients were divided into three groups according to the following resected stomach volume: 700-1,200 mL (group A, n = 21), 1,200-1,700 mL (group B, n = 62), and>1,700 mL (group C, n = 17). Mean values were compared among the groups by analysis of variance."}

FlagEmbedding/evaluation/air_bench/examples/qa/msmarco.jsonl

@@ -0,0 +1,3 @@
+{"query": "Corn on the cob boiling time?", "pos": "Corn on the Cob - Boiled In a large pot, enough to hold the corn, fill it with water to cover the corn (the corn should float). On a medium heat allow the pot of water to boil. Once the water is boiled, add in the corn into the pot and cover. Cook for 10-15 minutes depending on how soft you want your corn. Drain water and remove corn on the cob."}
+{"query": "Nitrous oxide is commonly used as an anesthetic or analgesic in medical and dental procedures.", "pos": "Nitrous oxide Nitrous oxide has significant medical uses, especially in surgery and dentistry, for its anaesthetic and analgesic effects. Its name laughing gas is due to the euphoric effects of inhaling it, a property that has led to its recreational use as a dissociative anaesthetic."}
+{"query": "At what temp do you start to roast?", "pos": "How long to cook 2.3 lb pork tenderloin in oven? Best Answer: For your seasoned pork loin, preheat your oven to 400 degrees F or (200C). Place the seasoned pork in the preheated oven and immediately turn the oven down to 350F (175C). Roast the pork loin or tenderloin for about 70-90 minutes or until it reaches an internal temperature of 145-150F (73-75C) degrees. If you prefer your pork cooked to medium well, cook it to an internal temperature of 155-160F (78-80C) degrees."}

FlagEmbedding/evaluation/air_bench/examples/qa/web.jsonl

@@ -0,0 +1,3 @@
+{"query": "Identify the principal landmarks that are emblematic of the Baha'i religious tradition.", "pos": "House of Baha'u'llah, Baghdad\n\"Grieve not, O House of God, if the veil of thy sanctity be rent asunder by the infidels. God hath, in the world of creation, adorned thee with the jewel of His remembrance. Such an ornament no man can, at any time, profane. Towards thee the eyes of thy Lord shall, under all conditions, remain directed. He, verily, will incline His ear to the prayer of every one that visiteth thee, who will circle around thee, and calleth upon Him in thy name. He, in truth, is the Forgiving, the All-Merciful.\"\n(Gleanings from the Writings of Bahá’u’lláh, LVII, part 7)"}
+{"query": "Which type of healthcare professional should one consult regarding the sensation of tingling in the feet?", "pos": "“Tingly feet\" can be a sign of nerve loss. The nerves in the feet come from the lower back. Pressure or chemical change in the nerve can cause a tingling sensation in the feet. Any sensation that is out of the ordinary can be an early sign of neurologic or vascular problems. In addition to tingling, feet may feel numb or feel like they are \"falling asleep.\" There may also be a burning sensation in the feet.\nDiabetes is one of the most common medical conditions with which \"tingly feet\" can be associated. A thorough evaluation by a foot and ankle surgeon is advised to determine the cause of \"tingly feet.\"\nSee also Diabetic Peripheral Neuropathy."}
+{"query": "How big is the old Kaguru Basket?", "pos": "Home — Vintage Kaguru Basket from Tanzania - 15\" x 10.5\"\nVintage Kaguru Basket from Tanzania - 15\" x 10.5\"\nFrom Tanzania, East Africa, these baskets are used the same way all other similar baskets are used everywhere else in Africa. They are the primary vessel for storage and transportation of grain, fruit, vegetables and any other food item. Baskets of this type are often seen in markets containing food for sale there. The patina on the rims and side of this basket suggests it was handled often. There is residue of some sort on the interior surfaces which proves they were often in use in the way I have described.\nAfter 36 years of traveling in Africa and buying similar African items, we thought we had seen it all. These are truly amazing baskets, at least to us.\nThis basket measures 15\" x 10.5\" (38cm x 26.75cm)"}

FlagEmbedding/evaluation/air_bench/runner.py

@@ -0,0 +1,65 @@
+from typing import Union, Tuple
+from air_benchmark import AIRBench
+
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalRunner,
+    EvalDenseRetriever, EvalReranker
+)
+
+from .arguments import AIRBenchEvalArgs, AIRBenchEvalModelArgs
+
+
+class AIRBenchEvalRunner:
+    """
+    Evaluation runner for AIR Bench.
+    
+    Args:
+        eval_args (AIRBenchEvalArgs): :class:AIRBenchEvalArgs object with the evaluation arguments.
+        model_args (AIRBenchEvalModelArgs): :class:AIRBenchEvalModelArgs object with the model arguments.
+    """
+    def __init__(
+        self,
+        eval_args: AIRBenchEvalArgs,
+        model_args: AIRBenchEvalModelArgs,
+    ):
+        self.eval_args = eval_args
+        self.model_args = model_args
+        self.model_args.cache_dir = model_args.model_cache_dir
+
+        self.retriever, self.reranker = self.load_retriever_and_reranker()
+
+    def load_retriever_and_reranker(self) -> Tuple[EvalDenseRetriever, Union[EvalReranker, None]]:
+        """Load retriever and reranker for evaluation
+
+        Returns:
+            Tuple[EvalDenseRetriever, Union[EvalReranker, None]]: A :class:EvalDenseRetriever object for retrieval, and a
+                :class:EvalReranker object if reranker provided.
+        """
+        embedder, reranker = AbsEvalRunner.get_models(self.model_args)
+        retriever = EvalDenseRetriever(
+            embedder,
+            search_top_k=self.eval_args.search_top_k,
+            overwrite=self.eval_args.overwrite
+        )
+        if reranker is not None:
+            reranker = EvalReranker(reranker, rerank_top_k=self.eval_args.rerank_top_k)
+        return retriever, reranker
+
+    def run(self):
+        """
+        Run the whole evaluation.
+        """
+        evaluation = AIRBench(
+            benchmark_version=self.eval_args.benchmark_version,
+            task_types=self.eval_args.task_types,
+            domains=self.eval_args.domains,
+            languages=self.eval_args.languages,
+            splits=self.eval_args.splits,
+            cache_dir=self.eval_args.cache_dir,
+        )
+        evaluation.run(
+            self.retriever,
+            reranker=self.reranker,
+            output_dir=self.eval_args.output_dir,
+            overwrite=self.eval_args.overwrite,
+        )

FlagEmbedding/evaluation/beir/__init__.py

@@ -0,0 +1,14 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalModelArgs as BEIREvalModelArgs,
+)
+
+from .data_loader import BEIREvalDataLoader
+from .arguments import BEIREvalArgs
+from .runner import BEIREvalRunner
+
+__all__ = [
+    "BEIREvalArgs",
+    "BEIREvalModelArgs",
+    "BEIREvalRunner",
+    "BEIREvalDataLoader",
+]

FlagEmbedding/evaluation/beir/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.beir import (
+    BEIREvalArgs, BEIREvalModelArgs,
+    BEIREvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        BEIREvalArgs,
+        BEIREvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: BEIREvalArgs
+    model_args: BEIREvalModelArgs
+
+    runner = BEIREvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/beir/arguments.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass, field
+
+from FlagEmbedding.abc.evaluation.arguments import AbsEvalArgs
+
+
+@dataclass
+class BEIREvalArgs(AbsEvalArgs):
+    """
+    Argument class for BEIR evaluation.
+    """
+    use_special_instructions: bool = field(
+        default=False, metadata={"help": "Whether to use specific instructions in `prompts.py` for evaluation. Default: False"}
+    )

FlagEmbedding/evaluation/beir/data_loader.py

@@ -0,0 +1,471 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+from beir import util
+from beir.datasets.data_loader import GenericDataLoader
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class BEIREvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for BEIR.
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return ['arguana', 'climate-fever', 'cqadupstack', 'dbpedia-entity', 'fever', 'fiqa', 'hotpotqa', 'msmarco', 'nfcorpus', 'nq', 'quora', 'scidocs', 'scifact', 'trec-covid', 'webis-touche2020']
+
+    def available_sub_dataset_names(self, dataset_name: Optional[str] = None) -> List[str]:
+        """
+        Get the available sub-dataset names.
+
+        Args:
+            dataset_name (Optional[str], optional): All the available sub-dataset names. Defaults to ``None``.
+
+        Returns:
+            List[str]: All the available sub-dataset names.
+        """
+        if dataset_name == 'cqadupstack':
+            return ['android', 'english', 'gaming', 'gis', 'mathematica', 'physics', 'programmers', 'stats', 'tex', 'unix', 'webmasters', 'wordpress']
+        return None
+
+    def available_splits(self, dataset_name: Optional[str] = None) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (str): Dataset name.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        if dataset_name == 'msmarco':
+            return ['dev']
+        return ['test']
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        sub_dataset_name: Optional[str] = None,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            sub_dataset_name (Optional[str]): Name of the sub-dataset. Defaults to ``None``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        if dataset_name != 'cqadupstack':
+            corpus = datasets.load_dataset(
+                'BeIR/{d}'.format(d=dataset_name),
+                'corpus',
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,
+                download_mode=self.hf_download_mode
+            )['corpus']
+
+            if save_dir is not None:
+                os.makedirs(save_dir, exist_ok=True)
+                save_path = os.path.join(save_dir, "corpus.jsonl")
+                corpus_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                        _data = {
+                            "id": data["_id"],
+                            "title": data["title"],
+                            "text": data["text"]
+                        }
+                        corpus_dict[data["_id"]] = {
+                            "title": data["title"],
+                            "text": data["text"]
+                        }
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+            else:
+                corpus_dict = {data["docid"]: {"title": data["title"], "text": data["text"]} for data in tqdm(corpus, desc="Loading corpus")}
+        else:
+            url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(dataset_name)
+            data_path = util.download_and_unzip(url, self.cache_dir)
+            full_path = os.path.join(data_path, sub_dataset_name)
+            corpus, _, _ = GenericDataLoader(data_folder=full_path).load(split="test")
+            if save_dir is not None:
+                new_save_dir = os.path.join(save_dir, sub_dataset_name)
+                os.makedirs(new_save_dir, exist_ok=True)
+                save_path = os.path.join(new_save_dir, "corpus.jsonl")
+                corpus_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for _id in tqdm(corpus.keys(), desc="Loading corpus"):
+                        _data = {
+                            "id": _id,
+                            "title": corpus[_id]["title"],
+                            "text": corpus[_id]["text"]
+                        }
+                        corpus_dict[_id] = {
+                            "title": corpus[_id]["title"],
+                            "text": corpus[_id]["text"]
+                        }
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+            else:
+                corpus_dict = {_id: {"title": corpus[_id]["title"], "text": corpus[_id]["text"]} for _id in tqdm(corpus.keys(), desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: Optional[str] = None,
+        sub_dataset_name: Optional[str] = None,
+        split: str = 'dev',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            sub_dataset_name (Optional[str]): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): Split of the dataset. Defaults to ``'dev'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        if dataset_name != 'cqadupstack':
+            qrels = datasets.load_dataset(
+                'BeIR/{d}-qrels'.format(d=dataset_name),
+                split=split if split != 'dev' else 'validation', 
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,
+                download_mode=self.hf_download_mode
+            )
+
+            if save_dir is not None:
+                os.makedirs(save_dir, exist_ok=True)
+                save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+                qrels_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for data in tqdm(qrels, desc="Loading and Saving qrels"):
+                        qid, docid, rel = str(data['query-id']), str(data['corpus-id']), int(data['score'])
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": rel
+                        }
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid][docid] = rel
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+            else:
+                qrels_dict = {}
+                for data in tqdm(qrels, desc="Loading queries"):
+                    qid, docid, rel = str(data['query-id']), str(data['corpus-id']), int(data['score'])
+                    if qid not in qrels_dict:
+                        qrels_dict[qid] = {}
+                    qrels_dict[qid][docid] = rel
+        else:
+            url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(dataset_name)
+            data_path = util.download_and_unzip(url, self.cache_dir)
+            full_path = os.path.join(data_path, sub_dataset_name)
+            _, _, qrels = GenericDataLoader(data_folder=full_path).load(split="test")
+            if save_dir is not None:
+                new_save_dir = os.path.join(save_dir, sub_dataset_name)
+                os.makedirs(new_save_dir, exist_ok=True)
+                save_path = os.path.join(new_save_dir, f"{split}_qrels.jsonl")
+                qrels_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for qid in tqdm(qrels.keys(), desc="Loading and Saving qrels"):
+                        for docid in tqdm(qrels[qid].keys()):
+                            rel = int(qrels[qid][docid])
+                            _data = {
+                                "qid": qid,
+                                "docid": docid,
+                                "relevance": rel
+                            }
+                            if qid not in qrels_dict:
+                                qrels_dict[qid] = {}
+                            qrels_dict[qid][docid] = rel
+                            f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+            else:
+                qrels_dict = {}
+                for qid in tqdm(qrels.keys(), desc="Loading qrels"):
+                    for docid in tqdm(qrels[qid].keys()):
+                        rel = int(qrels[qid][docid])
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid][docid] = rel
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: Optional[str] = None,
+        sub_dataset_name: Optional[str] = None,
+        split: str = 'test',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            sub_dataset_name (Optional[str]): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): Split of the dataset. Defaults to ``'dev'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        qrels = self.load_qrels(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+
+        if dataset_name != 'cqadupstack':
+            queries = datasets.load_dataset(
+                'BeIR/{d}'.format(d=dataset_name), 
+                'queries', 
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,
+                download_mode=self.hf_download_mode
+            )['queries']
+
+            if save_dir is not None:
+                os.makedirs(save_dir, exist_ok=True)
+                save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+                queries_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for data in tqdm(queries, desc="Loading and Saving queries"):
+                        qid, query = data['_id'], data['text']
+                        if qid not in qrels.keys(): continue
+                        _data = {
+                            "id": qid,
+                            "text": query
+                        }
+                        queries_dict[qid] = query
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+            else:
+                queries_dict = {}
+                for data in tqdm(queries, desc="Loading queries"):
+                    qid, query = data['_id'], data['text']
+                    if qid not in qrels.keys(): continue
+                    queries_dict[qid] = query
+        else:
+            url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(dataset_name)
+            data_path = util.download_and_unzip(url, self.cache_dir)
+            full_path = os.path.join(data_path, sub_dataset_name)
+            _, queries, _ = GenericDataLoader(data_folder=full_path).load(split="test")
+            if save_dir is not None:
+                new_save_dir = os.path.join(save_dir, sub_dataset_name)
+                os.makedirs(new_save_dir, exist_ok=True)
+                save_path = os.path.join(new_save_dir, f"{split}_queries.jsonl")
+                queries_dict = {}
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for qid in tqdm(queries.keys(), desc="Loading and Saving queries"):
+                        query = queries[qid]
+                        if qid not in qrels.keys(): continue
+                        _data = {
+                            "id": qid,
+                            "text": query
+                        }
+                        queries_dict[qid] = query
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+            else:
+                queries_dict = {}
+                for qid in tqdm(queries.keys(), desc="Loading queries"):
+                    query = queries[qid]
+                    if qid not in qrels.keys(): continue
+                    queries_dict[qid] = query
+        return datasets.DatasetDict(queries_dict)
+
+    def load_corpus(self, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None) -> datasets.DatasetDict:
+        """Load the corpus from the dataset.
+
+        Args:
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
+        """
+        if self.dataset_dir is not None:
+            if dataset_name is None:
+                save_dir = self.dataset_dir
+            else:
+                save_dir = os.path.join(self.dataset_dir, dataset_name)
+            return self._load_local_corpus(save_dir, dataset_name=dataset_name, sub_dataset_name=sub_dataset_name)
+        else:
+            return self._load_remote_corpus(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name)
+
+    def load_qrels(self, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None, split: str = 'test') -> datasets.DatasetDict:
+        """Load the qrels from the dataset.
+
+        Args:
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): The split to load relevance from. Defaults to ``'test'``.
+
+        Raises:
+            ValueError
+
+        Returns:
+            datasets.DatasetDict: A dict of relevance of query and document.
+        """
+        if self.dataset_dir is not None:
+            if dataset_name is None:
+                save_dir = self.dataset_dir
+            else:
+                checked_dataset_names = self.check_dataset_names(dataset_name)
+                if len(checked_dataset_names) == 0:
+                    raise ValueError(f"Dataset name {dataset_name} not found in the dataset.")
+                dataset_name = checked_dataset_names[0]
+
+                save_dir = os.path.join(self.dataset_dir, dataset_name)
+
+            return self._load_local_qrels(save_dir, dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+        else:
+            return self._load_remote_qrels(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+
+    def load_queries(self, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None, split: str = 'test') -> datasets.DatasetDict:
+        """Load the queries from the dataset.
+
+        Args:
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): The split to load queries from. Defaults to ``'test'``.
+
+        Raises:
+            ValueError
+
+        Returns:
+            datasets.DatasetDict: A dict of queries with id as key, query text as value.
+        """
+        if self.dataset_dir is not None:
+            if dataset_name is None:
+                save_dir = self.dataset_dir
+            else:
+                checked_dataset_names = self.check_dataset_names(dataset_name)
+                if len(checked_dataset_names) == 0:
+                    raise ValueError(f"Dataset name {dataset_name} not found in the dataset.")
+                dataset_name = checked_dataset_names[0]
+
+                save_dir = os.path.join(self.dataset_dir, dataset_name)
+
+            return self._load_local_queries(save_dir, dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+        else:
+            return self._load_remote_queries(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+
+    def _load_local_corpus(self, save_dir: str, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None) -> datasets.DatasetDict:
+        """Load corpus from local dataset.
+
+        Args:
+            save_dir (str): Path to save the loaded corpus.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
+        """
+        if sub_dataset_name is None:
+            corpus_path = os.path.join(save_dir, 'corpus.jsonl')
+        else:
+            corpus_path = os.path.join(save_dir, sub_dataset_name, 'corpus.jsonl')
+        if self.force_redownload or not os.path.exists(corpus_path):
+            logger.warning(f"Corpus not found in {corpus_path}. Trying to download the corpus from the remote and save it to {save_dir}.")
+            return self._load_remote_corpus(dataset_name=dataset_name, save_dir=save_dir, sub_dataset_name=sub_dataset_name)
+        else:
+            if sub_dataset_name is not None:
+                save_dir = os.path.join(save_dir, sub_dataset_name)
+            corpus_data = datasets.load_dataset('json', data_files=corpus_path, cache_dir=self.cache_dir)['train']
+
+            corpus = {}
+            for e in corpus_data:
+                corpus[e['id']] = {'title': e.get('title', ""), 'text': e['text']}
+
+            return datasets.DatasetDict(corpus)
+
+    def _load_local_qrels(self, save_dir: str, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None, split: str = 'test') -> datasets.DatasetDict:
+        """Load relevance from local dataset.
+
+        Args:
+            save_dir (str):  Path to save the loaded relevance.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
+
+        Raises:
+            ValueError
+
+        Returns:
+            datasets.DatasetDict: A dict of relevance of query and document.
+        """
+        checked_split = self.check_splits(split, dataset_name=dataset_name)
+        if len(checked_split) == 0:
+            raise ValueError(f"Split {split} not found in the dataset.")
+        split = checked_split[0]
+
+        if sub_dataset_name is None:
+            qrels_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+        else:
+            qrels_path = os.path.join(save_dir, sub_dataset_name, f"{split}_qrels.jsonl")
+        if self.force_redownload or not os.path.exists(qrels_path):
+            logger.warning(f"Qrels not found in {qrels_path}. Trying to download the qrels from the remote and save it to {save_dir}.")
+            return self._load_remote_qrels(dataset_name=dataset_name, split=split, sub_dataset_name=sub_dataset_name, save_dir=save_dir)
+        else:
+            if sub_dataset_name is not None:
+                save_dir = os.path.join(save_dir, sub_dataset_name)
+            qrels_data = datasets.load_dataset('json', data_files=qrels_path, cache_dir=self.cache_dir)['train']
+
+            qrels = {}
+            for data in qrels_data:
+                qid = data['qid']
+                if qid not in qrels:
+                    qrels[qid] = {}
+                qrels[qid][data['docid']] = data['relevance']
+
+            return datasets.DatasetDict(qrels)
+
+    def _load_local_queries(self, save_dir: str, dataset_name: Optional[str] = None, sub_dataset_name: Optional[str] = None, split: str = 'test') -> datasets.DatasetDict:
+        """Load queries from local dataset.
+
+        Args:
+            save_dir (str):  Path to save the loaded queries.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
+
+        Raises:
+            ValueError
+
+        Returns:
+            datasets.DatasetDict: A dict of queries with id as key, query text as value.
+        """
+        checked_split = self.check_splits(split, dataset_name=dataset_name)
+        if len(checked_split) == 0:
+            raise ValueError(f"Split {split} not found in the dataset.")
+        split = checked_split[0]
+
+        if sub_dataset_name is None:
+            queries_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+        else:
+            queries_path = os.path.join(save_dir, sub_dataset_name, f"{split}_queries.jsonl")
+        if self.force_redownload or not os.path.exists(queries_path):
+            logger.warning(f"Queries not found in {queries_path}. Trying to download the queries from the remote and save it to {save_dir}.")
+            return self._load_remote_queries(dataset_name=dataset_name, split=split, sub_dataset_name=sub_dataset_name, save_dir=save_dir)
+        else:
+            if sub_dataset_name is not None:
+                save_dir = os.path.join(save_dir, sub_dataset_name)
+            queries_data = datasets.load_dataset('json', data_files=queries_path, cache_dir=self.cache_dir)['train']
+
+            queries = {e['id']: e['text'] for e in queries_data}
+            return datasets.DatasetDict(queries)

FlagEmbedding/evaluation/beir/evaluator.py

@@ -0,0 +1,454 @@
+import json
+import logging
+import os
+import json
+from typing import Dict, Optional, List, Union
+
+from FlagEmbedding.abc.evaluation import AbsEvaluator, EvalRetriever, EvalReranker
+
+logger = logging.getLogger(__name__)
+
+
+class BEIREvaluator(AbsEvaluator):
+    """
+    Evaluator class of BEIR 
+    """
+    def check_data_info(
+        self,
+        data_info: Dict[str, str],
+        model_name: str,
+        reranker_name: str,
+        split: str,
+        dataset_name: Optional[str] = None,
+        sub_dataset_name: Optional[str] = None,
+    ):
+        """Check the validity of data info.
+
+        Args:
+            data_info (Dict[str, str]): The loaded data info to be check.
+            model_name (str): Name of model used.
+            reranker_name (str): Name of reranker used.
+            split (str): Split used in searching.
+            dataset_name (Optional[str], optional): Name of dataset used. Defaults to None.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+
+        Raises:
+            ValueError: eval_name mismatch
+            ValueError: model_name or reranker_name mismatch
+            ValueError: split mismatch
+            ValueError: dataset_name mismatch
+            ValueError: sub_dataset_name mismatch
+        """
+        if data_info["eval_name"] != self.eval_name:
+            raise ValueError(
+                f'eval_name mismatch: {data_info["eval_name"]} vs {self.eval_name}'
+            )
+        if (
+            data_info["model_name"] != model_name
+            or data_info["reranker_name"] != reranker_name
+        ):
+            raise ValueError(
+                f'model_name or reranker_name mismatch: {data_info["model_name"]} vs {model_name} or {data_info["reranker_name"]} vs {reranker_name}'
+            )
+        if (data_info["split"] != split):
+            raise ValueError(
+                f'split mismatch: {data_info["split"]} vs {split}'
+            )
+        if dataset_name is not None and data_info["dataset_name"] != dataset_name:
+            raise ValueError(
+                f'dataset_name mismatch: {data_info["dataset_name"]} vs {dataset_name}'
+            )
+        if sub_dataset_name is not None and data_info["sub_dataset_name"] != sub_dataset_name:
+            raise ValueError(
+                f'sub_dataset_name mismatch: {data_info["sub_dataset_name"]} vs {sub_dataset_name}'
+            )
+
+    def __call__(
+        self,
+        splits: Union[str, List[str]],
+        search_results_save_dir: str,
+        retriever: EvalRetriever,
+        reranker: Optional[EvalReranker] = None,
+        corpus_embd_save_dir: Optional[str] = None,
+        ignore_identical_ids: bool = False,
+        k_values: List[int] = [1, 3, 5, 10, 100, 1000],
+        dataset_name: Optional[str] = None,
+        **kwargs,
+    ):
+        sub_dataset_name = None
+        sub_dataset_names = self.data_loader.available_sub_dataset_names(dataset_name=dataset_name)
+        # Check Splits
+        checked_splits = self.data_loader.check_splits(splits, dataset_name=dataset_name)
+        if len(checked_splits) == 0:
+            logger.warning(f"{splits} not found in the dataset. Skipping evaluation.")
+            return
+        splits = checked_splits
+
+        if sub_dataset_names is None:
+            if dataset_name is not None:
+                save_name = f"{dataset_name}-" + "{split}.json"
+                if corpus_embd_save_dir is not None:
+                    corpus_embd_save_dir = os.path.join(corpus_embd_save_dir, str(retriever), dataset_name)
+            else:
+                save_name = "{split}.json"
+
+            # Retrieval Stage
+            no_reranker_search_results_save_dir = os.path.join(
+                search_results_save_dir, str(retriever), "NoReranker"
+            )
+            os.makedirs(no_reranker_search_results_save_dir, exist_ok=True)
+
+            flag = False
+            for split in splits:
+                split_no_reranker_search_results_save_path = os.path.join(
+                    no_reranker_search_results_save_dir, save_name.format(split=split)
+                )
+                if not os.path.exists(split_no_reranker_search_results_save_path) or self.overwrite:
+                    flag = True
+                    break
+
+            no_reranker_search_results_dict = {}
+            if flag:
+                corpus = self.data_loader.load_corpus(dataset_name=dataset_name)
+
+                queries_dict = {
+                    split: self.data_loader.load_queries(dataset_name=dataset_name, split=split)
+                    for split in splits
+                }
+
+                all_queries = {}
+                for _, split_queries in queries_dict.items():
+                    all_queries.update(split_queries)
+
+                all_no_reranker_search_results = retriever(
+                    corpus=corpus,
+                    queries=all_queries,
+                    corpus_embd_save_dir=corpus_embd_save_dir,
+                    ignore_identical_ids=ignore_identical_ids,
+                    **kwargs,
+                )
+
+                for split in splits:
+                    split_queries = queries_dict[split]
+                    no_reranker_search_results_dict[split] = {
+                        qid: all_no_reranker_search_results[qid] for qid in split_queries
+                    }
+                    split_no_reranker_search_results_save_path = os.path.join(
+                        no_reranker_search_results_save_dir, save_name.format(split=split)
+                    )
+                    self.save_search_results(
+                        eval_name=self.eval_name,
+                        model_name=str(retriever),
+                        reranker_name="NoReranker",
+                        search_results=no_reranker_search_results_dict[split],
+                        output_path=split_no_reranker_search_results_save_path,
+                        split=split,
+                        dataset_name=dataset_name,
+                        sub_dataset_name=sub_dataset_name,
+                    )
+            else:
+                for split in splits:
+                    split_no_reranker_search_results_save_path = os.path.join(
+                        no_reranker_search_results_save_dir, save_name.format(split=split)
+                    )
+                    data_info, search_results = self.load_search_results(split_no_reranker_search_results_save_path)
+                    
+                    self.check_data_info(
+                        data_info=data_info,
+                        model_name=str(retriever),
+                        reranker_name="NoReranker",
+                        split=split,
+                        dataset_name=dataset_name,
+                        sub_dataset_name=sub_dataset_name,
+                    )
+                    no_reranker_search_results_dict[split] = search_results
+            retriever.stop_multi_process_pool()
+            eval_results_save_path = os.path.join(no_reranker_search_results_save_dir, 'EVAL', 'eval_results.json')
+            if not os.path.exists(eval_results_save_path) or self.overwrite or flag:
+                retriever_eval_results = self.evaluate_results(no_reranker_search_results_save_dir, k_values=k_values)
+                self.output_eval_results_to_json(retriever_eval_results, eval_results_save_path)
+
+            # Reranking Stage
+            if reranker is not None:
+                reranker_search_results_save_dir = os.path.join(
+                    search_results_save_dir, str(retriever), str(reranker)
+                )
+                os.makedirs(reranker_search_results_save_dir, exist_ok=True)
+
+                corpus = self.data_loader.load_corpus(dataset_name=dataset_name)
+
+                queries_dict = {
+                    split: self.data_loader.load_queries(dataset_name=dataset_name, split=split)
+                    for split in splits
+                }
+
+                flag = False
+                for split in splits:
+                    rerank_search_results_save_path = os.path.join(
+                        reranker_search_results_save_dir, save_name.format(split=split)
+                    )
+
+                    if os.path.exists(rerank_search_results_save_path) and not self.overwrite:
+                        continue
+
+                    flag = True
+                    rerank_search_results = reranker(
+                        corpus=corpus,
+                        queries=queries_dict[split],
+                        search_results=no_reranker_search_results_dict[split],
+                        ignore_identical_ids=ignore_identical_ids,
+                        **kwargs,
+                    )
+
+                    self.save_search_results(
+                        eval_name=self.eval_name,
+                        model_name=str(retriever),
+                        reranker_name=str(reranker),
+                        search_results=rerank_search_results,
+                        output_path=rerank_search_results_save_path,
+                        split=split,
+                        dataset_name=dataset_name,
+                        sub_dataset_name=sub_dataset_name,
+                    )
+                eval_results_save_path = os.path.join(reranker_search_results_save_dir, 'EVAL', 'eval_results.json')
+                if not os.path.exists(eval_results_save_path) or self.overwrite or flag:
+                    reranker_eval_results = self.evaluate_results(reranker_search_results_save_dir, k_values=k_values)
+                    self.output_eval_results_to_json(reranker_eval_results, eval_results_save_path)
+        else:
+            for sub_dataset_name in sub_dataset_names:
+                if dataset_name is not None:
+                    save_name = f"{dataset_name}-{sub_dataset_name}-" + "{split}.json"
+                    if corpus_embd_save_dir is not None:
+                        corpus_embd_save_dir = os.path.join(corpus_embd_save_dir, str(retriever), dataset_name, sub_dataset_name)
+                else:
+                    save_name = f"{sub_dataset_name}-" + "{split}.json"
+
+                # Retrieval Stage
+                no_reranker_search_results_save_dir = os.path.join(
+                    search_results_save_dir, str(retriever), "NoReranker"
+                )
+                os.makedirs(no_reranker_search_results_save_dir, exist_ok=True)
+
+                flag = False
+                for split in splits:
+                    split_no_reranker_search_results_save_path = os.path.join(
+                        no_reranker_search_results_save_dir, save_name.format(split=split)
+                    )
+                    if not os.path.exists(split_no_reranker_search_results_save_path) or self.overwrite:
+                        flag = True
+                        break
+
+                no_reranker_search_results_dict = {}
+                if flag:
+                    corpus = self.data_loader.load_corpus(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name)
+
+                    queries_dict = {
+                        split: self.data_loader.load_queries(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+                        for split in splits
+                    }
+
+                    all_queries = {}
+                    for _, split_queries in queries_dict.items():
+                        all_queries.update(split_queries)
+
+                    all_no_reranker_search_results = retriever(
+                        corpus=corpus,
+                        queries=all_queries,
+                        corpus_embd_save_dir=corpus_embd_save_dir,
+                        ignore_identical_ids=ignore_identical_ids,
+                        **kwargs,
+                    )
+
+                    for split in splits:
+                        split_queries = queries_dict[split]
+                        no_reranker_search_results_dict[split] = {
+                            qid: all_no_reranker_search_results[qid] for qid in split_queries
+                        }
+                        split_no_reranker_search_results_save_path = os.path.join(
+                            no_reranker_search_results_save_dir, save_name.format(split=split)
+                        )
+
+                        self.save_search_results(
+                            eval_name=self.eval_name,
+                            model_name=str(retriever),
+                            reranker_name="NoReranker",
+                            search_results=no_reranker_search_results_dict[split],
+                            output_path=split_no_reranker_search_results_save_path,
+                            split=split,
+                            dataset_name=dataset_name,
+                            sub_dataset_name=sub_dataset_name,
+                        )
+                else:
+                    for split in splits:
+                        split_no_reranker_search_results_save_path = os.path.join(
+                            no_reranker_search_results_save_dir, save_name.format(split=split)
+                        )
+                        data_info, search_results = self.load_search_results(split_no_reranker_search_results_save_path)
+                        
+                        self.check_data_info(
+                            data_info=data_info,
+                            model_name=str(retriever),
+                            reranker_name="NoReranker",
+                            split=split,
+                            dataset_name=dataset_name,
+                            sub_dataset_name=sub_dataset_name,
+                        )
+                        no_reranker_search_results_dict[split] = search_results
+                eval_results_save_path = os.path.join(no_reranker_search_results_save_dir, 'EVAL', 'eval_results.json')
+                if not os.path.exists(eval_results_save_path) or self.overwrite or flag:
+                    retriever_eval_results = self.evaluate_results(no_reranker_search_results_save_dir, k_values=k_values)
+                    self.output_eval_results_to_json(retriever_eval_results, eval_results_save_path)
+
+                # Reranking Stage
+                if reranker is not None:
+                    reranker_search_results_save_dir = os.path.join(
+                        search_results_save_dir, str(retriever), str(reranker)
+                    )
+                    os.makedirs(reranker_search_results_save_dir, exist_ok=True)
+
+                    corpus = self.data_loader.load_corpus(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name)
+
+                    queries_dict = {
+                        split: self.data_loader.load_queries(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+                        for split in splits
+                    }
+
+                    flag = False
+                    for split in splits:
+                        rerank_search_results_save_path = os.path.join(
+                            reranker_search_results_save_dir, save_name.format(split=split)
+                        )
+
+                        if os.path.exists(rerank_search_results_save_path) and not self.overwrite:
+                            continue
+
+                        flag = True
+                        rerank_search_results = reranker(
+                            corpus=corpus,
+                            queries=queries_dict[split],
+                            search_results=no_reranker_search_results_dict[split],
+                            ignore_identical_ids=ignore_identical_ids,
+                            **kwargs,
+                        )
+
+                        self.save_search_results(
+                            eval_name=self.eval_name,
+                            model_name=str(retriever),
+                            reranker_name=str(reranker),
+                            search_results=rerank_search_results,
+                            output_path=rerank_search_results_save_path,
+                            split=split,
+                            dataset_name=dataset_name,
+                            sub_dataset_name=sub_dataset_name,
+                        )
+                    eval_results_save_path = os.path.join(reranker_search_results_save_dir, 'EVAL', 'eval_results.json')
+                    if not os.path.exists(eval_results_save_path) or self.overwrite or flag:
+                        reranker_eval_results = self.evaluate_results(reranker_search_results_save_dir, k_values=k_values)
+                        self.output_eval_results_to_json(reranker_eval_results, eval_results_save_path)
+            if reranker is not None:
+                reranker.stop_multi_process_pool()
+                
+    def evaluate_results(
+        self,
+        search_results_save_dir: str,
+        k_values: List[int] = [1, 3, 5, 10, 100, 1000]
+    ):
+        """Compute metrics according to the results in the directory.
+
+        Args:
+            search_results_save_dir (str): Path to the search results.
+            k_values (List[int], optional): Cutoffs. Defaults to :data:`[1, 3, 5, 10, 100, 1000]`.
+
+        Returns:
+            dict: Evaluation results.
+        """
+        eval_results_dict = {}
+        cqadupstack_results = None
+        cqadupstack_num = 0
+
+        for file in os.listdir(search_results_save_dir):
+            if not file.endswith('.json'):
+                continue
+
+            file_path = os.path.join(search_results_save_dir, file)
+            data_info, search_results = self.load_search_results(file_path)
+
+            _eval_name = data_info['eval_name']
+            assert _eval_name == self.eval_name, f'Mismatch eval_name: {_eval_name} vs {self.eval_name} in {file_path}'
+
+            split = data_info['split']
+            dataset_name = data_info.get('dataset_name', None)
+            sub_dataset_name = data_info.get('sub_dataset_name', None)
+            qrels = self.data_loader.load_qrels(dataset_name=dataset_name, sub_dataset_name=sub_dataset_name, split=split)
+
+            eval_results = self.compute_metrics(
+                qrels=qrels,
+                search_results=search_results,
+                k_values=k_values
+            )
+
+            if dataset_name is not None:
+                if sub_dataset_name is None:
+                    key = f"{dataset_name}-{split}"
+                else:
+                    key = f"{dataset_name}-{sub_dataset_name}-{split}"
+            else:
+                if sub_dataset_name is None:
+                    key = split
+                else:
+                    key = f"{sub_dataset_name}-{split}"
+            if sub_dataset_name is None:
+                eval_results_dict[key] = eval_results
+            else:
+                if cqadupstack_results is None:
+                    cqadupstack_results = eval_results
+                    cqadupstack_num += 1
+                else:
+                    for k, v in eval_results.items():
+                        cqadupstack_results[k] += v
+                    cqadupstack_num += 1
+        
+        if cqadupstack_num > 0:
+            for k in cqadupstack_results.keys():
+                cqadupstack_results[k] /= cqadupstack_num
+            eval_results_dict['cqadupstack-test'] = cqadupstack_results
+
+        return eval_results_dict
+    
+    def save_search_results(
+        self,
+        eval_name: str,
+        model_name: str,
+        reranker_name: str,
+        search_results: Dict[str, Dict[str, float]],
+        output_path: str,
+        split: str,
+        dataset_name: Optional[str] = None,
+        sub_dataset_name: Optional[str] = None,
+    ):
+        """Save the metadata and search results into a file.
+
+        Args:
+            eval_name (str): The experiment name of current evaluation.
+            model_name (str): Name of model used.
+            reranker_name (str): Name of reranker used.
+            search_results (Dict[str, Dict[str, float]]): Dictionary of search results.
+            output_path (str): Output path to write the results.
+            split (str): Split used in searching.
+            dataset_name (Optional[str], optional): Name of dataset used. Defaults to ``None``.
+            sub_dataset_name (Optional[str], optional): Name of the sub-dataset. Defaults to ``None``.
+        """
+        data = {
+            "eval_name": eval_name,
+            "model_name": model_name,
+            "reranker_name": reranker_name,
+            "split": split,
+            "dataset_name": dataset_name,
+            "sub_dataset_name": sub_dataset_name,
+            "search_results": search_results,
+        }
+
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=4)

FlagEmbedding/evaluation/beir/prompts.py

@@ -0,0 +1,17 @@
+BEIRInstructions = {
+    'dbpedia-entity': 'Given a query, retrieve relevant entity descriptions from DBPedia.',
+    'arguana': 'Given a claim, find documents that refute the claim.',
+    'climate-fever': 'Given a claim about climate change, retrieve documents that support or refute the claim.',
+    'cqadupstack': 'Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question.',
+    'fever': 'Given a claim, retrieve documents that support or refute the claim.',
+    'fiqa': 'Given a financial question, retrieve user replies that best answer the question.',
+    'hotpotqa': 'Given a multi-hop question, retrieve documents that can help answer the question.',
+    'msmarco': 'Given a web search query, retrieve relevant passages that answer the query.',
+    'nfcorpus': 'Given a question, retrieve relevant documents that best answer the question.',
+    'nq': 'Given a question, retrieve Wikipedia passages that answer the question.',
+    'quora': 'Given a question, retrieve questions that are semantically equivalent to the given question.',
+    'scidocs': 'Given a scientific paper title, retrieve paper abstracts that are cited by the given paper.',
+    'scifact': 'Given a scientific claim, retrieve documents that support or refute the claim.',
+    'webis-touche2020': 'Given a question, retrieve detailed and persuasive arguments that answer the question.',
+    'trec-covid': 'Given a query on COVID-19, retrieve documents that answer the query.',
+}

FlagEmbedding/evaluation/beir/runner.py

@@ -0,0 +1,89 @@
+import logging
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import BEIREvalDataLoader
+from .prompts import BEIRInstructions
+from .evaluator import BEIREvaluator
+
+logger = logging.getLogger(__name__)
+
+
+class BEIREvalRunner(AbsEvalRunner):
+    """
+    Runner class of BEIR evaluation.
+    """
+    def run(self):
+        """
+        Run the whole evaluation.
+        """
+        if self.eval_args.dataset_names is None:
+            dataset_names = self.data_loader.available_dataset_names()
+        else:
+            dataset_names = self.data_loader.check_dataset_names(self.eval_args.dataset_names)
+
+        if len(dataset_names) == 0:
+            logger.info(f"Running {self.eval_args.eval_name} evaluation on the default dataset.")
+            self.evaluator(
+                splits=self.eval_args.splits,
+                search_results_save_dir=self.eval_args.output_dir,
+                retriever=self.retriever,
+                reranker=self.reranker,
+                corpus_embd_save_dir=self.eval_args.corpus_embd_save_dir,
+                ignore_identical_ids=self.eval_args.ignore_identical_ids,
+                k_values=self.eval_args.k_values
+            )
+            logger.info(f"{self.eval_args.eval_name} evaluation completed.")
+        else:
+            logger.info(f"Running {self.eval_args.eval_name} evaluation on the following dataset names: {dataset_names}")
+            for dataset_name in dataset_names:
+                if self.eval_args.use_special_instructions:
+                    self.retriever.stop_multi_process_pool()
+                    self.retriever.embedder.query_instruction_for_retrieval = BEIRInstructions[dataset_name]
+                logger.info(f"Running {self.eval_args.eval_name} evaluation on: {dataset_name}")
+                self.evaluator(
+                    splits=self.eval_args.splits,
+                    search_results_save_dir=self.eval_args.output_dir,
+                    retriever=self.retriever,
+                    reranker=self.reranker,
+                    corpus_embd_save_dir=self.eval_args.corpus_embd_save_dir,
+                    ignore_identical_ids=self.eval_args.ignore_identical_ids,
+                    k_values=self.eval_args.k_values,
+                    dataset_name=dataset_name,
+                )
+            logger.info(f"{self.eval_args.eval_name} evaluation on {dataset_names} completed.")
+
+        logger.info("Start computing metrics.")
+        self.evaluate_metrics(
+            search_results_save_dir=self.eval_args.output_dir,
+            output_method=self.eval_args.eval_output_method,
+            output_path=self.eval_args.eval_output_path,
+            metrics=self.eval_args.eval_metrics
+        )
+        
+    def load_data_loader(self) -> BEIREvalDataLoader:
+        """Load the data loader
+
+        Returns:
+            BEIREvalDataLoader: BEIR data loader object.
+        """
+        data_loader = BEIREvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader
+
+    def load_evaluator(self) -> BEIREvaluator:
+        """Load the evaluator for evaluation
+
+        Returns:
+            BEIREvaluator: The BEIR evaluator to run the evaluation.
+        """
+        evaluator = BEIREvaluator(
+            eval_name=self.eval_args.eval_name,
+            data_loader=self.data_loader,
+            overwrite=self.eval_args.overwrite,
+        )
+        return evaluator

FlagEmbedding/evaluation/bright/__init__.py

@@ -0,0 +1,17 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalModelArgs as BrightEvalModelArgs,
+)
+
+from .data_loader import BrightShortEvalDataLoader, BrightLongEvalDataLoader
+from .arguments import BrightEvalArgs
+from .runner import BrightEvalRunner
+from .searcher import BrightEvalDenseRetriever
+
+__all__ = [
+    "BrightEvalArgs",
+    "BrightEvalModelArgs",
+    "BrightEvalRunner",
+    "BrightEvalDenseRetriever",
+    "BrightShortEvalDataLoader",
+    "BrightLongEvalDataLoader",
+]

FlagEmbedding/evaluation/bright/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.bright import (
+    BrightEvalArgs, BrightEvalModelArgs,
+    BrightEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        BrightEvalArgs,
+        BrightEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: BrightEvalArgs
+    model_args: BrightEvalModelArgs
+
+    runner = BrightEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/bright/arguments.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass, field
+
+from FlagEmbedding.abc.evaluation.arguments import AbsEvalArgs
+
+
+@dataclass
+class BrightEvalArgs(AbsEvalArgs):
+    """
+    Argument class for Bright evaluation.
+    """
+    task_type: str = field(
+        default="short", metadata={"help": "The task type to evaluate on. Available options: ['short', 'long']. Default: short", "choices": ["short", "long"]}
+    )
+    use_special_instructions: bool = field(
+        default=True, metadata={"help": "Whether to use specific instructions in `prompts.py` for evaluation. Default: True"}
+    )

FlagEmbedding/evaluation/bright/data_loader.py

@@ -0,0 +1,399 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+from collections import defaultdict
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class BrightShortEvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for Bright(short).
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return [
+            # StackExchange
+            "biology", "earth_science", "economics", "psychology", "robotics", "stackoverflow", "sustainable_living",
+            # Coding
+            "leetcode", "pony",
+            # Theorem-based
+            "aops", "theoremqa_questions", "theoremqa_theorems"
+        ]
+
+    def available_splits(self, dataset_name: str) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (str): Dataset name.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        return [
+            # normal splits
+            "examples",
+            # w/ reasoning splits
+            "Gemini-1.0_reason", "claude-3-opus_reason", "gpt4_reason", "grit_reason", "llama3-70b_reason",
+        ]
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        corpus = datasets.load_dataset(
+            "xlangai/bright", "documents",
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    docid, text = str(data["id"]), data["content"]
+                    _data = {
+                        "id": docid,
+                        "text": text
+                    }
+                    corpus_dict[docid] = {"text": text}
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+        else:
+            corpus_dict = {str(data["id"]): {"text": data["content"]} for data in tqdm(corpus, desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: str,
+        split: str = 'examples',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'examples'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        examples = datasets.load_dataset(
+            "xlangai/bright", split,
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = defaultdict(dict)
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(examples, desc="Loading and Saving qrels"):
+
+                    # NOTE: we modify the qid here to distinguish the queries from different splits
+                    qid = f'{split}-{data["id"]}'
+
+                    for docid in data["gold_ids"]:
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": 1
+                        }
+                        qrels_dict[qid][docid] = 1
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+
+                    # NOTE: we record the excluded_ids in qrels with relevance 0 to remove corresponding documents from raw search results. Refer to `searcher.py` for details.
+                    for ex_docid in list(set(data["excluded_ids"])):
+                        if ex_docid == "N/A":
+                            continue
+                        assert ex_docid not in qrels_dict[qid], f"{ex_docid} in {qid}"
+                        _data = {
+                            "qid": qid,
+                            "docid": ex_docid,
+                            "relevance": 0
+                        }
+                        qrels_dict[qid][ex_docid] = 0
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+        else:
+            qrels_dict = defaultdict(dict)
+            for data in tqdm(examples, desc="Loading qrels"):
+
+                # NOTE: we modify the qid here to distinguish the queries from different splits
+                qid = f'{split}-{data["id"]}'
+
+                for docid in data["gold_ids"]:
+                    qrels_dict[qid][docid] = 1
+
+                # NOTE: we record the excluded_ids in qrels with relevance 0 to remove corresponding documents from raw search results. Refer to `searcher.py` for details.
+                for ex_docid in data["excluded_ids"]:
+                    if ex_docid == "N/A":
+                        continue
+                    assert ex_docid not in qrels_dict[qid], f"{ex_docid} in {qid}"
+                    _data = {
+                        "qid": qid,
+                        "docid": ex_docid,
+                        "relevance": 0
+                    }
+                    qrels_dict[qid][ex_docid] = 0
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: str,
+        split: str = 'examples',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'examples'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        examples = datasets.load_dataset(
+            "xlangai/bright", split,
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(examples, desc="Loading and Saving queries"):
+
+                    # NOTE: we modify the qid here to distinguish the queries from different splits
+                    qid, query = f'{split}-{data["id"]}', data["query"]
+
+                    _data = {
+                        "id": qid,
+                        "text": query
+                    }
+                    queries_dict[qid] = query
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+        else:
+            # NOTE: we modify the qid here to distinguish the queries from different splits
+            queries_dict = {f'{split}-{data["id"]}': data["query"] for data in tqdm(examples, desc="Loading queries")}
+        return datasets.DatasetDict(queries_dict)
+
+
+class BrightLongEvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for Bright(long).
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return [
+            # StackExchange
+            "biology", "earth_science", "economics", "psychology", "robotics", "stackoverflow", "sustainable_living",
+            # Coding
+            "pony",
+        ]
+
+    def available_splits(self, dataset_name: str) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (str): Dataset name.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        return [
+            # normal splits
+            "examples",
+            # w/ reasoning splits
+            "Gemini-1.0_reason", "claude-3-opus_reason", "gpt4_reason", "grit_reason", "llama3-70b_reason",
+        ]
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        corpus = datasets.load_dataset(
+            "xlangai/bright", "long_documents",
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    docid, text = str(data["id"]), data["content"]
+                    _data = {
+                        "id": docid,
+                        "text": text
+                    }
+                    corpus_dict[docid] = {"text": text}
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+        else:
+            corpus_dict = {str(data["id"]): {"text": data["content"]} for data in tqdm(corpus, desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: str,
+        split: str = 'examples',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'examples'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        examples = datasets.load_dataset(
+            "xlangai/bright", split,
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = defaultdict(dict)
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(examples, desc="Loading and Saving qrels"):
+
+                    # NOTE: we modify the qid here to distinguish the queries from different splits
+                    qid = f'{split}-{data["id"]}'
+
+                    for docid in data["gold_ids_long"]:
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": 1
+                        }
+                        qrels_dict[qid][docid] = 1
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+
+                    # NOTE: we record the excluded_ids in qrels with relevance 0 to remove corresponding documents from raw search results. Refer to `searcher.py` for details.
+                    for ex_docid in list(set(data["excluded_ids"])):
+                        if ex_docid == "N/A":
+                            continue
+                        assert ex_docid not in qrels_dict[qid], f"{ex_docid} in {qid}"
+                        _data = {
+                            "qid": qid,
+                            "docid": ex_docid,
+                            "relevance": 0
+                        }
+                        qrels_dict[qid][ex_docid] = 0
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+        else:
+            qrels_dict = defaultdict(dict)
+            for data in tqdm(examples, desc="Loading qrels"):
+
+                # NOTE: we modify the qid here to distinguish the queries from different splits
+                qid = f'{split}-{data["id"]}'
+
+                for docid in data["gold_ids_long"]:
+                    qrels_dict[qid][docid] = 1
+
+                # NOTE: we record the excluded_ids in qrels with relevance 0 to remove corresponding documents from raw search results. Refer to `searcher.py` for details.
+                for ex_docid in data["excluded_ids"]:
+                    if ex_docid == "N/A":
+                        continue
+                    assert ex_docid not in qrels_dict[qid], f"{ex_docid} in {qid}"
+                    _data = {
+                        "qid": qid,
+                        "docid": ex_docid,
+                        "relevance": 0
+                    }
+                    qrels_dict[qid][ex_docid] = 0
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: str,
+        split: str = 'examples',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'examples'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        examples = datasets.load_dataset(
+            "xlangai/bright", split,
+            cache_dir=self.cache_dir,
+            download_mode=self.hf_download_mode
+        )[dataset_name]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(examples, desc="Loading and Saving queries"):
+
+                    # NOTE: we modify the qid here to distinguish the queries from different splits
+                    qid, query = f'{split}-{data["id"]}', data["query"]
+
+                    _data = {
+                        "id": qid,
+                        "text": query
+                    }
+                    queries_dict[qid] = query
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+        else:
+            # NOTE: we modify the qid here to distinguish the queries from different splits
+            queries_dict = {f'{split}-{data["id"]}': data["query"] for data in tqdm(examples, desc="Loading queries")}
+        return datasets.DatasetDict(queries_dict)

FlagEmbedding/evaluation/bright/prompts.py

@@ -0,0 +1,31 @@
+BrightShortInstructions = {
+    # StackExchange
+    "biology": "Given a Biology post, retrieve relevant passages that help answer the post.",
+    "earth_science": "Given an Earth Science post, retrieve relevant passages that help answer the post.",
+    "economics": "Given an Economics post, retrieve relevant passages that help answer the post.",
+    "psychology": "Given a Psychology post, retrieve relevant passages that help answer the post.",
+    "robotics": "Given a Robotics post, retrieve relevant passages that help answer the post.",
+    "stackoverflow": "Given a Stack Overflow post, retrieve relevant passages that help answer the post.",
+    "sustainable_living": "Given a Sustainable Living post, retrieve relevant passages that help answer the post.",
+    # Coding
+    "leetcode": "Given a Coding problem, retrieve relevant examples that help answer the problem.",
+    "pony": "Given a Pony question, retrieve relevant passages that help answer the question.",
+    # Theorem-based
+    "aops": "Given a Math problem, retrieve relevant examples that help answer the problem.",
+    "theoremqa_questions": "Given a Math problem, retrieve relevant examples that help answer the problem.",
+    "theoremqa_theorems": "Given a Math problem, retrieve relevant theorems that help answer the problem.",
+}
+
+
+BrightLongInstructions = {
+    # StackExchange
+    "biology": "Given a Biology post, retrieve relevant documents that help answer the post.",
+    "earth_science": "Given an Earth Science post, retrieve relevant documents that help answer the post.",
+    "economics": "Given an Economics post, retrieve relevant documents that help answer the post.",
+    "psychology": "Given a Psychology post, retrieve relevant documents that help answer the post.",
+    "robotics": "Given a Robotics post, retrieve relevant documents that help answer the post.",
+    "stackoverflow": "Given a Stack Overflow post, retrieve relevant documents that help answer the post.",
+    "sustainable_living": "Given a Sustainable Living post, retrieve relevant documents that help answer the post.",
+    # Coding
+    "pony": "Given a Pony question, retrieve relevant documents that help answer the question",
+}

FlagEmbedding/evaluation/bright/runner.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Union, Tuple
+from FlagEmbedding.abc.evaluation import AbsEvalRunner, EvalReranker, \
+    AbsEvalModelArgs as BrightEvalModelArgs
+
+from .prompts import BrightShortInstructions, BrightLongInstructions
+from .arguments import BrightEvalArgs
+from .data_loader import BrightShortEvalDataLoader, BrightLongEvalDataLoader
+from .searcher import BrightEvalDenseRetriever
+
+logger = logging.getLogger(__name__)
+
+
+class BrightEvalRunner(AbsEvalRunner):
+    """
+    Evaluation runner of Bright.
+    """
+    def __init__(self, eval_args: BrightEvalArgs, model_args: BrightEvalModelArgs):
+        super().__init__(eval_args, model_args)
+        self.eval_args: BrightEvalArgs
+        self.model_args: BrightEvalModelArgs
+
+    def load_data_loader(self) -> Union[BrightShortEvalDataLoader, BrightLongEvalDataLoader]:
+        """Load the data loader instance by args.
+
+        Returns:
+            Union[BrightShortEvalDataLoader, BrightLongEvalDataLoader]: The Bright data loader instance.
+        """
+        if self.eval_args.task_type == "short":
+            data_loader_class = BrightShortEvalDataLoader
+        elif self.eval_args.task_type == "long":
+            data_loader_class = BrightLongEvalDataLoader
+        else:
+            raise ValueError(f"Invalid task type: {self.eval_args.task_type}")
+
+        data_loader = data_loader_class(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader
+
+    def load_retriever_and_reranker(self) -> Tuple[BrightEvalDenseRetriever, Union[EvalReranker, None]]:
+        """Load retriever and reranker for evaluation
+
+        Returns:
+            Tuple[BrightEvalDenseRetriever, Union[EvalReranker, None]]: A :class:BrightEvalDenseRetriever object for retrieval, and a
+                :class:EvalReranker object if reranker provided.
+        """
+        embedder, reranker = self.get_models(self.model_args)
+        retriever = BrightEvalDenseRetriever(
+            embedder,
+            search_top_k=self.eval_args.search_top_k,
+            overwrite=self.eval_args.overwrite
+        )
+        if reranker is not None:
+            reranker = EvalReranker(reranker, rerank_top_k=self.eval_args.rerank_top_k)
+        return retriever, reranker
+
+    def run(self):
+        """
+        Run the whole evaluation.
+        """
+        if self.eval_args.dataset_names is None:
+            dataset_names = self.data_loader.available_dataset_names()
+        else:
+            dataset_names = self.data_loader.check_dataset_names(self.eval_args.dataset_names)
+
+        if len(dataset_names) == 0:
+            logger.info(f"Running {self.eval_args.eval_name} evaluation on the default dataset.")
+            self.evaluator(
+                splits=self.eval_args.splits,
+                search_results_save_dir=self.eval_args.output_dir,
+                retriever=self.retriever,
+                reranker=self.reranker,
+                corpus_embd_save_dir=self.eval_args.corpus_embd_save_dir,
+                ignore_identical_ids=self.eval_args.ignore_identical_ids,
+                k_values=self.eval_args.k_values
+            )
+            logger.info(f"{self.eval_args.eval_name} evaluation completed.")
+        else:
+            logger.info(f"Running {self.eval_args.eval_name} evaluation on the following dataset names: {dataset_names}")
+            for dataset_name in dataset_names:
+                if self.eval_args.use_special_instructions:
+                    self.retriever.stop_multi_process_pool()
+                    if self.eval_args.task_type == "short":
+                        self.retriever.embedder.query_instruction_for_retrieval = BrightShortInstructions[dataset_name]
+                    elif self.eval_args.task_type == "long":
+                        self.retriever.embedder.query_instruction_for_retrieval = BrightLongInstructions[dataset_name]
+                    else:
+                        raise ValueError(f"Invalid task type: {self.eval_args.task_type}")
+
+                # NOTE: pass qrels to searcher to exclude documents from raw search results
+                evaluator_kwargs = {}
+                evaluator_kwargs["retriever_qrels"] = self.data_loader.load_qrels(dataset_name=dataset_name, split=self.eval_args.splits)
+
+                logger.info(f"Running {self.eval_args.eval_name} evaluation on: {dataset_name}")
+                self.evaluator(
+                    splits=self.eval_args.splits,
+                    search_results_save_dir=self.eval_args.output_dir,
+                    retriever=self.retriever,
+                    reranker=self.reranker,
+                    corpus_embd_save_dir=self.eval_args.corpus_embd_save_dir,
+                    ignore_identical_ids=self.eval_args.ignore_identical_ids,
+                    k_values=self.eval_args.k_values,
+                    dataset_name=dataset_name,
+                    **evaluator_kwargs,
+                )
+            logger.info(f"{self.eval_args.eval_name} evaluation on {dataset_names} completed.")
+
+        logger.info("Start computing metrics.")
+        self.evaluate_metrics(
+            search_results_save_dir=self.eval_args.output_dir,
+            output_method=self.eval_args.eval_output_method,
+            output_path=self.eval_args.eval_output_path,
+            metrics=self.eval_args.eval_metrics
+        )

FlagEmbedding/evaluation/bright/searcher.py

@@ -0,0 +1,127 @@
+import os
+import logging
+import gc
+import torch
+import numpy as np
+from typing import Any, Dict, Optional
+
+from FlagEmbedding.abc.evaluation.utils import index, search
+
+from FlagEmbedding.abc.evaluation import EvalRetriever
+
+logger = logging.getLogger(__name__)
+
+
+class BrightEvalDenseRetriever(EvalRetriever):
+    """
+    Child class of :class:EvalRetriever for dense retrieval.
+    """
+    def __call__(
+        self,
+        corpus: Dict[str, Dict[str, Any]],
+        queries: Dict[str, str],
+        corpus_embd_save_dir: Optional[str] = None,
+        ignore_identical_ids: bool = False,
+        **kwargs,
+    ) -> Dict[str, Dict[str, float]]:
+        """
+        This is called during the retrieval process.
+
+        Parameters:
+            corpus: Dict[str, Dict[str, Any]]: Corpus of documents. 
+                Structure: {<docid>: {"text": <text>}}.
+                Example: {"doc-0": {"text": "This is a document."}}
+            queries: Dict[str, str]: Queries to search for.
+                Structure: {<qid>: <query>}.
+                Example: {"q-0": "This is a query."}
+            corpus_embd_save_dir (Optional[str]): Defaults to :data:`None`.
+            ignore_identical_ids (bool): Defaults to :data:`False`.
+            **kwargs: Any: Additional arguments.
+
+        Returns: Dict[str, Dict[str, float]]: Top-k search results for each query. k is specified by search_top_k.
+            Structure: {qid: {docid: score}}. The higher is the score, the more relevant is the document.
+            Example: {"q-0": {"doc-0": 0.9}}
+        """
+        if ignore_identical_ids:
+            logger.warning("ignore_identical_ids is set to True. This means that the search results will not contain identical ids. Note: Dataset such as MIRACL should NOT set this to True.")
+
+        # dense embedding models do not require language as input: AIRBench evaluation
+        kwargs.pop("language", None)
+
+        corpus_ids = []
+        corpus_texts = []
+        for docid, doc in corpus.items():
+            corpus_ids.append(docid)
+            corpus_texts.append(
+                doc["text"] if "title" not in doc 
+                else f"{doc['title']} {doc['text']}".strip()
+            )
+        queries_ids = []
+        queries_texts = []
+        for qid, query in queries.items():
+            queries_ids.append(qid)
+            queries_texts.append(query)
+
+        # NOTE: obtain excluded ids from qrels to remove corresponding documents from raw search results
+        excluded_ids = {}
+        qrels = kwargs.pop("retriever_qrels", None)
+        if qrels is not None:
+            for qid in qrels:
+                excluded_ids[qid] = []
+                for docid, score in qrels[qid].items():
+                    if score != 1:
+                        excluded_ids[qid].append(docid)
+        else:
+            logger.warning("No qrels provided, so no documents will be excluded.")
+
+        if corpus_embd_save_dir is not None:
+            if os.path.exists(os.path.join(corpus_embd_save_dir, "doc.npy")) and not self.overwrite:
+                corpus_emb = np.load(os.path.join(corpus_embd_save_dir, "doc.npy"))
+            else:
+                corpus_emb = self.embedder.encode_corpus(corpus_texts, **kwargs)
+        else:
+            corpus_emb = self.embedder.encode_corpus(corpus_texts, **kwargs)
+
+        queries_emb = self.embedder.encode_queries(queries_texts, **kwargs)
+
+        # check if the embeddings are in dictionary format: M3Embedder
+        if isinstance(corpus_emb, dict):
+            corpus_emb = corpus_emb["dense_vecs"]
+        if isinstance(queries_emb, dict):
+            queries_emb = queries_emb["dense_vecs"]
+
+        if corpus_embd_save_dir is not None and \
+            (not os.path.exists(os.path.join(corpus_embd_save_dir, "doc.npy")) or self.overwrite):
+            os.makedirs(corpus_embd_save_dir, exist_ok=True)
+            np.save(os.path.join(corpus_embd_save_dir, "doc.npy"), corpus_emb)
+
+        gc.collect()
+        torch.cuda.empty_cache()
+
+        faiss_index = index(corpus_embeddings=corpus_emb)
+        all_scores, all_indices = search(query_embeddings=queries_emb, faiss_index=faiss_index, k=self.search_top_k)
+
+        results = {}
+        for idx, (scores, indices) in enumerate(zip(all_scores, all_indices)):
+            query_id = queries_ids[idx]
+
+            results[query_id] = {}
+            for score, indice in zip(scores, indices):
+                if indice != -1:
+                    if ignore_identical_ids and corpus_ids[indice] == query_id:
+                        continue
+                    results[query_id][corpus_ids[indice]] = float(score)
+
+            if qrels is not None:
+                # NOTE: Filter out documents with ids in excluded_ids
+                for docid in set(excluded_ids[query_id]):
+                    if docid != "N/A":
+                        results[query_id].pop(docid, None)
+
+            sorted_scores = sorted(results[query_id].items(), key=lambda item: item[1], reverse=True)
+            # Store the top-k results for the current query
+            results[query_id] = {}
+            for docid, score in sorted_scores[:self.search_top_k]:
+                results[query_id][docid] = float(score)
+
+        return results

FlagEmbedding/evaluation/custom/__init__.py

@@ -0,0 +1,14 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalArgs as CustomEvalArgs,
+    AbsEvalModelArgs as CustomEvalModelArgs,
+)
+
+from .data_loader import CustomEvalDataLoader
+from .runner import CustomEvalRunner
+
+__all__ = [
+    "CustomEvalArgs",
+    "CustomEvalModelArgs",
+    "CustomEvalRunner",
+    "CustomEvalDataLoader",
+]

FlagEmbedding/evaluation/custom/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.custom import (
+    CustomEvalArgs, CustomEvalModelArgs,
+    CustomEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        CustomEvalArgs,
+        CustomEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: CustomEvalArgs
+    model_args: CustomEvalModelArgs
+
+    runner = CustomEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/custom/data_loader.py

@@ -0,0 +1,15 @@
+import logging
+from tqdm import tqdm
+from typing import List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class CustomEvalDataLoader(AbsEvalDataLoader):
+    def available_dataset_names(self) -> List[str]:
+        return []
+
+    def available_splits(self, dataset_name: Optional[str] = None) -> List[str]:
+        return ["test"]

FlagEmbedding/evaluation/custom/runner.py

@@ -0,0 +1,15 @@
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import CustomEvalDataLoader
+
+
+class CustomEvalRunner(AbsEvalRunner):
+    def load_data_loader(self) -> CustomEvalDataLoader:
+        data_loader = CustomEvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader

FlagEmbedding/evaluation/miracl/__init__.py

@@ -0,0 +1,14 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalArgs as MIRACLEvalArgs,
+    AbsEvalModelArgs as MIRACLEvalModelArgs,
+)
+
+from .data_loader import MIRACLEvalDataLoader
+from .runner import MIRACLEvalRunner
+
+__all__ = [
+    "MIRACLEvalArgs",
+    "MIRACLEvalModelArgs",
+    "MIRACLEvalRunner",
+    "MIRACLEvalDataLoader",
+]

FlagEmbedding/evaluation/miracl/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.miracl import (
+    MIRACLEvalArgs, MIRACLEvalModelArgs,
+    MIRACLEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        MIRACLEvalArgs,
+        MIRACLEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: MIRACLEvalArgs
+    model_args: MIRACLEvalModelArgs
+
+    runner = MIRACLEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/miracl/data_loader.py

@@ -0,0 +1,179 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class MIRACLEvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for MIRACL.
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return ["ar", "bn", "en", "es", "fa", "fi", "fr", "hi", "id", "ja", "ko", "ru", "sw", "te", "th", "zh", "de", "yo"]
+
+    def available_splits(self, dataset_name: str) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (str): Dataset name.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        if dataset_name in ["de", "yo"]:
+            return ["dev"]
+        else:
+            return ["train", "dev"]
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        corpus = datasets.load_dataset(
+            "miracl/miracl-corpus", dataset_name,
+            cache_dir=self.cache_dir,
+            trust_remote_code=True,
+            download_mode=self.hf_download_mode
+        )["train"]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    docid, title, text = str(data["docid"]), data["title"], data["text"]
+                    _data = {
+                        "id": docid,
+                        "title": title,
+                        "text": text
+                    }
+                    corpus_dict[docid] = {
+                        "title": title,
+                        "text": text
+                    }
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+        else:
+            corpus_dict = {str(data["docid"]): {"title": data["title"], "text": data["text"]} for data in tqdm(corpus, desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: str,
+        split: str = 'dev',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'dev'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        endpoint = f"{os.getenv('HF_ENDPOINT', 'https://huggingface.co')}/datasets/miracl/miracl"
+        qrels_download_url = f"{endpoint}/resolve/main/miracl-v1.0-{dataset_name}/qrels/qrels.miracl-v1.0-{dataset_name}-{split}.tsv"
+
+        qrels_save_path = self._download_file(qrels_download_url, self.cache_dir)
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f1:
+                with open(qrels_save_path, "r", encoding="utf-8") as f2:
+                    for line in tqdm(f2.readlines(), desc="Loading and Saving qrels"):
+                        qid, _, docid, rel = line.strip().split("\t")
+                        qid, docid, rel = str(qid), str(docid), int(rel)
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": rel
+                        }
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid][docid] = rel
+                        f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+        else:
+            qrels_dict = {}
+            with open(qrels_save_path, "r", encoding="utf-8") as f:
+                for line in tqdm(f.readlines(), desc="Loading qrels"):
+                    qid, _, docid, rel = line.strip().split("\t")
+                    qid, docid, rel = str(qid), str(docid), int(rel)
+                    if qid not in qrels_dict:
+                        qrels_dict[qid] = {}
+                    qrels_dict[qid][docid] = rel
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: str,
+        split: str = 'dev',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'dev'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        endpoint = f"{os.getenv('HF_ENDPOINT', 'https://huggingface.co')}/datasets/miracl/miracl"
+        queries_download_url = f"{endpoint}/resolve/main/miracl-v1.0-{dataset_name}/topics/topics.miracl-v1.0-{dataset_name}-{split}.tsv"
+
+        queries_save_path = self._download_file(queries_download_url, self.cache_dir)
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f1:
+                with open(queries_save_path, "r", encoding="utf-8") as f2:
+                    for line in tqdm(f2.readlines(), desc="Loading and Saving queries"):
+                        qid, query = line.strip().split("\t")
+                        qid = str(qid)
+                        _data = {
+                            "id": qid,
+                            "text": query
+                        }
+                        queries_dict[qid] = query
+                        f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+        else:
+            queries_dict = {}
+            with open(queries_save_path, "r", encoding="utf-8") as f:
+                for line in tqdm(f.readlines(), desc="Loading queries"):
+                    qid, query = line.strip().split("\t")
+                    qid = str(qid)
+                    queries_dict[qid] = query
+        return datasets.DatasetDict(queries_dict)

FlagEmbedding/evaluation/miracl/runner.py

@@ -0,0 +1,23 @@
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import MIRACLEvalDataLoader
+
+
+class MIRACLEvalRunner(AbsEvalRunner):
+    """
+    Evaluation runner of MIRACL.
+    """
+    def load_data_loader(self) -> MIRACLEvalDataLoader:
+        """Load the data loader instance by args.
+
+        Returns:
+            MIRACLEvalDataLoader: The MIRACL data loader instance.
+        """
+        data_loader = MIRACLEvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader

FlagEmbedding/evaluation/mkqa/__init__.py

@@ -0,0 +1,16 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalArgs as MKQAEvalArgs,
+    AbsEvalModelArgs as MKQAEvalModelArgs,
+)
+
+from .data_loader import MKQAEvalDataLoader
+from .evaluator import MKQAEvaluator
+from .runner import MKQAEvalRunner
+
+__all__ = [
+    "MKQAEvalArgs",
+    "MKQAEvalModelArgs",
+    "MKQAEvalRunner",
+    "MKQAEvalDataLoader",
+    "MKQAEvaluator"
+]

FlagEmbedding/evaluation/mkqa/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.mkqa import (
+    MKQAEvalArgs, MKQAEvalModelArgs,
+    MKQAEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        MKQAEvalArgs,
+        MKQAEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: MKQAEvalArgs
+    model_args: MKQAEvalModelArgs
+
+    runner = MKQAEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/mkqa/data_loader.py

@@ -0,0 +1,228 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+from .utils.normalize_text import normalize_text
+
+logger = logging.getLogger(__name__)
+
+
+class MKQAEvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for MKQA.
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return ['en', 'ar', 'fi', 'ja', 'ko', 'ru', 'es', 'sv', 'he', 'th', 'da', 'de', 'fr', 'it', 'nl', 'pl', 'pt', 'hu', 'vi', 'ms', 'km', 'no', 'tr', 'zh_cn', 'zh_hk', 'zh_tw']
+
+    def available_splits(self, dataset_name: Optional[str] = None) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (str): Dataset name.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        return ["test"]
+
+    def load_corpus(self, dataset_name: Optional[str] = None) -> datasets.DatasetDict:
+        """Load the corpus.
+
+        Args:
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to None.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        if self.dataset_dir is not None:
+            # same corpus for all languages
+            save_dir = self.dataset_dir
+            return self._load_local_corpus(save_dir, dataset_name=dataset_name)
+        else:
+            return self._load_remote_corpus(dataset_name=dataset_name)
+
+    def _load_local_qrels(self, save_dir: str, dataset_name: Optional[str] = None, split: str = 'test') -> datasets.DatasetDict:
+        """Try to load qrels from local datasets.
+
+        Args:
+            save_dir (str): Directory that save the data files.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+
+        Raises:
+            ValueError: No local qrels found, will try to download from remote.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrels.
+        """
+        checked_split = self.check_splits(split)
+        if len(checked_split) == 0:
+            raise ValueError(f"Split {split} not found in the dataset.")
+        split = checked_split[0]
+
+        qrels_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+        if self.force_redownload or not os.path.exists(qrels_path):
+            logger.warning(f"Qrels not found in {qrels_path}. Trying to download the qrels from the remote and save it to {save_dir}.")
+            return self._load_remote_qrels(dataset_name=dataset_name, split=split, save_dir=save_dir)
+        else:
+            qrels_data = datasets.load_dataset('json', data_files=qrels_path, cache_dir=self.cache_dir)['train']
+
+            qrels = {}
+            for data in qrels_data:
+                qid = data['qid']
+                qrels[qid] = data['answers']
+
+            return datasets.DatasetDict(qrels)
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: Optional[str] = None,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """
+        Refer to: https://arxiv.org/pdf/2402.03216. We use the corpus from the BeIR dataset.
+        """
+        corpus = datasets.load_dataset(
+            "BeIR/nq", "corpus",
+            cache_dir=self.cache_dir,
+            trust_remote_code=True,
+            download_mode=self.hf_download_mode
+        )["corpus"]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    docid, title, text = str(data["_id"]), normalize_text(data["title"]).lower(), normalize_text(data["text"]).lower()
+                    _data = {
+                        "id": docid,
+                        "title": title,
+                        "text": text
+                    }
+                    corpus_dict[docid] = {
+                        "title": title,
+                        "text": text
+                    }
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} corpus saved to {save_path}")
+        else:
+            corpus_dict = {}
+            for data in tqdm(corpus, desc="Loading corpus"):
+                docid, title, text = str(data["_id"]), normalize_text(data["title"]), normalize_text(data["text"])
+                corpus_dict[docid] = {
+                    "title": title,
+                    "text": text
+                }
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: str,
+        split: str = 'test',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load remote qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        endpoint = f"{os.getenv('HF_ENDPOINT', 'https://huggingface.co')}/datasets/Shitao/bge-m3-data"
+        queries_download_url = f"{endpoint}/resolve/main/MKQA_test-data.zip"
+
+        qrels_save_dir = self._download_zip_file(queries_download_url, self.cache_dir)
+        qrels_save_path = os.path.join(qrels_save_dir, f"{dataset_name}.jsonl")
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f1:
+                with open(qrels_save_path, "r", encoding="utf-8") as f2:
+                    for line in tqdm(f2.readlines(), desc="Loading and Saving qrels"):
+                        data = json.loads(line)
+                        qid, answers = str(data["id"]), data["answers"]
+                        _data = {
+                            "qid": qid,
+                            "answers": answers
+                        }
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid] = answers
+                        f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+        else:
+            qrels_dict = {}
+            with open(qrels_save_path, "r", encoding="utf-8") as f:
+                for line in tqdm(f.readlines(), desc="Loading qrels"):
+                    data = json.loads(line)
+                    qid, answers = str(data["id"]), data["answers"]
+                    if qid not in qrels_dict:
+                        qrels_dict[qid] = {}
+                    qrels_dict[qid] = answers
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: str,
+        split: str = 'test',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        endpoint = f"{os.getenv('HF_ENDPOINT', 'https://huggingface.co')}/datasets/Shitao/bge-m3-data"
+        queries_download_url = f"{endpoint}/resolve/main/MKQA_test-data.zip"
+
+        queries_save_dir = self._download_zip_file(queries_download_url, self.cache_dir)
+        queries_save_path = os.path.join(queries_save_dir, f"{dataset_name}.jsonl")
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f1:
+                with open(queries_save_path, "r", encoding="utf-8") as f2:
+                    for line in tqdm(f2.readlines(), desc="Loading and Saving queries"):
+                        data = json.loads(line)
+                        qid, query = str(data["id"]), data["question"]
+                        _data = {
+                            "id": qid,
+                            "text": query
+                        }
+                        queries_dict[qid] = query
+                        f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+        else:
+            queries_dict = {}
+            with open(queries_save_path, "r", encoding="utf-8") as f:
+                for line in tqdm(f.readlines(), desc="Loading queries"):
+                    data = json.loads(line)
+                    qid, query = str(data["id"]), data["question"]
+                    queries_dict[qid] = query
+        return datasets.DatasetDict(queries_dict)

FlagEmbedding/evaluation/mkqa/evaluator.py

@@ -0,0 +1,117 @@
+import os
+from tqdm import tqdm
+from typing import Dict, List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvaluator
+
+from .utils.compute_metrics import evaluate_qa_recall
+
+
+class MKQAEvaluator(AbsEvaluator):
+    """
+    The evaluator class of MKQA.
+    """
+    def get_corpus_embd_save_dir(
+        self,
+        retriever_name: str,
+        corpus_embd_save_dir: Optional[str] = None,
+        dataset_name: Optional[str] = None
+    ):
+        """Get the directory to save the corpus embedding.
+
+        Args:
+            retriever_name (str): Name of the retriever.
+            corpus_embd_save_dir (Optional[str], optional): Directory to save the corpus embedding. Defaults to ``None``.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+
+        Returns:
+            str: The final directory to save the corpus embedding.
+        """
+        if corpus_embd_save_dir is not None:
+            # Save the corpus embeddings in the same directory for all dataset_name
+            corpus_embd_save_dir = os.path.join(corpus_embd_save_dir, retriever_name)
+        return corpus_embd_save_dir
+
+    def evaluate_results(
+        self,
+        search_results_save_dir: str,
+        k_values: List[int] = [1, 3, 5, 10, 100, 1000]
+    ):
+        """Compute the metrics and get the eval results.
+
+        Args:
+            search_results_save_dir (str): Directory that saves the search results.
+            k_values (List[int], optional): Cutoffs. Defaults to ``[1, 3, 5, 10, 100, 1000]``.
+
+        Returns:
+            dict: The evaluation results.
+        """
+        eval_results_dict = {}
+
+        corpus = self.data_loader.load_corpus()
+        corpus_dict = {}
+        for docid, data in tqdm(corpus.items(), desc="Loading corpus for evaluation"):
+            title, text = data["title"], data["text"]
+            corpus_dict[docid] = f"{title} {text}".strip()
+
+        for file in os.listdir(search_results_save_dir):
+            if not file.endswith('.json'):
+                continue
+
+            file_path = os.path.join(search_results_save_dir, file)
+            data_info, search_results = self.load_search_results(file_path)
+
+            _eval_name = data_info['eval_name']
+            assert _eval_name == self.eval_name, f'Mismatch eval_name: {_eval_name} vs {self.eval_name} in {file_path}'
+
+            split = data_info['split']
+            dataset_name = data_info.get('dataset_name', None)
+            qrels = self.data_loader.load_qrels(dataset_name=dataset_name, split=split)
+
+            eval_results = self.compute_metrics(
+                corpus_dict=corpus_dict,
+                qrels=qrels,
+                search_results=search_results,
+                k_values=k_values
+            )
+
+            if dataset_name is not None:
+                key = f"{dataset_name}-{split}"
+            else:
+                key = split
+            eval_results_dict[key] = eval_results
+
+        return eval_results_dict
+    
+    @staticmethod
+    def compute_metrics(
+        corpus_dict: Dict[str, str],
+        qrels: Dict[str, List[str]],
+        search_results: Dict[str, Dict[str, float]],
+        k_values: List[int],
+    ):
+        """
+        Compute Recall@k for QA task. The definition of recall in QA task is different from the one in IR task. Please refer to the paper of RocketQA: https://aclanthology.org/2021.naacl-main.466.pdf.
+        
+        Args:
+            corpus_dict (Dict[str, str]): Dictionary of the corpus with doc id and contents.
+            qrels (Dict[str, List[str]]): Relevances of queries and passage.
+            search_results (Dict[str, Dict[str, float]]): Search results of the model to evaluate.
+        
+        Returns:
+            dict: The model's scores of the metrics.
+        """
+        contexts = []
+        answers = []
+        top_k = max(k_values)
+        for qid, doc_score_dict in search_results.items():
+            doc_score_pair = sorted(doc_score_dict.items(), key=lambda x: x[1], reverse=True)
+            _ctxs = [corpus_dict[docid] for docid, _ in doc_score_pair[:top_k]]
+            contexts.append(_ctxs)
+            answers.append(qrels[qid])
+
+        recall = evaluate_qa_recall(contexts, answers, k_values=k_values)
+        scores = {f"qa_recall_at_{k}": v for k, v in zip(k_values, recall)}
+
+        return scores
+    

FlagEmbedding/evaluation/mkqa/runner.py

@@ -0,0 +1,37 @@
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import MKQAEvalDataLoader
+from .evaluator import MKQAEvaluator
+
+
+class MKQAEvalRunner(AbsEvalRunner):
+    """
+    Evaluation runner of MKQA.
+    """
+    def load_data_loader(self) -> MKQAEvalDataLoader:
+        """Load the data loader instance by args.
+
+        Returns:
+            MKQAEvalDataLoader: The MKQA data loader instance.
+        """
+        data_loader = MKQAEvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader
+
+    def load_evaluator(self) -> MKQAEvaluator:
+        """Load the evaluator instance by args.
+
+        Returns:
+            MKQAEvaluator: The MKQA evaluator instance.
+        """
+        evaluator = MKQAEvaluator(
+            eval_name=self.eval_args.eval_name,
+            data_loader=self.data_loader,
+            overwrite=self.eval_args.overwrite,
+        )
+        return evaluator

FlagEmbedding/evaluation/mkqa/utils/compute_metrics.py

@@ -0,0 +1,95 @@
+"""
+Ref: https://github.com/facebookresearch/contriever
+"""
+import regex
+import unicodedata
+from functools import partial
+from typing import List, Union
+
+
+class SimpleTokenizer:
+    ALPHA_NUM = r'[\p{L}\p{N}\p{M}]+'
+    NON_WS = r'[^\p{Z}\p{C}]'
+
+    def __init__(self):
+        """
+        Args:
+            annotators: None or empty set (only tokenizes).
+        """
+        self._regexp = regex.compile(
+            '(%s)|(%s)' % (self.ALPHA_NUM, self.NON_WS),
+            flags=regex.IGNORECASE + regex.UNICODE + regex.MULTILINE
+        )
+
+    def tokenize(self, text, uncased=False):
+        matches = [m for m in self._regexp.finditer(text)]
+        if uncased:
+            tokens = [m.group().lower() for m in matches]
+        else:
+            tokens = [m.group() for m in matches]
+        return tokens
+
+
+def _normalize(text):
+    return unicodedata.normalize('NFD', text)
+
+
+def has_answer(answers, text, tokenizer) -> bool:
+    """Check if a document contains an answer string."""
+    text = _normalize(text)
+    text = tokenizer.tokenize(text, uncased=True)
+
+    for answer in answers:
+        answer = _normalize(answer)
+        answer = tokenizer.tokenize(answer, uncased=True)
+        for i in range(0, len(text) - len(answer) + 1):
+            if answer == text[i: i + len(answer)]:
+                return True
+    return False
+
+
+def check_answer(example, tokenizer) -> List[bool]:
+    """Search through all the top docs to see if they have any of the answers."""
+    answers = example['answers']
+    ctxs = example['ctxs']
+
+    hits = []
+    for i, text in enumerate(ctxs):
+        if text is None:  # cannot find the document for some reason
+            hits.append(False)
+            continue
+        hits.append(has_answer(answers, text, tokenizer))
+    return hits
+
+
+def evaluate_qa_recall(ctxs, answers, k_values: Union[int, List[int]]=100):
+    # compute Recall@k for QA task
+    data = []
+    assert len(ctxs) == len(answers)
+    for i in range(len(ctxs)):
+        _ctxs, _answers = ctxs[i], answers[i]
+        data.append({
+            'answers': _answers,
+            'ctxs': _ctxs,
+        })
+    tokenizer = SimpleTokenizer()
+    get_score_partial = partial(check_answer, tokenizer=tokenizer)
+
+    scores = map(get_score_partial, data)
+
+    n_docs = len(data[0]['ctxs'])
+    top_k_hits = [0] * n_docs
+    for question_hits in scores:
+        best_hit = next((i for i, x in enumerate(question_hits) if x), None)
+        if best_hit is not None:
+            top_k_hits[best_hit:] = [v + 1 for v in top_k_hits[best_hit:]]
+
+    if isinstance(k_values, int):
+        k = min(k_values, len(top_k_hits))
+        return top_k_hits[k - 1] / len(data)
+    else:
+        scores = []
+        for k in k_values:
+            k = min(k, len(top_k_hits))
+            scores.append(top_k_hits[k - 1] / len(data))
+        return scores

FlagEmbedding/evaluation/mkqa/utils/normalize_text.py

@@ -0,0 +1,162 @@
+"""
+adapted from chemdataextractor.text.normalize
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Tools for normalizing text.
+https://github.com/mcs07/ChemDataExtractor
+:copyright: Copyright 2016 by Matt Swain.
+:license: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+#: Control characters.
+CONTROLS = {
+    '\u0001', '\u0002', '\u0003', '\u0004', '\u0005', '\u0006', '\u0007', '\u0008', '\u000e', '\u000f', '\u0011',
+    '\u0012', '\u0013', '\u0014', '\u0015', '\u0016', '\u0017', '\u0018', '\u0019', '\u001a', '\u001b',
+}
+# There are further control characters, but they are instead replaced with a space by unicode normalization
+# '\u0009', '\u000a', '\u000b', '\u000c', '\u000d', '\u001c',  '\u001d', '\u001e', '\u001f'
+
+
+#: Hyphen and dash characters.
+HYPHENS = {
+    '-',  # \u002d Hyphen-minus
+    '‐',  # \u2010 Hyphen
+    '‑',  # \u2011 Non-breaking hyphen
+    '⁃',  # \u2043 Hyphen bullet
+    '‒',  # \u2012 figure dash
+    '–',  # \u2013 en dash
+    '—',  # \u2014 em dash
+    '―',  # \u2015 horizontal bar
+}
+
+#: Minus characters.
+MINUSES = {
+    '-',  # \u002d Hyphen-minus
+    '−',  # \u2212 Minus
+    '－',  # \uff0d Full-width Hyphen-minus
+    '⁻',  # \u207b Superscript minus
+}
+
+#: Plus characters.
+PLUSES = {
+    '+',  # \u002b Plus
+    '＋',  # \uff0b Full-width Plus
+    '⁺',  # \u207a Superscript plus
+}
+
+#: Slash characters.
+SLASHES = {
+    '/',  # \u002f Solidus
+    '⁄',  # \u2044 Fraction slash
+    '∕',  # \u2215 Division slash
+}
+
+#: Tilde characters.
+TILDES = {
+    '~',  # \u007e Tilde
+    '˜',  # \u02dc Small tilde
+    '⁓',  # \u2053 Swung dash
+    '∼',  # \u223c Tilde operator #in mbert vocab
+    '∽',  # \u223d Reversed tilde
+    '∿',  # \u223f Sine wave
+    '〜',  # \u301c Wave dash #in mbert vocab
+    '～',  # \uff5e Full-width tilde #in mbert vocab
+}
+
+#: Apostrophe characters.
+APOSTROPHES = {
+    "'",  # \u0027
+    '’',  # \u2019
+    '՚',  # \u055a
+    'Ꞌ',  # \ua78b
+    'ꞌ',  # \ua78c
+    '＇',  # \uff07
+}
+
+#: Single quote characters.
+SINGLE_QUOTES = {
+    "'",  # \u0027
+    '‘',  # \u2018
+    '’',  # \u2019
+    '‚',  # \u201a
+    '‛',  # \u201b
+
+}
+
+#: Double quote characters.
+DOUBLE_QUOTES = {
+    '"',  # \u0022
+    '“',  # \u201c
+    '”',  # \u201d
+    '„',  # \u201e
+    '‟',  # \u201f
+}
+
+#: Accent characters.
+ACCENTS = {
+    '`',  # \u0060
+    '´',  # \u00b4
+}
+
+#: Prime characters.
+PRIMES = {
+    '′',  # \u2032
+    '″',  # \u2033
+    '‴',  # \u2034
+    '‵',  # \u2035
+    '‶',  # \u2036
+    '‷',  # \u2037
+    '⁗',  # \u2057
+}
+
+#: Quote characters, including apostrophes, single quotes, double quotes, accents and primes.
+QUOTES = APOSTROPHES | SINGLE_QUOTES | DOUBLE_QUOTES | ACCENTS | PRIMES
+
+def normalize_text(text: str):
+    for control in CONTROLS:
+        text = text.replace(control, '')
+    text = text.replace('\u000b', ' ').replace('\u000c', ' ').replace(u'\u0085', ' ')
+
+    for hyphen in HYPHENS | MINUSES:
+        text = text.replace(hyphen, '-')
+    text = text.replace('\u00ad', '')
+
+    for double_quote in DOUBLE_QUOTES:
+        text = text.replace(double_quote, '"')  # \u0022
+    for single_quote in (SINGLE_QUOTES | APOSTROPHES | ACCENTS):
+        text = text.replace(single_quote, "'")  # \u0027
+    text = text.replace('′', "'")     # \u2032 prime
+    text = text.replace('‵', "'")     # \u2035 reversed prime
+    text = text.replace('″', "''")    # \u2033 double prime
+    text = text.replace('‶', "''")    # \u2036 reversed double prime
+    text = text.replace('‴', "'''")   # \u2034 triple prime
+    text = text.replace('‷', "'''")   # \u2037 reversed triple prime
+    text = text.replace('⁗', "''''")  # \u2057 quadruple prime
+
+    text = text.replace('…', '...').replace(' . . . ', ' ... ')  # \u2026
+
+    for slash in SLASHES:
+        text = text.replace(slash, '/')
+
+    #for tilde in TILDES:
+    #    text = text.replace(tilde, '~')
+
+    return text

FlagEmbedding/evaluation/mldr/__init__.py

@@ -0,0 +1,14 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalArgs as MLDREvalArgs,
+    AbsEvalModelArgs as MLDREvalModelArgs,
+)
+
+from .data_loader import MLDREvalDataLoader
+from .runner import MLDREvalRunner
+
+__all__ = [
+    "MLDREvalArgs",
+    "MLDREvalModelArgs",
+    "MLDREvalRunner",
+    "MLDREvalDataLoader",
+]

FlagEmbedding/evaluation/mldr/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.mldr import (
+    MLDREvalArgs, MLDREvalModelArgs,
+    MLDREvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        MLDREvalArgs,
+        MLDREvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: MLDREvalArgs
+    model_args: MLDREvalModelArgs
+
+    runner = MLDREvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/mldr/data_loader.py

@@ -0,0 +1,184 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class MLDREvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for MLDR.
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return ["ar", "de", "en", "es", "fr", "hi", "it", "ja", "ko", "pt", "ru", "th", "zh"]
+
+    def available_splits(self, dataset_name: Optional[str] = None) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (Optional[str], optional): Dataset name. Defaults to ``None``.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        return ["train", "dev", "test"]
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        corpus = datasets.load_dataset(
+            "Shitao/MLDR", f"corpus-{dataset_name}",
+            cache_dir=self.cache_dir,
+            trust_remote_code=True,
+            download_mode=self.hf_download_mode
+        )["corpus"]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    docid, text = str(data["docid"]), data["text"]
+                    _data = {
+                        "id": docid,
+                        "text": text
+                    }
+                    corpus_dict[docid] = {"text": text}
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+        else:
+            corpus_dict = {str(data["docid"]): {"text": data["text"]} for data in tqdm(corpus, desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: str,
+        split: str = "test",
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        qrels_data = datasets.load_dataset(
+            "Shitao/MLDR", dataset_name,
+            cache_dir=self.cache_dir,
+            trust_remote_code=True,
+            download_mode=self.hf_download_mode
+        )[split]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(qrels_data, desc="Loading and Saving qrels"):
+                    qid = str(data["query_id"])
+                    if qid not in qrels_dict:
+                        qrels_dict[qid] = {}
+                    for doc in data["positive_passages"]:
+                        docid = str(doc["docid"])
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": 1
+                        }
+                        qrels_dict[qid][docid] = 1
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+                    for doc in data["negative_passages"]:
+                        docid = str(doc["docid"])
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": 0
+                        }
+                        qrels_dict[qid][docid] = 0
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+        else:
+            qrels_dict = {}
+            for data in tqdm(qrels_data, desc="Loading qrels"):
+                qid = str(data["query_id"])
+                if qid not in qrels_dict:
+                    qrels_dict[qid] = {}
+                for doc in data["positive_passages"]:
+                    docid = str(doc["docid"])
+                    qrels_dict[qid][docid] = 1
+                for doc in data["negative_passages"]:
+                    docid = str(doc["docid"])
+                    qrels_dict[qid][docid] = 0
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: str,
+        split: str = "test",
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        queries_data = datasets.load_dataset(
+            "Shitao/MLDR", dataset_name,
+            cache_dir=self.cache_dir,
+            trust_remote_code=True,
+            download_mode=self.hf_download_mode
+        )[split]
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(queries_data, desc="Loading and Saving queries"):
+                    qid, query = str(data["query_id"]), data["query"]
+                    _data = {
+                        "id": qid,
+                        "text": query
+                    }
+                    queries_dict[qid] = query
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+        else:
+            queries_dict = {}
+            for data in tqdm(queries_data, desc="Loading queries"):
+                qid, query = str(data["query_id"]), data["query"]
+                queries_dict[qid] = query
+        return datasets.DatasetDict(queries_dict)

FlagEmbedding/evaluation/mldr/runner.py

@@ -0,0 +1,23 @@
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import MLDREvalDataLoader
+
+
+class MLDREvalRunner(AbsEvalRunner):
+    """
+    Evaluation runner of MIRACL.
+    """
+    def load_data_loader(self) -> MLDREvalDataLoader:
+        """Load the data loader instance by args.
+
+        Returns:
+            MLDREvalDataLoader: The MLDR data loader instance.
+        """
+        data_loader = MLDREvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader

FlagEmbedding/evaluation/msmarco/__init__.py

@@ -0,0 +1,14 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalArgs as MSMARCOEvalArgs,
+    AbsEvalModelArgs as MSMARCOEvalModelArgs,
+)
+
+from .data_loader import MSMARCOEvalDataLoader
+from .runner import MSMARCOEvalRunner
+
+__all__ = [
+    "MSMARCOEvalArgs",
+    "MSMARCOEvalModelArgs",
+    "MSMARCOEvalRunner",
+    "MSMARCOEvalDataLoader",
+]

FlagEmbedding/evaluation/msmarco/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.msmarco import (
+    MSMARCOEvalArgs, MSMARCOEvalModelArgs,
+    MSMARCOEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        MSMARCOEvalArgs,
+        MSMARCOEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: MSMARCOEvalArgs
+    model_args: MSMARCOEvalModelArgs
+
+    runner = MSMARCOEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/msmarco/data_loader.py

@@ -0,0 +1,277 @@
+import os
+import json
+import logging
+import datasets
+from tqdm import tqdm
+from typing import List, Optional
+
+from FlagEmbedding.abc.evaluation import AbsEvalDataLoader
+
+logger = logging.getLogger(__name__)
+
+
+class MSMARCOEvalDataLoader(AbsEvalDataLoader):
+    """
+    Data loader class for MSMARCO.
+    """
+    def available_dataset_names(self) -> List[str]:
+        """
+        Get the available dataset names.
+
+        Returns:
+            List[str]: All the available dataset names.
+        """
+        return ["passage", "document"]
+
+    def available_splits(self, dataset_name: Optional[str] = None) -> List[str]:
+        """
+        Get the avaialble splits.
+
+        Args:
+            dataset_name (Optional[str], optional): Dataset name. Defaults to ``None``.
+
+        Returns:
+            List[str]: All the available splits for the dataset.
+        """
+        return ["dev", "dl19", "dl20"]
+
+    def _load_remote_corpus(
+        self,
+        dataset_name: str,
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the corpus dataset from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of corpus.
+        """
+        if dataset_name == 'passage':
+            corpus = datasets.load_dataset(
+                'Tevatron/msmarco-passage-corpus', 
+                'default', 
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,
+                download_mode=self.hf_download_mode
+            )['train']
+        else:
+            corpus = datasets.load_dataset(
+                'irds/msmarco-document', 
+                'docs', 
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,
+                download_mode=self.hf_download_mode
+            )
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, "corpus.jsonl")
+            corpus_dict = {}
+            with open(save_path, "w", encoding="utf-8") as f:
+                for data in tqdm(corpus, desc="Loading and Saving corpus"):
+                    if dataset_name == 'passage':
+                        _data = {
+                            "id": data["docid"],
+                            "title": data["title"],
+                            "text": data["text"]
+                        }
+                        corpus_dict[data["docid"]] = {
+                            "title": data["title"],
+                            "text": data["text"]
+                        }
+                    else:
+                        _data = {
+                            "id": data["doc_id"],
+                            "title": data["title"],
+                            "text": data["body"]
+                        }
+                        corpus_dict[data["doc_id"]] = {
+                            "title": data["title"],
+                            "text": data["body"]
+                        }
+                    f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} corpus saved to {save_path}")
+        else:
+            if dataset_name == 'passage':
+                corpus_dict = {data["docid"]: {"title": data["title"], "text": data["text"]} for data in tqdm(corpus, desc="Loading corpus")}
+            else:
+                corpus_dict = {data["doc_id"]: {"title": data["title"], "text": data["body"]} for data in tqdm(corpus, desc="Loading corpus")}
+        return datasets.DatasetDict(corpus_dict)
+
+    def _load_remote_qrels(
+        self,
+        dataset_name: Optional[str] = None,
+        split: str = 'dev',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the qrels from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'dev'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of qrel.
+        """
+        if dataset_name == 'passage':
+            if split == 'dev':
+                qrels = datasets.load_dataset(
+                    'BeIR/msmarco-qrels', 
+                    split='validation',
+                    trust_remote_code=True,
+                    cache_dir=self.cache_dir,
+                    download_mode=self.hf_download_mode
+                )
+                qrels_download_url = None
+            elif split == 'dl19':
+                qrels_download_url = "https://trec.nist.gov/data/deep/2019qrels-pass.txt"
+            else:
+                qrels_download_url = "https://trec.nist.gov/data/deep/2020qrels-pass.txt"
+        else:
+            if split == 'dev':
+                qrels_download_url = "https://msmarco.z22.web.core.windows.net/msmarcoranking/msmarco-docdev-qrels.tsv.gz"
+            elif split == 'dl19':
+                qrels_download_url = "https://trec.nist.gov/data/deep/2019qrels-docs.txt"
+            else:
+                qrels_download_url = "https://trec.nist.gov/data/deep/2020qrels-docs.txt"
+
+        if qrels_download_url is not None:
+            qrels_save_path = self._download_file(qrels_download_url, self.cache_dir)
+        else:
+            qrels_save_path = None
+        
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_qrels.jsonl")
+            qrels_dict = {}
+            if qrels_save_path is not None:
+                with open(save_path, "w", encoding="utf-8") as f1:
+                    with open(qrels_save_path, "r", encoding="utf-8") as f2:
+                        for line in tqdm(f2.readlines(), desc="Loading and Saving qrels"):
+                            qid, _, docid, rel = line.strip().split()
+                            qid, docid, rel = str(qid), str(docid), int(rel)
+                            _data = {
+                                "qid": qid,
+                                "docid": docid,
+                                "relevance": rel
+                            }
+                            if qid not in qrels_dict:
+                                qrels_dict[qid] = {}
+                            qrels_dict[qid][docid] = rel
+                            f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            else:
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for data in tqdm(qrels, desc="Loading and Saving qrels"):
+                        qid, docid, rel = str(data['query-id']), str(data['corpus-id']), int(data['score'])
+                        _data = {
+                            "qid": qid,
+                            "docid": docid,
+                            "relevance": rel
+                        }
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid][docid] = rel
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} qrels saved to {save_path}")
+        else:
+            qrels_dict = {}
+            if qrels_save_path is None:
+                with open(qrels_save_path, "r", encoding="utf-8") as f:
+                    for line in tqdm(f.readlines(), desc="Loading qrels"):
+                        qid, _, docid, rel = line.strip().split()
+                        qid, docid, rel = str(qid), str(docid), int(rel)
+                        if qid not in qrels_dict:
+                            qrels_dict[qid] = {}
+                        qrels_dict[qid][docid] = rel
+            else:
+                for data in tqdm(qrels, desc="Loading queries"):
+                    qid, docid, rel = str(data['query-id']), str(data['corpus-id']), int(data['score'])
+                    if qid not in qrels_dict:
+                        qrels_dict[qid] = {}
+                    qrels_dict[qid][docid] = rel
+        return datasets.DatasetDict(qrels_dict)
+
+    def _load_remote_queries(
+        self,
+        dataset_name: Optional[str] = None,
+        split: str = 'test',
+        save_dir: Optional[str] = None
+    ) -> datasets.DatasetDict:
+        """Load the queries from HF.
+
+        Args:
+            dataset_name (str): Name of the dataset.
+            split (str, optional): Split of the dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Directory to save the dataset. Defaults to ``None``.
+
+        Returns:
+            datasets.DatasetDict: Loaded datasets instance of queries.
+        """
+        if split == 'dev':
+            if dataset_name == 'passage':
+                queries = datasets.load_dataset(
+                    'BeIR/msmarco', 
+                    'queries',
+                    trust_remote_code=True,
+                    cache_dir=self.cache_dir,
+                    download_mode=self.hf_download_mode
+                )['queries']
+                queries_save_path = None
+            else:
+                queries_download_url = "https://msmarco.z22.web.core.windows.net/msmarcoranking/msmarco-docdev-qrels.tsv.gz"
+                queries_save_path = self._download_gz_file(queries_download_url, self.cache_dir)
+        else:
+            year = split.replace("dl", "")
+            queries_download_url = f"https://msmarco.z22.web.core.windows.net/msmarcoranking/msmarco-test20{year}-queries.tsv.gz"
+            queries_save_path = self._download_gz_file(queries_download_url, self.cache_dir)
+
+        qrels = self.load_qrels(dataset_name=dataset_name, split=split)
+
+        if save_dir is not None:
+            os.makedirs(save_dir, exist_ok=True)
+            save_path = os.path.join(save_dir, f"{split}_queries.jsonl")
+            queries_dict = {}
+            if queries_save_path is not None:
+                with open(save_path, "w", encoding="utf-8") as f1:
+                    with open(queries_save_path, "r", encoding="utf-8") as f2:
+                        for line in tqdm(f2.readlines(), desc="Loading and Saving queries"):
+                            qid, query = line.strip().split("\t")
+                            if qid not in qrels.keys(): continue
+                            qid = str(qid)
+                            _data = {
+                                "id": qid,
+                                "text": query
+                            }
+                            queries_dict[qid] = query
+                            f1.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            else:
+                with open(save_path, "w", encoding="utf-8") as f:
+                    for data in tqdm(queries, desc="Loading and Saving queries"):
+                        qid, query = data['_id'], data['text']
+                        if qid not in qrels.keys(): continue
+                        _data = {
+                            "id": qid,
+                            "text": query
+                        }
+                        queries_dict[qid] = query
+                        f.write(json.dumps(_data, ensure_ascii=False) + "\n")
+            logging.info(f"{self.eval_name} {dataset_name} queries saved to {save_path}")
+        else:
+            queries_dict = {}
+            if queries_save_path is not None:
+                with open(queries_save_path, "r", encoding="utf-8") as f:
+                    for line in tqdm(f.readlines(), desc="Loading queries"):
+                        qid, query = line.strip().split("\t")
+                        qid = str(qid)
+                        if qid not in qrels.keys(): continue
+                        queries_dict[qid] = query
+            else:
+                for data in tqdm(queries, desc="Loading queries"):
+                    qid, query = data['_id'], data['text']
+                    if qid not in qrels.keys(): continue
+                    queries_dict[qid] = query
+        return datasets.DatasetDict(queries_dict)

FlagEmbedding/evaluation/msmarco/runner.py

@@ -0,0 +1,23 @@
+from FlagEmbedding.abc.evaluation import AbsEvalRunner
+
+from .data_loader import MSMARCOEvalDataLoader
+
+
+class MSMARCOEvalRunner(AbsEvalRunner):
+    """
+    Evaluation runner of MSMARCO.
+    """
+    def load_data_loader(self) -> MSMARCOEvalDataLoader:
+        """Load the data loader instance by args.
+
+        Returns:
+            MSMARCOEvalDataLoader: The MSMARCO data loader instance.
+        """
+        data_loader = MSMARCOEvalDataLoader(
+            eval_name=self.eval_args.eval_name,
+            dataset_dir=self.eval_args.dataset_dir,
+            cache_dir=self.eval_args.cache_path,
+            token=self.eval_args.token,
+            force_redownload=self.eval_args.force_redownload,
+        )
+        return data_loader

FlagEmbedding/evaluation/mteb/__init__.py

@@ -0,0 +1,12 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalModelArgs as MTEBEvalModelArgs,
+)
+
+from .arguments import MTEBEvalArgs
+from .runner import MTEBEvalRunner
+
+__all__ = [
+    "MTEBEvalArgs",
+    "MTEBEvalModelArgs",
+    "MTEBEvalRunner",
+]

FlagEmbedding/evaluation/mteb/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.mteb import (
+    MTEBEvalArgs, MTEBEvalModelArgs,
+    MTEBEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        MTEBEvalArgs,
+        MTEBEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: MTEBEvalArgs
+    model_args: MTEBEvalModelArgs
+
+    runner = MTEBEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/mteb/arguments.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass, field
+from typing import List
+
+from FlagEmbedding.abc.evaluation.arguments import AbsEvalArgs
+
+
+@dataclass
+class MTEBEvalArgs(AbsEvalArgs):
+    """
+    Argument class for MTEB evaluation.
+    """
+    languages: List[str] = field(
+        default=None, metadata={"help": "Languages to evaluate. Default: eng"}
+    )
+    tasks: List[str] = field(
+        default=None, metadata={"help": "Tasks to evaluate. Default: None"}
+    )
+    task_types: List[str] = field(
+        default=None, metadata={"help": "The task types to evaluate. Default: None"}
+    )
+    use_special_instructions: bool = field(
+        default=False, metadata={"help": "Whether to use specific instructions in `prompts.py` for evaluation. Default: False"}
+    )
+    examples_path: str = field(
+        default=None, metadata={"help": "Use specific examples in the path. Default: None"}
+    )

FlagEmbedding/evaluation/mteb/examples/AmazonCounterfactualClassification.csv

@@ -0,0 +1,4 @@
+text,label
+"I wish I could have used this head set but the day I received it it wouldn't even turn on and I really wanted this product to work I'm very disappointed.","counterfactual"
+"I would advise that instead of trying to follow these poor instructions, Google it.","not-counterfactual"
+"I wrote to Monster customer service before ordering and they told me it would be fine to use without a converter and it was absolutely true.","not-counterfactual"

FlagEmbedding/evaluation/mteb/examples/AmazonPolarityClassification.csv

@@ -0,0 +1,5 @@
+text,label
+"Hunting the Hard Way Thia was a gift for my Husband, who loved the book. It arrived on the date we were told it would.",positive
+"Poor DVD Has too many interviews with people at the Live THomas day in Penn. My kids were annoyed and hated this DVD.",negative
+"Ludicrous and silly I remember getting this book so faintly that that says alot about my opinion of it. Basically, while I will entertain lots of odd ideas and theories, this book was basically silly.",negative
+"Artistry I think that the Deodato concerts are very rich, as he used real strings and band musicians, as well as you can appreciate the John Tropea excelent renditions on guitar.",positive

FlagEmbedding/evaluation/mteb/examples/AmazonReviewsClassification.csv

@@ -0,0 +1,6 @@
+text,label
+"DO NOT ORDER THIS\n\nThis isn't what's described at all. Taking it out of the package lace was cut upon arrival, wig was cut to like 14 inch, not curly, and smelled like cigarettes. I obviously was sent what someone returned, disgusting.Not what I ordered at all, not pleased at all. I want my money back DO NOT ORDER","1 star"
+"And I can’t return it\n\nThis product seemed like good quality but it does not stay stuck to the soles at all. You walk a few steps and then you find the black shoe grip somewhere on the floor.","2 star"
+"Three Stars\n\nnew yearly subscription plan is horrible, but the product still works as it did in the past","3 star"
+"I like how it has lots of pockets to put stuff ...\n\nI like how it has lots of pockets to put stuff in. I would have liked to have a shorter securing strap so it would not slide around so much. Good product.","4 star"
+"Great\n\nIt is really good. That's my favorite. THANK YOU","5 star"

FlagEmbedding/evaluation/mteb/examples/ArguAna.csv

@@ -0,0 +1,4 @@
+query,pos
+"People will die if we don’t do animal testing Every year, 23 new drugs are introduced in the UK alone.[13] Almost all will be tested on animals. A new drug will be used for a long time. Think of all the people saved by the use of penicillin. If drugs cost more to test, that means drug companies will develop less. This means more people suffering and dying","animals science science general ban animal testing junior Many of these drugs are “me too” drugs – ones with a slight change that doesn’t make much difference to an existing drug. [14] So often the benefits from animal testing are marginal, and even if there was a slight increase in human suffering, it would be worth it based on the animal suffering saved."
+"Survival of the fittest It is natural for human beings to farm, kill, and eat other species. In the wild there is a brutal struggle for existence as is shown by Darwin’s On the Origin of the Species. The fact that we humans have succeeded in that struggle by exploiting our natural environment means that we have a natural right over lower species. The concept of survival of the fittest may seem outdated but it is still the defining order of nature. In fact farming animals is much less brutal than the pain and hardship that animals inflict on each other naturally in the wild.","The claim of human entitlement over other species based on 'survival of the fittest' is flawed. While Darwin's theory highlights competition, it doesn't justify exploitation. Our capacity for empathy and moral reasoning surpasses mere survival instincts. Farming still inflicts suffering, contradicting the notion of human superiority. Ethical considerations should guide our treatment of animals, not outdated notions of natural selection."
+"Underground Nuclear Storage is Expensive. Underground nuclear storage is expensive. This is because the deep geological repositories needed to deal with such waste are difficult to construct. This is because said repositories need to be 300m underground and also need failsafe systems so that they can be sealed off should there be a leak. For smaller countries, implementing this idea is almost completely impossible. Further, the maintenance of the facilities also requires a lot of long-term investment as the structural integrity of the facilities must consistently be monitored and maintained so that if there is a leak, the relevant authorities can be informed quickly and efficiently. This is seen with the Yucca mountain waste repository site which has cost billions of dollars since the 1990s and was eventually halted due to public fears about nuclear safety.","While initial construction and maintenance entail significant costs, advancements in technology offer more cost-effective solutions. Modular storage designs and improved monitoring systems mitigate expenses. Collaborative international efforts can also distribute costs. Additionally, public concerns can be addressed through transparent safety protocols and community engagement, ensuring responsible nuclear waste management without exorbitant expenditure. Underground nuclear storage isn't inherently prohibitive."

FlagEmbedding/evaluation/mteb/examples/ArxivClusteringP2P.csv

@@ -0,0 +1,6 @@
+text,label
+"A Novel Approach to Enhancing Cybersecurity in Smart Grids through Deep Reinforcement Learning   The integration of renewable energy sources and advanced metering infrastructure in smart grids introduces complex cybersecurity challenges. In this paper, we propose a novel approach utilizing deep reinforcement learning (DRL) to enhance the resilience of smart grids against cyber attacks. Our method leverages DRL agents to dynamically optimize intrusion detection and response strategies based on real-time grid conditions and attack patterns. We demonstrate through simulations on a realistic smart grid testbed that our approach effectively reduces the impact of cyber threats while maintaining grid operational efficiency and reliability. The results highlight significant improvements in security posture compared to traditional rule-based and anomaly detection approaches.",cs
+"Dynamics of Frobenius Endomorphisms in Characteristic p   This paper investigates the dynamics of Frobenius endomorphisms in characteristic 𝑝, focusing on their algebraic and arithmetic properties. We explore the behavior of Frobenius endomorphisms on varieties over finite fields and delve into their applications in number theory and algebraic geometry. Specifically, we analyze the distribution of fixed points, the growth rates of orbits under iteration, and connections to zeta functions and L-functions. Theoretical results are complemented by computational experiments that illustrate the interplay between Frobenius endomorphisms and geometric structures. Our findings contribute to a deeper understanding of the arithmetic nature of varieties and their representations in characteristic 𝑝, offering insights into fundamental questions in modern algebraic and arithmetic geometry.",math
+"Probing Exoplanetary Atmospheres Using Transmission Spectroscopy with the James Webb Space Telescope   Transmission spectroscopy has revolutionized our understanding of exoplanetary atmospheres, revealing key insights into their chemical compositions and physical properties. With the upcoming launch of the James Webb Space Telescope (JWST), we explore the potential of this technique to characterize exoplanetary atmospheres across a wide range of wavelengths and planetary types. We present a comprehensive analysis framework that incorporates high-resolution spectroscopic data and advanced atmospheric models to interpret transmission spectra obtained by JWST. Our simulations predict detectability thresholds for key molecular species and atmospheric features, offering critical guidance for future observational campaigns aimed at unraveling the diversity and origins of exoplanetary atmospheres.",astro-ph
+"Quantum Coherence and Information Transfer in Photosynthetic Complexes: Insights from Coherent Spectroscopy   Photosynthetic complexes are renowned for their efficient energy transfer mechanisms, driven by quantum coherence phenomena over femtosecond timescales. This paper explores the role of coherent spectroscopy techniques in elucidating the quantum dynamics underlying energy transfer processes in natural photosynthetic systems. We review recent experimental findings and theoretical models that highlight the significance of quantum coherence in optimizing energy capture and transport efficiency in photosynthetic complexes. Our analysis integrates insights from ultrafast spectroscopy experiments with advanced quantum mechanical simulations, providing a comprehensive framework for understanding the interplay between coherence, environmental influences, and biological functionality in photosynthesis.",quant-ph
+"Quantum Hall Effect in Moiré Superlattices of Twisted Bilayer Graphene   The discovery of the quantum Hall effect in moiré superlattices formed by twisted bilayer graphene has opened new avenues in the study of correlated electron systems. This paper investigates the emergence of fractional quantum Hall states and their robustness against disorder and varying twist angles in twisted bilayer graphene. We analyze experimental observations of Landau level spectra and magnetotransport measurements, revealing distinctive features such as enhanced localization and unconventional symmetry breaking effects. Our theoretical framework integrates effective model descriptions and numerical simulations to elucidate the underlying mechanisms driving the quantum Hall phenomena in moiré superlattices, paving the way for future applications in quantum devices and topological materials.",cond-mat

FlagEmbedding/evaluation/mteb/examples/ArxivClusteringS2S.csv

@@ -0,0 +1,6 @@
+text,label
+"A Survey on Graph Neural Networks: Algorithms and Applications",cs
+"Hamiltonian Dynamics and KAM Theory for Infinite-Dimensional Systems",math
+"Dark Matter Distribution in Dwarf Spheroidal Galaxies: Constraints from Stellar Kinematics",astro-ph
+"Decoherence and Quantum Error Correction in Topological Quantum Computers",quant-ph
+"Spin-Orbit Coupling Effects in Low-Dimensional Quantum Materials",cond-mat

FlagEmbedding/evaluation/mteb/examples/AskUbuntuDupQuestions.csv

@@ -0,0 +1,4 @@
+query,positive
+angularjs infinite scroll in a container,AngularJS ng-infinite-scroll not working on a specific container/div
+Java: Efficiently converting an array of longs to an array of bytes,Most Compact way to Serialize an Array of Longs in Java
+PyVISA missing methods,NI VISA + pyVisa on Mac OS X (Snow Leopard)

FlagEmbedding/evaluation/mteb/examples/BIOSSES.csv

@@ -0,0 +1,4 @@
+sent1,sent2
+"Recent studies have highlighted the crucial role of p53 in regulating cell cycle progression.","Recent research underscores p53's pivotal function in controlling cellular division."
+"Neuroscience has revealed intricate pathways linking dopamine to reward and motivation.","Recent neuroscientific findings have illuminated complex dopamine pathways associated with motivation and reward."
+"Stem cell research holds promise for treating a variety of degenerative diseases.","The potential of stem cell research in combating degenerative illnesses is widely recognized."

FlagEmbedding/evaluation/mteb/examples/Banking77Classification.csv

@@ -0,0 +1,7 @@
+text,label
+"What is my money worth in other countries?",exchange_rate
+"What can I do if my card still hasn't arrived after 2 weeks?",card_arrival
+"Would I be able to open an account for my daughter?",age_limit
+"My address details have changed and I want to update them",edit_personal_details
+"If my cash withdrawal is still not showing, is something wrong?",pending_cash_withdrawal
+"How long do transfers typically take? Is there a way of speeding the process up? My friend needs the money I sent her desperately.",transfer_not_received_by_recipient

FlagEmbedding/evaluation/mteb/examples/BiorxivClusteringP2P.csv

@@ -0,0 +1,6 @@
+text,label
+"Neural Mechanisms of Social Cognition: A Study on Mirror Neurons and EmpathySocial cognition is the mental process involved in understanding, recognizing, and predicting others' behavior and emotions. In this study, we investigate the role of mirror neurons in the process of empathy by using a combination of functional magnetic resonance imaging (fMRI) and electroencephalography (EEG). Our experiments involve observing the neural activation of participants as they watch videos of individuals experiencing various emotional states. We demonstrate that specific mirror neuron systems in the premotor cortex and the inferior parietal lobule are significantly activated when participants empathize with others. This suggests that mirror neurons might be fundamental to the neural basis of empathy, facilitating an understanding of others' emotions by simulating them internally. These findings provide insights into the neural mechanisms underlying social cognition and offer potential pathways for therapeutic interventions in conditions like autism and psychopathy, where social cognition is often impaired.",neuroscience
+"Methicillin-resistant Staphylococcus aureus (MRSA) is a major health threat due to its resistance to multiple antibiotics. This study analyzed 50 clinical MRSA isolates using whole-genome sequencing and phenotypic assays. We identified mecA and mecC genes encoding beta-lactam-resistant penicillin-binding proteins. Mutations in rpoB conferred rifampicin resistance, while changes in gyrA and grlA were linked to fluoroquinolone resistance. Biofilm formation was also found to enhance antibiotic resistance. These findings highlight genetic mechanisms and suggest potential targets for developing new treatments against MRSA infections.",microbiology
+"Deep Learning Approaches for Predicting Protein-Protein Interactions from Sequence Data\nProtein-protein interactions (PPIs) are fundamental to numerous biological processes, and understanding these interactions is critical for uncovering cellular mechanisms and developing therapeutic strategies. Traditional experimental methods for identifying PPIs are labor-intensive and time-consuming, highlighting the need for computational approaches. In this study, we present DeepPPI, a deep learning-based framework designed to predict PPIs directly from protein sequence data. DeepPPI employs a combination of convolutional neural networks (CNNs) and recurrent neural networks (RNNs) to capture both local and global sequence features. We trained DeepPPI on a comprehensive dataset of known PPIs and benchmarked its performance against existing methods, demonstrating superior accuracy and generalizability. Additionally, we applied DeepPPI to predict novel interactions in the human proteome and validated a subset of these predictions experimentally. Our results indicate that DeepPPI not only achieves high prediction accuracy but also provides insights into the structural and functional basis of protein interactions, making it a valuable tool for the bioinformatics community.",bioinformatics
+"Cell migration, pivotal in wound healing, immune responses, and cancer metastasis, relies on the actin cytoskeleton for membrane protrusions and movement. We explore phosphoinositides' role—key membrane phospholipids—in this process. Using live-cell imaging and FRET-based biosensors, we track phosphoinositide dynamics during migration. Our findings reveal distinct distributions: phosphatidylinositol 4,5-bisphosphate (PIP2) enriches actin polymerization sites, while phosphatidylinositol 3,4,5-trisphosphate (PIP3) predominates in membrane ruffles and lamellipodia. Modulating these phosphoinositides via kinases and phosphatases alters actin filament organization and migration speed, suggesting therapeutic targets for diseases involving abnormal cell migration.",cell biology
+"Cell membranes, comprising lipids and proteins, regulate molecular transport and signaling. Lipid rafts, enriched in cholesterol and sphingolipids, organize membrane proteins and influence cellular functions. Using AFM and fluorescence microscopy, we studied how lipid rafts and cholesterol impact membrane mechanics. Manipulating cholesterol levels and disrupting rafts with MβCD revealed changes in stiffness and lipid density. Rafts enhance rigidity and resistance to deformation, while cholesterol depletion increases fluidity and reduces stability. Lipid-protein interactions in rafts maintain membrane integrity. These insights into membrane organization offer strategies for manipulating cellular responses through lipid raft modulation.",biophysics

FlagEmbedding/evaluation/mteb/examples/BiorxivClusteringS2S.csv

@@ -0,0 +1,6 @@
+text,label
+"Neural Circuit Dynamics in Decision-Making: A Computational Model of Prefrontal-Striatal Interactions",neuroscience
+"Metagenomic Insights into Extreme Environments: Microbial Diversity and Functional Adaptations in Antarctic Lakes",microbiology
+"Machine Learning Approaches for Predicting Protein Structure and Function from Sequence Data",bioinformatics
+"Regulation of Stem Cell Fate Decisions by the Hippo Signaling Pathway: Implications for Tissue Regeneration and Cancer Therapy",cell biology
+"Optical Tweezers and Single-Molecule Force Spectroscopy: Probing Protein Folding Dynamics and Mechanical Properties of Biomolecules",biophysics

FlagEmbedding/evaluation/mteb/examples/CQADupstack.csv

@@ -0,0 +1,4 @@
+query,positive
+angularjs infinite scroll in a container,AngularJS ng-infinite-scroll not working on a specific container/div
+Java: Efficiently converting an array of longs to an array of bytes,Most Compact way to Serialize an Array of Longs in Java
+PyVISA missing methods,NI VISA + pyVisa on Mac OS X (Snow Leopard)

FlagEmbedding/evaluation/mteb/examples/CQADupstackRetrieval.csv

@@ -0,0 +1 @@
+query,pos

FlagEmbedding/evaluation/mteb/examples/ClimateFEVER.csv

@@ -0,0 +1,3 @@
+query,pos
+"Global warming is causing more frequent and intense hurricanes.","Hurricanes and Climate Change  Hurricanes, also known as tropical cyclones or typhoons depending on their location, are powerful and destructive weather systems characterized by strong winds, heavy rainfall, and storm surges. The formation and intensity of hurricanes are influenced by a variety of factors, including sea surface temperatures, atmospheric moisture, and wind patterns. Scientific research indicates that global warming is having a significant impact on these factors, leading to changes in hurricane behavior. As sea surface temperatures rise due to increased greenhouse gas emissions, the energy available for hurricane formation and intensification also increases. This has been linked to an increase in the frequency of the most intense hurricanes, categorized as Category 4 and 5 storms. Additionally, warmer air can hold more moisture, leading to heavier rainfall and greater flooding potential during hurricanes. The Intergovernmental Panel on Climate Change (IPCC) reports that while the total number of hurricanes may not be increasing, there is a clear trend towards more intense and damaging storms in a warming world."
+"The Arctic sea ice extent has decreased by nearly 40% since the late 1970s due to global warming.","The Arctic sea ice extent refers to the surface area of the Arctic Ocean covered by sea ice . Observations indicate a significant reduction in Arctic sea ice extent over recent decades . Satellite measurements have shown that the minimum sea ice extent , typically occurring in September , has declined by about 40% since the late 1970s . This decrease is largely attributed to rising global temperatures , which have led to warmer ocean waters and higher air temperatures in the Arctic region . Climate models predict that the Arctic could be nearly ice-free during summer within this century if the current rate of warming continues . This decline in sea ice has profound implications for Arctic ecosystems , global weather patterns , and sea levels ."

FlagEmbedding/evaluation/mteb/examples/DBPedia.csv

@@ -0,0 +1,4 @@
+query,pos 
+"Chefs with a show on the Food Network.","Robert Irvine  Robert Irvine (born 24 September 1965) is a British celebrity chef who has appeared on a variety of Food Network programs including Dinner: Impossible, Worst Cooks in America, Restaurant: Impossible, and Restaurant Express.'"
+"houses of the Russian parliament","State Duma  The State Duma (Russian: Госуда́рственная ду́ма (Gosudarstvennaya Duma), common abbreviation: Госду́ма (Gosduma)) in the Russian Federation is the lower house of the Federal Assembly of Russia (legislature), the upper house being the Federation Council of Russia. The Duma headquarters are located in central Moscow, a few steps from Manege Square. Its members are referred to as deputies."
+"tango music instruments","Tango music  Tango is a style of music in 2/4 or 4/4 time that originated among European immigrant populations of Argentina and Uruguay (collectively, the 'Rioplatenses'). It is traditionally played on a solo guitar, guitar duo, or an ensemble, known as the orquesta típica, which includes at least two violins, flute, piano, double bass, and at least two bandoneóns. Sometimes guitars and a clarinet join the ensemble. Tango may be purely instrumental or may include a vocalist."

FlagEmbedding/evaluation/mteb/examples/EmotionClassification.csv

@@ -0,0 +1,7 @@
+text,label_text
+"i am bothered is that he might changed his feelings once he get back in us and leave me heartbroken",sadness
+"i have always loved my jobs and loved to work and i truly feel like being back there with my patients and co workers will do me a lot of good even if it is only for a few weeks",joy
+"i certainly feel loved and appreciated and grateful for all that i have",love
+"im grabbing a minute to post i feel greedy wrong",anger
+"i was stymied a little bit as i wrote feeling unsure that i might go somewhere with the story unintended",fear
+"i keep feeling pleasantly surprised at his supportiveness and also his ease in new situations",surprise

FlagEmbedding/evaluation/mteb/examples/FEVER.csv

@@ -0,0 +1,5 @@
+query,pos
+"Ricky Martin acts.","Ricky Martin Enrique Martín Morales ( born December 24 , 1971 ) , commonly known as Ricky Martin , is a Puerto Rican singer , actor and author . Martin began his career at age twelve with the all-boy pop group Menudo . After five years with the group , he released several Spanish-language solo albums throughout the 1990s . He also acted on stage and on TV in Mexico , becoming a modest star in the country . In 1994 he starred on the US TV soap opera General Hospital , playing a Puerto Rican singer .   In early 1999 , after releasing several albums in Spanish , Martin performed `` The Cup of Life '' at the 41st Grammy Awards show , which became a catalyst in bringing Latin pop to the forefront of the U.S. music scene . Following its success , Martin released `` Livin ' la Vida Loca '' which helped him obtain enormous success worldwide and is generally seen as the song that began the Latin pop explosion of 1999 and made the transition easier for other Spanish-speaking artists to move into the English-speaking market . Since its release , the song has sold over 8 million copies , making it one of the best selling singles of all time . His first English-language album ( also titled Ricky Martin ) , has sold 22 million copies and is one of the best selling albums of all time . His other studio albums include : Me Amarás ( 1993 ) , A Medio Vivir ( 1995 ) , Vuelve ( 1998 ) , Sound Loaded ( 2000 ) , Almas del Silencio ( 2003 ) , Life ( 2005 ) , Música + Alma + Sexo ( 2011 ) , and A Quien Quiera Escuchar ( 2015 ) ."
+"The 19th G7 summit only included Russia.","19th G7 summit The 19th G7 Summit was held in Tokyo , Japan , on July 7 -- 9 , 1993 . The venue for the summit meetings was the State Guesthouse in Tokyo , Japan .   The Group of Seven ( G7 ) was an unofficial forum which brought together the heads of the richest industrialized countries : France , Germany , Italy , Japan , the United Kingdom , the United States , Canada ( since 1976 ) and the President of the European Commission ( starting officially in 1981 ) . The summits were not meant to be linked formally with wider international institutions ; and in fact , a mild rebellion against the stiff formality of other international meetings was a part of the genesis of cooperation between France 's President Giscard d'Estaing and West Germany 's Chancellor Helmut Schmidt as they conceived the first Group of Six ( G6 ) summit in 1975 ."
+"Ayn Rand condemned force.","Ayn Rand Ayn Rand ( -LSB- ˈaɪn_ˈrænd -RSB- born Alisa Zinov ` yevna Rosenbaum , Али́са Зино́вьевна Розенба́ум -- March 6 , 1982 ) was a Russian-American novelist , philosopher , playwright , and screenwriter . She is known for her two best-selling novels , The Fountainhead and Atlas Shrugged , and for developing a philosophical system she called Objectivism . Educated in Russia , she moved to the United States in 1926 . She had a play produced on Broadway in 1935 -- 1936 . After two early novels that were initially unsuccessful in America , she achieved fame with her 1943 novel , The Fountainhead .   In 1957 , Rand published her best-known work , the novel Atlas Shrugged . Afterward , she turned to non-fiction to promote her philosophy , publishing her own magazines and releasing several collections of essays until her death in 1982 . Rand advocated reason as the only means of acquiring knowledge , and rejected faith and religion . She supported rational and ethical egoism , and rejected altruism . In politics , she condemned the initiation of force as immoral , and opposed collectivism and statism as well as anarchism , and instead supported laissez-faire capitalism , which she defined as the system based on recognizing individual rights . In art , Rand promoted romantic realism . She was sharply critical of most philosophers and philosophical traditions known to her , except for Aristotle , Thomas Aquinas , and classical liberals .   Literary critics received Rand 's fiction with mixed reviews , and academia generally ignored or rejected her philosophy , though academic interest has increased in recent decades . The Objectivist movement attempts to spread her ideas , both to the public and in academic settings . She has been a significant influence among libertarians and American conservatives ."
+"The Bachelorette is not a reality television dating game show.","The Bachelorette The Bachelorette is an American reality television dating game show that debuted on ABC on January 8 , 2003 . The show is a spin-off of The Bachelor aired on the same network . The first season featured Trista Rehn , the runner-up date from the first season of The Bachelor , offering the opportunity for Rehn to choose a husband among 25 bachelors . The 2004 season of The Bachelorette again took a runner-up from the previous season of The Bachelor . After last airing on February 28 , 2005 , the series returned to ABC during the spring of 2008 , following an absence of three years ."

FlagEmbedding/evaluation/mteb/examples/FiQA2018.csv

@@ -0,0 +1,4 @@
+query,pos
+What is a negotiable security and how are they related to derivatives?,"A negotiable security is a financial instrument that can be easily transferred or traded, such as stocks and bonds. Derivatives, like options and futures, derive their value from these securities. They're interrelated because derivatives often involve contracts based on the value of underlying negotiable securities. This relationship allows investors to hedge risk or speculate on price movements without owning the actual assets."
+Why is it important to research a stock before buying it?,"Researching a stock before buying is crucial to assess its financial health, growth prospects, and potential risks. Understanding the company's fundamentals, earnings history, management team, and industry trends helps investors make informed decisions. It also aids in evaluating whether the stock is fairly valued or overvalued, reducing the risk of making uninformed investment choices and potentially suffering losses."
+When are investments taxed?,"Investments are taxed based on various factors, including the type of investment, holding period, and applicable tax laws. Generally, investments incur taxes when they generate income, such as interest, dividends, or capital gains. Interest and dividends are typically taxed in the year they're received, while capital gains tax applies when assets like stocks or real estate are sold at a profit. Tax rates may vary based on investment duration and jurisdiction."

FlagEmbedding/evaluation/mteb/examples/HotpotQA.csv

@@ -0,0 +1,4 @@
+query,pos 
+"Which tennis player Anna-Lena Grönefeld or Mats Wilander turned professional first ?","Anna-Lena Grönefeld  Anna-Lena Grönefeld (born 4 June 1985) is a German tennis player. She turned professional in April 2003."
+"What South Korean K-pop group has 13 members and their own online TV program?","Seventeen (band)  Seventeen (Hangul: 세븐틴 ), also stylized as SEVENTEEN or SVT, is a South Korean boy group formed by Pledis Entertainment in 2015. The group consists of thirteen members who are separated into three sub-units, each with different areas of specialization: a 'Hip-Hop Unit', 'Vocal Unit', and 'Performance Unit'. They have released one studio album and four extended plays."
+"The game show Keep It in the Family was hosted by an actor that played what role in "Coronation Street"?","Keep It in the Family (UK game show)  Keep It in the Family is a British game show that aired on ITV from 26 October 2014 to 19 December 2015 and is hosted by Bradley Walsh."

FlagEmbedding/evaluation/mteb/examples/ImdbClassification.csv

@@ -0,0 +1,2 @@
+text,label_text
+"Renny Harlin's first American film was one of the best of a slew of prison-set horror films(like 'Death House' or 'The Chair')in the late 80's.Twenty years before,guard Lane Smith had wrongfully executed a condemned man.Now,he is the warden of the newly re-opened prison,and the man's ghost is back for bloody revenge.This atmospheric and very moody film features lots of gruesome gore and violence.Viggo Mortensen,Tiny Lister,Tom Everett and Kane Hodder are onhand for the entertaining carnage.","positive"

FlagEmbedding/evaluation/mteb/examples/MSMARCO.csv

@@ -0,0 +1,4 @@
+query,pos
+"what is a pms color","PMS is a solid-color matching system, used primarily for specifying second or third colors in printing, meaning colors in addition to black, (although, obviously, one can certainly print a one-color piece using a PMS color and no black all)."
+"when was snowboarding invented","Snowboarding Modern snowboarding began in 1965 when Sherman Poppen, an engineer in Muskegon, Michigan, invented a toy for his daughters by fastening two skis together and attaching a rope to one end so he would have some control as they stood on the board and glided downhill."
+"difference between pollination fertilization","What is the difference between pollination & fertilization in flowering plants? • Pollination is a process flowering plants only undergo. It is the transfer of pollen to the plant’s stigma. The process can be done by the plant itself or through outside agents. • Fertilization is basically the joining of sperm and egg."

FlagEmbedding/evaluation/mteb/examples/MTOPDomainClassification.csv

@@ -0,0 +1,8 @@
+text,label
+"I am no longer available",calling
+"Cancel my reminder about my dentist appointment",reminder
+"Will it rain tomorrow?",weather
+"Create an appointment alarm for 11:30am.",allarm
+"Play a different playlist",music
+"What's the best way to fry chicken",recipes
+"what city does Ahmed live in ?",people

FlagEmbedding/evaluation/mteb/examples/MTOPIntentClassification.csv

@@ -0,0 +1,8 @@
+text,label	
+"When will my next alarm start",GET_ALARM
+"I need you to message Zachary Fletcher",SEND_MESSAGE
+"show me video messages from Atlas",GET_MESSAGE
+"I want to listen to AC/DC please",PLAY_MUSIC
+"Make an alarm for the next 7 weeks for Thursday at 6pm",CREATE_ALARM
+"fairs happening in ann arbor next week",GET_EVENT
+"Will we get a frost this week?",GET_WEATHER

FlagEmbedding/evaluation/mteb/examples/MassiveIntentClassification.csv

@@ -0,0 +1,9 @@
+text,label
+"remind me to pay rent every month",calendar_set
+"please play yesterday from beatles",play_music
+"what will the temperatures be for the next week",weather_query
+"give me the detailed schedule for next week",calendar_query
+"what's happening in my day",general_quirky
+"dolores how was your day",general_quirky
+"who was appointed as deputy centimeter of uttar pradesh",qa_factoid
+"find me news about trumps speech",news_query

FlagEmbedding/evaluation/mteb/examples/MassiveScenarioClassification.csv

@@ -0,0 +1,7 @@
+text,label
+"can you confirm that my meeting for tomorrow has been canceled",calendar
+"please open my music application and play games by disturbed",play
+"what's the word orange mean",qa
+"find me all mails from magda with holidays word in the title",email
+"get a cup of coffee ready now",iot
+"good morning olly",general

FlagEmbedding/evaluation/mteb/examples/MedrxivClusteringP2P.csv

@@ -0,0 +1,8 @@
+text,label
+"Socioeconomic Disparities in COVID-19 Transmission Risk: A Population-Based Study from Norway\nObjective: Explore socioeconomic disparities in COVID-19 transmission risk across occupational categories in Norway.\nMethods: Analyzed data from 3,559,694 residents aged 20-70 using the International Standard Classification of Occupations (ISCO-08). Logistic regression models adjusted for various factors examined the association between occupation and SARS-CoV-2 infection risk and hospitalization during different pandemic phases.\nResults: Occupations with varying socioeconomic statuses showed different COVID-19 infection risks. Healthcare professionals had higher odds during the initial wave, while service workers had increased odds during later waves. Teachers and administrative personnel also had moderate risk increases. Occupation had limited association with hospitalization after adjusting for confounders.\nConclusion: Socioeconomic factors significantly influence COVID-19 transmission in occupational settings. Targeted public health interventions addressing workplace conditions, testing accessibility, and socioeconomic vulnerability are essential for mitigating future pandemic impacts and developing equitable pandemic preparedness strategies.\nKeywords: COVID-19, Socioeconomic Disparities, Occupational Risk, Pandemic Preparedness, Public Health, Norway, ISCO-08, SARS-CoV-2",infectious diseases
+"Assessing Socioeconomic Determinants of Infectious Disease Spread: A Cross-National Analysis Using Machine Learning Approaches\nBackground: Understanding socioeconomic factors influencing infectious disease transmission is crucial for targeted public health interventions.\nMethods: This study uses machine learning techniques and Bayesian optimization to analyze the impact of socioeconomic variables such as income, education, and healthcare access on disease dynamics. It integrates datasets on disease transmission and socio-demographic characteristics.\nResults: Significant associations between socioeconomic indicators and infectious disease spread were found, highlighting disparities in vulnerability and transmission rates.\nConclusion: Advanced analytical techniques provide nuanced insights into the socioeconomic determinants of disease transmission, aiding evidence-based policymaking to reduce health disparities and enhance epidemic preparedness.\nKeywords: Socioeconomic Determinants, Infectious Disease, Machine Learning, Public Health, Epidemiology, Health Disparities, Bayesian Optimization",epidemiology
+"The COVID-19 pandemic has significantly impacted mental health in Japan, a country with a historically high suicide rate. This study analyzed nationwide data from January 2019 to December 2021 to compare pre-pandemic and pandemic periods. Findings revealed increased anxiety and depression, especially among young adults and women. Suicide rates, which had been declining, saw a notable rise in late 2020, particularly in economically disadvantaged regions and among those facing job loss or financial strain. The pandemic has exacerbated mental health issues, necessitating targeted interventions and support to mitigate long-term public health impacts.",public and global health
+"The application of whole genome sequencing (WGS) in neonatal care can revolutionize early detection and management of rare genetic disorders, often undiagnosed through traditional methods. The NEOseq project, part of the Neonatal Genomics Initiative, enrolled over 12,000 newborns between January 2019 and December 2021 to evaluate WGS feasibility in routine screening. The study demonstrated WGS's technical and clinical utility, identifying disorders undetectable by conventional means. This research aligns with the UK's genomic medicine advancements, suggesting WGS integration into national screening programmes could enhance neonatal healthcare and personalized medicine, setting a precedent for global genomic technologies in public health.",genetic and genomic medicine
+"Longitudinal Analysis of Sleep Disturbances and Cognitive Decline in Older Adults: A 5-Year Prospective Cohort Study Background: Sleep disturbances in older adults are a recognized risk factor for cognitive decline. This study examines their impact on cognitive function over five years.\nMethods: 3,200 participants aged 60+ from Karnataka, India, were assessed annually using sleep questionnaires and cognitive tests. Exclusions included major neuropsychiatric disorders.\nResults: 25% reported sleep disturbances at baseline; 30% developed mild cognitive impairment, and 15% progressed to dementia. Insomnia and sleep apnea significantly accelerated cognitive decline. CPAP for sleep apnea showed modest protective effects.\nConclusion: Addressing sleep disturbances is crucial for mitigating cognitive decline in older adults.",neurology
+
+

FlagEmbedding/evaluation/mteb/examples/MedrxivClusteringS2S.csv

@@ -0,0 +1,6 @@
+text,label
+"Evaluating the Efficacy of New Therapeutic Agents in the Management of Hypertension-Induced Kidney Damage",nephrology
+"Exploring the Relationship Between ICU Staffing Levels and Patient Outcomes in Severe Trauma Cases",intensive care and critical care medicine
+"The Impact of Environmental Allergens on Pediatric Asthma and Ear Infections",otolaryngology
+"Patient-Reported Outcomes in Rehabilitation: The Importance of Psychosocial Factors in Recovery",rehabilitation medicine and physical therapy
+"The Role of Micronutrients in Supporting Immune Function During Viral Infections",nutrition

FlagEmbedding/evaluation/mteb/examples/MindSmallReranking.csv

@@ -0,0 +1,3 @@
+query,pos
+"'Wheel Of Fortune' Guest Delivers Hilarious, Off The Rails Introduction","Charles Rogers, former Michigan State football, Detroit Lions star, dead at 38"
+"Eliud Kipchoge runs 1:59 marathon, first to break 2 hours","AP-NORC poll: Many youths say high school diploma is enough"

FlagEmbedding/evaluation/mteb/examples/NFCorpus.csv

@@ -0,0 +1,4 @@
+query,pos
+"lung disease","Hibiscus anthocyanins rich extract-induced apoptotic cell death in human promyelocytic leukemia cells.  Hibiscus sabdariffa Linne (Malvaceae), an attractive plant believed to be native to Africa, is cultivated in the Sudan and Eastern Taiwan. Anthocyanins exist widely in many vegetables and fruits. Some reports demonstrated that anthocyanins extracted from H. sabdariffa L., Hibiscus anthocyanins (HAs) (which are a group of natural pigments existing in the dried calyx of H. sabdariffa L.) exhibited antioxidant activity and liver protection. Therefore, in this study, we explored the effect of HAs on human cancer cells. The result showed that HAs could cause cancer cell apoptosis, especially in HL-60 cells. Using flow cytometry, we found that HAs treatment (0-4 mg/ml) markedly induced apoptosis in HL-60 cells in a dose- and time-dependent manner. The result also revealed increased phosphorylation in p38 and c-Jun, cytochrome c release, and expression of tBid, Fas, and FasL in the HAs-treated HL-60 cells. We further used SB203580 (p38 inhibitor), PD98059 (MEK inhibitor), SP600125 (JNK inhibitor), and wortmannin (phosphatidylinositol 3-kinase; PI-3K inhibitor) to evaluate their effect on the HAs-induced HL-60 death. The data showed that only SB203580 had strong potential in inhibiting HL-60 cell apoptosis and related protein expression and phosphorylation. Therefore, we suggested that HAs mediated HL-60 apoptosis via the p38-FasL and Bid pathway. According to these results, HAs could be developed as chemopreventive agents. However, further investigations into the specificity and mechanism(s) of HAs are needed."
+"arthritis","A clustering of immune-mediated polyradiculoneuropathy among swine abattoir workers exposed to aerosolized porcine brains, Indiana, United States.  In November 2007 a novel neuropathy, immune-mediated polyradiculoneuropathy (IP), was identified among workers at a Minnesota swine abattoir where a unique compressed air technique was used to remove porcine brains. An epidemiologic investigation at another abattoir in Indiana that also uses this process was launched to evaluate workers self-reporting neurologic illness compatible with IP. A nested case-control study was performed to identify cases and risk factors. Six confirmed, one probable, and three possible IP cases were detected. IP cases were 28-52 years old, of Latino origin, and 62.5% female. Onset dates ranged from April 2005-December 2007; 60% were hospitalized. IP cases at this plant were similar in clinical presentation and exposure risks to those detected in Minnesota. Swine abattoirs using similar brain extraction methods should discontinue this process."
+"vitamin C","Which population level environmental factors are associated with asthma, rhinoconjunctivitis and eczema? Review of the ecological analyses of ISAAC Phase One  The International Study of Asthma and Allergies in Childhood (ISAAC) Phase One showed large worldwide variations in the prevalence of symptoms of asthma, rhinoconjunctivitis and eczema, up to 10 to 20 fold between countries. Ecological analyses were undertaken with ISAAC Phase One data to explore factors that may have contributed to these variations, and are summarised and reviewed here. In ISAAC Phase One the prevalence of symptoms in the past 12 months of asthma, rhinoconjunctivitis and eczema were estimated from studies in 463,801 children aged 13 - 14 years in 155 centres in 56 countries, and in 257,800 children aged 6-7 years in 91 centres in 38 countries. Ecological analyses were undertaken between symptom prevalence and the following: Gross National Product per capita (GNP), food intake, immunisation rates, tuberculosis notifications, climatic factors, tobacco consumption, pollen, antibiotic sales, paracetamol sales, and outdoor air pollution. Symptom prevalence of all three conditions was positively associated with GNP, trans fatty acids, paracetamol, and women smoking, and inversely associated with food of plant origin, pollen, immunisations, tuberculosis notifications, air pollution, and men smoking. The magnitude of these associations was small, but consistent in direction between conditions. There were mixed associations of climate and antibiotic sales with symptom prevalence. The potential causality of these associations warrant further investigation. Factors which prevent the development of these conditions, or where there is an absence of a positive correlation at a population level may be as important from the policy viewpoint as a focus on the positive risk factors. Interventions based on small associations may have the potential for a large public health benefit."

FlagEmbedding/evaluation/mteb/examples/NQ.csv

@@ -0,0 +1,4 @@
+query,pos
+"what is the capital of australia","Canberra   Canberra is the capital city of Australia. Founded following the federation of the colonies of Australia as the seat of government for the new nation, it is Australia's largest inland city and the eighth-largest city overall. Located at the northern end of the Australian Capital Territory, Canberra is an entirely planned city."
+"who invented the world wide web","Tim Berners-Lee    Sir Timothy John Berners-Lee, also known as TimBL, is an English engineer and computer scientist, best known as the inventor of the World Wide Web. He implemented the first successful communication between a Hypertext Transfer Protocol (HTTP) client and server via the Internet in mid-November 1989. Berners-Lee is a professor at the Massachusetts Institute of Technology (MIT) and the University of Oxford."
+"what is the Higgs boson","Higgs Boson    The Higgs boson is an elementary particle in the Standard Model of particle physics. It is the quantum excitation of the Higgs field, which is pivotal to explaining how particles acquire mass. The discovery of the Higgs boson was announced in 2012 by physicists working with the Large Hadron Collider at CERN."

FlagEmbedding/evaluation/mteb/examples/QuoraRetrieval.csv

@@ -0,0 +1,4 @@
+query,pos
+"Why do people say Dhanush (South Indian actor) is ugly? I don't think so.?","Why do people say Dhanush (South Indian actor) is ugly? I don't think so?"
+"What are some hit and nice ideas about architecture dissertation topics?","What are some interesting undergraduate architecture thesis topics?"
+"Could someone please motivate me?","Can you motivate me?"

FlagEmbedding/evaluation/mteb/examples/RedditClustering.csv

@@ -0,0 +1,5 @@
+text,label
+"Financial Meltdown: Strategies for Surviving Economic Collapse",collapse.txt
+"Exclusive Comic Book Sale: Don't Miss Out on January 13th!",comicbooks.txt
+"Tchaikovsky's Untold Story: The Mystery Behind Symphony No. 7",classicalmusic.txt
+"Coffee Addiction: When It's More Than Just a Drink",Coffee.txt