chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,36 @@
|
||||
BGE-Code-v1
|
||||
===========
|
||||
|
||||
**`BGE-Code-v1 <https://huggingface.co/BAAI/bge-code-v1>`_** is an LLM-based code embedding model that supports code retrieval, text retrieval, and multilingual retrieval. It primarily demonstrates the following capabilities:
|
||||
- Superior Code Retrieval Performance: The model demonstrates exceptional code retrieval capabilities, supporting natural language queries in both English and Chinese, as well as 20 programming languages.
|
||||
- Robust Text Retrieval Capabilities: The model maintains strong text retrieval capabilities comparable to text embedding models of similar scale.
|
||||
- Extensive Multilingual Support: BGE-Code-v1 offers comprehensive multilingual retrieval capabilities, excelling in languages such as English, Chinese, Japanese, French, and more.
|
||||
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+----------------------------------------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+===================================================================+=================+============+==============+====================================================================================================+
|
||||
| `BAAI/bge-code-v1 <https://huggingface.co/BAAI/bge-code-v1>`_ | Multilingual | 1.5B | 6.18 GB | SOTA code retrieval model, with exceptional multilingual text retrieval performance as well |
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+----------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
.. code:: python
|
||||
from FlagEmbedding import FlagLLMModel
|
||||
|
||||
queries = [
|
||||
"Delete the record with ID 4 from the 'Staff' table.",
|
||||
'Delete all records in the "Livestock" table where age is greater than 5'
|
||||
]
|
||||
documents = [
|
||||
"DELETE FROM Staff WHERE StaffID = 4;",
|
||||
"DELETE FROM Livestock WHERE age > 5;"
|
||||
]
|
||||
|
||||
model = FlagLLMModel('BAAI/bge-code-v1',
|
||||
query_instruction_format="<instruct>{}\n<query>{}",
|
||||
query_instruction_for_retrieval="Given a question in text, retrieve SQL queries that are appropriate responses to the question.",
|
||||
trust_remote_code=True,
|
||||
use_fp16=True) # Setting use_fp16 to True speeds up computation with a slight performance degradation
|
||||
embeddings_1 = model.encode_queries(queries)
|
||||
embeddings_2 = model.encode_corpus(documents)
|
||||
similarity = embeddings_1 @ embeddings_2.T
|
||||
print(similarity)
|
||||
@@ -0,0 +1,51 @@
|
||||
BGE-EN-ICL
|
||||
==========
|
||||
|
||||
BGE-EN-ICL is the new SoTA embedding model in BGE series with capabilities:
|
||||
- In-context learning ability: By providing few-shot examples in the query, it can significantly enhance the model's ability to handle new tasks.
|
||||
- Outstanding performance: The model has achieved state-of-the-art (SOTA) performance on MTEB and AIR-Bench.
|
||||
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+----------------------------------------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+===================================================================+=================+============+==============+====================================================================================================+
|
||||
| `BAAI/bge-en-icl <https://huggingface.co/BAAI/bge-en-icl>`_ | English | 7.1B | 28.5 GB | In-context learning capabilities, fully leverage the model's potential based on a few shot examples|
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+----------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import FlagICLModel
|
||||
|
||||
documents = [
|
||||
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
|
||||
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."
|
||||
]
|
||||
|
||||
examples = [
|
||||
{
|
||||
'instruct': 'Given a web search query, retrieve relevant passages that answer the query.',
|
||||
'query': 'what is a virtual interface',
|
||||
'response': "A virtual interface is a software-defined abstraction that mimics the behavior and characteristics of a physical network interface. It allows multiple logical network connections to share the same physical network interface, enabling efficient utilization of network resources. Virtual interfaces are commonly used in virtualization technologies such as virtual machines and containers to provide network connectivity without requiring dedicated hardware. They facilitate flexible network configurations and help in isolating network traffic for security and management purposes."
|
||||
},
|
||||
{
|
||||
'instruct': 'Given a web search query, retrieve relevant passages that answer the query.',
|
||||
'query': 'causes of back pain in female for a week',
|
||||
'response': "Back pain in females lasting a week can stem from various factors. Common causes include muscle strain due to lifting heavy objects or improper posture, spinal issues like herniated discs or osteoporosis, menstrual cramps causing referred pain, urinary tract infections, or pelvic inflammatory disease. Pregnancy-related changes can also contribute. Stress and lack of physical activity may exacerbate symptoms. Proper diagnosis by a healthcare professional is crucial for effective treatment and management."
|
||||
}
|
||||
]
|
||||
|
||||
queries = ["how much protein should a female eat", "summit define"]
|
||||
|
||||
model = FlagICLModel('BAAI/bge-en-icl',
|
||||
examples_for_task=examples, # set `examples_for_task=None` to use model without examples
|
||||
examples_instruction_format="<instruct>{}\n<query>{}\n<response>{}") # specify the format to use examples_for_task
|
||||
|
||||
embeddings_1 = model.encode_queries(queries)
|
||||
embeddings_2 = model.encode_corpus(documents)
|
||||
similarity = embeddings_1 @ embeddings_2.T
|
||||
|
||||
print(similarity)
|
||||
@@ -0,0 +1,124 @@
|
||||
======
|
||||
BGE-M3
|
||||
======
|
||||
|
||||
BGE-M3 is a compound and powerful embedding model distinguished for its versatility in:
|
||||
|
||||
- **Multi-Functionality**: It can simultaneously perform the three common retrieval functionalities of embedding model: dense retrieval, multi-vector retrieval, and sparse retrieval.
|
||||
- **Multi-Linguality**: It can support more than 100 working languages.
|
||||
- **Multi-Granularity**: It is able to process inputs of different granularities, spanning from short sentences to long documents of up to 8192 tokens.
|
||||
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+===================================================================+=================+============+==============+=======================================================================+
|
||||
| `BAAI/bge-m3 <https://huggingface.co/BAAI/bge-m3>`_ | Multi-Lingual | 569M | 2.27 GB | Multi-Functionality, Multi-Linguality, and Multi-Granularity |
|
||||
+-------------------------------------------------------------------+-----------------+------------+--------------+-----------------------------------------------------------------------+
|
||||
|
||||
Multi-Linguality
|
||||
================
|
||||
|
||||
BGE-M3 was trained on multiple datasets covering up to 170+ different languages.
|
||||
While the amount of training data on languages are highly unbalanced, the actual model performance on different languages will have difference.
|
||||
|
||||
For more information of datasets and evaluation results, please check out our `paper <https://arxiv.org/pdf/2402.03216s>`_ for details.
|
||||
|
||||
Multi-Granularity
|
||||
=================
|
||||
|
||||
We extend the max position to 8192, enabling the embedding of larger corpus.
|
||||
Proposing a simple but effective method: MCLS (Multiple CLS) to enhance the model's ability on long text without additional fine-tuning.
|
||||
|
||||
Multi-Functionality
|
||||
===================
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import BGEM3FlagModel
|
||||
|
||||
model = BGEM3FlagModel('BAAI/bge-m3')
|
||||
sentences_1 = ["What is BGE M3?", "Defination of BM25"]
|
||||
sentences_2 = ["BGE M3 is an embedding model supporting dense retrieval, lexical matching and multi-vector interaction.",
|
||||
"BM25 is a bag-of-words retrieval function that ranks a set of documents based on the query terms appearing in each document"]
|
||||
|
||||
Dense Retrieval
|
||||
---------------
|
||||
|
||||
Similar to BGE v1 or v1.5 models, BGE-M3 use the normalized hidden state of the special token [CLS] as the dense embedding:
|
||||
|
||||
.. math:: e_q = norm(H_q[0])
|
||||
|
||||
Next, to compute the relevance score between the query and passage:
|
||||
|
||||
.. math:: s_{dense}=f_{sim}(e_p, e_q)
|
||||
|
||||
where :math:`e_p, e_q` are the embedding vectors of passage and query, respectively.
|
||||
|
||||
:math:`f_{sim}` is the score function (such as inner product and L2 distance) for comupting two embeddings' similarity.
|
||||
|
||||
Sparse Retrieval
|
||||
----------------
|
||||
|
||||
BGE-M3 generates sparce embeddings by adding a linear layer and a ReLU activation function following the hidden states:
|
||||
|
||||
.. math:: w_{qt} = \text{Relu}(W_{lex}^T H_q [i])
|
||||
|
||||
where :math:`W_{lex}` representes the weights of linear layer and :math:`H_q[i]` is the encoder's output of the :math:`i^{th}` token.
|
||||
|
||||
Based on the tokens' weights of query and passage, the relevance score between them is computed by the joint importance of the co-existed terms within the query and passage:
|
||||
|
||||
.. math:: s_{lex} = \sum_{t\in q\cap p}(w_{qt} * w_{pt})
|
||||
|
||||
where :math:`w_{qt}, w_{pt}` are the importance weights of each co-existed term :math:`t` in query and passage, respectively.
|
||||
|
||||
Multi-Vector
|
||||
------------
|
||||
|
||||
The multi-vector method utilizes the entire output embeddings for the representation of query :math:`E_q` and passage :math:`E_p`.
|
||||
|
||||
.. math::
|
||||
|
||||
E_q = norm(W_{mul}^T H_q)
|
||||
|
||||
E_p = norm(W_{mul}^T H_p)
|
||||
|
||||
where :math:`W_{mul}` is the learnable projection matrix.
|
||||
|
||||
Following ColBert, BGE-M3 use late-interaction to compute the fine-grained relevance score:
|
||||
|
||||
.. math:: s_{mul}=\frac{1}{N}\sum_{i=1}^N\max_{j=1}^M E_q[i]\cdot E_p^T[j]
|
||||
|
||||
where :math:`E_q, E_p` are the entire output embeddings of query and passage, respectively.
|
||||
|
||||
This is a summation of average of maximum similarity of each :math:`v\in E_q` with vectors in :math:`E_p`.
|
||||
|
||||
Hybrid Ranking
|
||||
--------------
|
||||
|
||||
BGE-M3's multi-functionality gives the possibility of hybrid ranking to improve retrieval.
|
||||
Firstly, due to the heavy cost of multi-vector method, we can retrieve the candidate results by either of the dense or sparse method.
|
||||
Then, to get the final result, we can rerank the candidates based on the integrated relevance score:
|
||||
|
||||
.. math:: s_{rank} = w_1\cdot s_{dense}+w_2\cdot s_{lex} + w_3\cdot s_{mul}
|
||||
|
||||
where the values chosen for :math:`w_1`, :math:`w_2` and :math:`w_3` varies depending on the downstream scenario.
|
||||
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import BGEM3FlagModel
|
||||
|
||||
model = BGEM3FlagModel('BAAI/bge-m3')
|
||||
|
||||
sentences_1 = ["What is BGE M3?", "Defination of BM25"]
|
||||
|
||||
output = model.encode(sentences_1, return_dense=True, return_sparse=True, return_colbert_vecs=True)
|
||||
dense, sparse, multiv = output['dense_vecs'], output['lexical_weights'], output['colbert_vecs']
|
||||
|
||||
Useful Links:
|
||||
|
||||
`API <../API/inference/embedder/encoder_only/M3Embedder>`_
|
||||
`Tutorial <https://github.com/FlagOpen/FlagEmbedding/blob/master/Tutorials/1_Embedding/1.2.4_BGE-M3.ipynb>`_
|
||||
`Example <https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/inference/embedder/encoder_only>`_
|
||||
@@ -0,0 +1,37 @@
|
||||
BGE-Reranker
|
||||
============
|
||||
|
||||
Different from embedding model, reranker, or cross-encoder uses question and document as input and directly output similarity instead of embedding.
|
||||
To balance the accuracy and time cost, cross-encoder is widely used to re-rank top-k documents retrieved by other simple models.
|
||||
For examples, use a bge embedding model to first retrieve top 100 relevant documents, and then use bge reranker to re-rank the top 100 document to get the final top-3 results.
|
||||
|
||||
The first series of BGE-Reranker contains two models, large and base.
|
||||
|
||||
+-------------------------------------------------------------------------------+-----------------------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+===============================================================================+=======================+============+==============+=======================================================================+
|
||||
| `BAAI/bge-reranker-large <https://huggingface.co/BAAI/bge-reranker-large>`_ | English & Chinese | 560M | 2.24 GB | Larger reranker model, easy to deploy with better inference |
|
||||
+-------------------------------------------------------------------------------+-----------------------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-reranker-base <https://huggingface.co/BAAI/bge-reranker-base>`_ | English & Chinese | 278M | 1.11 GB | Lightweight reranker model, easy to deploy with fast inference |
|
||||
+-------------------------------------------------------------------------------+-----------------------+------------+--------------+-----------------------------------------------------------------------+
|
||||
|
||||
bge-reranker-large and bge-reranker-base used `XLM-RoBERTa-Large <https://huggingface.co/FacebookAI/xlm-roberta-large>`_ and `XLM-RoBERTa-Base <https://huggingface.co/FacebookAI/xlm-roberta-base>`_ respectively as the base model.
|
||||
They were trained on high quality English and Chinese data, and acheived State-of-The-Art performance in the level of same size models at the time released.
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import FlagReranker
|
||||
|
||||
reranker = FlagReranker(
|
||||
'BAAI/bge-reranker-base',
|
||||
query_max_length=256,
|
||||
use_fp16=True,
|
||||
devices=['cuda:1'],
|
||||
)
|
||||
|
||||
score = reranker.compute_score(['I am happy to help', 'Assisting you is my pleasure'])
|
||||
print(score)
|
||||
@@ -0,0 +1,90 @@
|
||||
BGE-Reranker-v2
|
||||
===============
|
||||
|
||||
+------------------------------------------------------------------------------------------------------------------+-----------------------+-------------+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+==================================================================================================================+=======================+=============+==============+=========================================================================================================================================================+
|
||||
| `BAAI/bge-reranker-v2-m3 <https://huggingface.co/BAAI/bge-reranker-v2-m3>`_ | Multilingual | 568M | 2.27 GB | Lightweight reranker model, possesses strong multilingual capabilities, easy to deploy, with fast inference. |
|
||||
+------------------------------------------------------------------------------------------------------------------+-----------------------+-------------+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| `BAAI/bge-reranker-v2-gemma <https://huggingface.co/BAAI/bge-reranker-v2-gemma>`_ | Multilingual | 2.51B | 10 GB | Suitable for multilingual contexts, performs well in both English proficiency and multilingual capabilities. |
|
||||
+------------------------------------------------------------------------------------------------------------------+-----------------------+-------------+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| `BAAI/bge-reranker-v2-minicpm-layerwise <https://huggingface.co/BAAI/bge-reranker-v2-minicpm-layerwise>`_ | Multilingual | 2.72B | 10.9 GB | Suitable for multilingual contexts, allows freedom to select layers for output, facilitating accelerated inference. |
|
||||
+------------------------------------------------------------------------------------------------------------------+-----------------------+-------------+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| `BAAI/bge-reranker-v2.5-gemma2-lightweight <https://huggingface.co/BAAI/bge-reranker-v2.5-gemma2-lightweight>`_ | Multilingual | 2.72B | 10.9 GB | Suitable for multilingual contexts, allows freedom to select layers, compress ratio and compress layers for output, facilitating accelerated inference. |
|
||||
+------------------------------------------------------------------------------------------------------------------+-----------------------+-------------+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
.. tip::
|
||||
|
||||
You can select the model according your senario and resource:
|
||||
|
||||
- For multilingual, utilize :code:`BAAI/bge-reranker-v2-m3`, :code:`BAAI/bge-reranker-v2-gemma` and :code:`BAAI/bge-reranker-v2.5-gemma2-lightweight`.
|
||||
- For Chinese or English, utilize :code:`BAAI/bge-reranker-v2-m3` and :code:`BAAI/bge-reranker-v2-minicpm-layerwise`.
|
||||
- For efficiency, utilize :code:`BAAI/bge-reranker-v2-m3` and the low layer of :code:`BAAI/bge-reranker-v2-minicpm-layerwise`.
|
||||
- For better performance, recommand :code:`BAAI/bge-reranker-v2-minicpm-layerwise` and :code:`BAAI/bge-reranker-v2-gemma`.
|
||||
|
||||
Make sure always test on your real use case and choose the one with best speed-quality balance!
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
**bge-reranker-v2-m3**
|
||||
|
||||
Use :code:`bge-reranker-v2-m3` in the same way as bge-reranker-base and bge-reranker-large.
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import FlagReranker
|
||||
|
||||
# Setting use_fp16 to True speeds up computation with a slight performance degradation
|
||||
reranker = FlagReranker('BAAI/bge-reranker-v2-m3', use_fp16=True)
|
||||
|
||||
score = reranker.compute_score(['query', 'passage'])
|
||||
# or set "normalize=True" to apply a sigmoid function to the score for 0-1 range
|
||||
score = reranker.compute_score(['query', 'passage'], normalize=True)
|
||||
|
||||
print(score)
|
||||
|
||||
**bge-reranker-v2-gemma**
|
||||
|
||||
Use the :code:`FlagLLMReranker` class for bge-reranker-v2-gemma.
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import FlagLLMReranker
|
||||
|
||||
# Setting use_fp16 to True speeds up computation with a slight performance degradation
|
||||
reranker = FlagLLMReranker('BAAI/bge-reranker-v2-gemma', use_fp16=True)
|
||||
|
||||
score = reranker.compute_score(['query', 'passage'])
|
||||
print(score)
|
||||
|
||||
**bge-reranker-v2-minicpm-layerwise**
|
||||
|
||||
Use the :code:`LayerWiseFlagLLMReranker` class for bge-reranker-v2-minicpm-layerwise.
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import LayerWiseFlagLLMReranker
|
||||
|
||||
# Setting use_fp16 to True speeds up computation with a slight performance degradation
|
||||
reranker = LayerWiseFlagLLMReranker('BAAI/bge-reranker-v2-minicpm-layerwise', use_fp16=True)
|
||||
|
||||
# Adjusting 'cutoff_layers' to pick which layers are used for computing the score.
|
||||
score = reranker.compute_score(['query', 'passage'], cutoff_layers=[28])
|
||||
print(score)
|
||||
|
||||
**bge-reranker-v2.5-gemma2-lightweight**
|
||||
|
||||
Use the :code:`LightWeightFlagLLMReranker` class for bge-reranker-v2.5-gemma2-lightweight.
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import LightWeightFlagLLMReranker
|
||||
|
||||
# Setting use_fp16 to True speeds up computation with a slight performance degradation
|
||||
reranker = LightWeightFlagLLMReranker('BAAI/bge-reranker-v2.5-gemma2-lightweight', use_fp16=True)
|
||||
|
||||
# Adjusting 'cutoff_layers' to pick which layers are used for computing the score.
|
||||
score = reranker.compute_score(['query', 'passage'], cutoff_layers=[28], compress_ratio=2, compress_layer=[24, 40])
|
||||
print(score)
|
||||
@@ -0,0 +1,99 @@
|
||||
BGE v1 & v1.5
|
||||
=============
|
||||
|
||||
BGE v1 and v1.5 are series of encoder only models base on BERT. They achieved best performance among the models of the same size at the time of release.
|
||||
|
||||
BGE
|
||||
---
|
||||
|
||||
The first group of BGE models was released in Aug 2023. The :code:`bge-large-en` and :code:`bge-large-zh` ranked 1st on MTEB and
|
||||
C-MTEB benchmarks at the time released.
|
||||
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+===================================================================+===========+============+==============+=======================================================================+
|
||||
| `BAAI/bge-large-en <https://huggingface.co/BAAI/bge-large-en>`_ | English | 335M | 1.34 GB | Embedding Model which map text into vector |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-base-en <https://huggingface.co/BAAI/bge-base-en>`_ | English | 109M | 438 MB | a base-scale model but with similar ability to `BAAI/bge-large-en` |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-small-en <https://huggingface.co/BAAI/bge-small-en>`_ | English | 33.4M | 133 MB | a small-scale model but with competitive performance |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-large-zh <https://huggingface.co/BAAI/bge-large-zh>`_ | Chinese | 326M | 1.3 GB | Embedding Model which map text into vector |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-base-zh <https://huggingface.co/BAAI/bge-base-zh>`_ | Chinese | 102M | 409 MB | a base-scale model but with similar ability to `BAAI/bge-large-zh` |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-small-zh <https://huggingface.co/BAAI/bge-small-zh>`_ | Chinese | 24M | 95.8 MB | a small-scale model but with competitive performance |
|
||||
+-------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
|
||||
BGE-v1.5
|
||||
--------
|
||||
|
||||
Then to enhance its retrieval ability without instruction and alleviate the issue of the similarity distribution, :code:`bge-*-v1.5` models
|
||||
were released in Sep 2023. They are still the most popular embedding models that balanced well between embedding quality and model sizes.
|
||||
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+--------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+=============================================================================+===========+============+==============+==============+
|
||||
| `BAAI/bge-large-en-v1.5 <https://huggingface.co/BAAI/bge-large-en-v1.5>`_ | English | 335M | 1.34 GB | version 1.5 |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+ with more +
|
||||
| `BAAI/bge-base-en-v1.5 <https://huggingface.co/BAAI/bge-base-en-v1.5>`_ | English | 109M | 438 MB | reasonable |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+ similarity +
|
||||
| `BAAI/bge-small-en-v1.5 <https://huggingface.co/BAAI/bge-small-en-v1.5>`_ | English | 33.4M | 133 MB | distribution |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+ and better +
|
||||
| `BAAI/bge-large-zh-v1.5 <https://huggingface.co/BAAI/bge-large-zh-v1.5>`_ | Chinese | 326M | 1.3 GB | performance |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+ +
|
||||
| `BAAI/bge-base-zh-v1.5 <https://huggingface.co/BAAI/bge-base-zh-v1.5>`_ | Chinese | 102M | 409 MB | |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+ +
|
||||
| `BAAI/bge-small-zh-v1.5 <https://huggingface.co/BAAI/bge-small-zh-v1.5>`_ | Chinese | 24M | 95.8 MB | |
|
||||
+-----------------------------------------------------------------------------+-----------+------------+--------------+--------------+
|
||||
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
To use BGE v1 or v1.5 model for inference, load model through
|
||||
|
||||
.. code:: python
|
||||
|
||||
from FlagEmbedding import FlagModel
|
||||
|
||||
model = FlagModel('BAAI/bge-base-en-v1.5')
|
||||
|
||||
sentences = ["Hello world", "I am inevitable"]
|
||||
embeddings = model.encode(sentences)
|
||||
|
||||
.. tip::
|
||||
|
||||
For simple tasks that only encode a few sentences like above, it's faster to use CPU or a single GPU instead of multi-GPUs
|
||||
|
||||
To use CPU:
|
||||
|
||||
.. code:: python
|
||||
|
||||
# make no GPU visible
|
||||
import os
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = ''
|
||||
|
||||
# or claim the devices during initialize the model
|
||||
model = FlagModel('BAAI/bge-base-en-v1.5', devices='cpu')
|
||||
|
||||
To use a single GPU:
|
||||
|
||||
.. code:: python
|
||||
|
||||
# select one sigle card to be visible
|
||||
import os
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
|
||||
|
||||
# or claim the devices during initialize the model
|
||||
model = FlagModel('BAAI/bge-base-en-v1.5', devices=0)
|
||||
|
||||
|
|
||||
|
||||
Useful Links:
|
||||
|
||||
`API <../API/inference/embedder/encoder_only/BaseEmbedder>`_
|
||||
|
||||
`Tutorial <https://github.com/FlagOpen/FlagEmbedding/blob/master/Tutorials/1_Embedding/1.2.3_BGE_v1%261.5.ipynb>`_
|
||||
|
||||
`Example <https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/inference/embedder/encoder_only>`_
|
||||
@@ -0,0 +1,158 @@
|
||||
BGE-VL
|
||||
======
|
||||
|
||||
BGE-VL is a series of multimodel retrieval models training on `MegaPairs <https://github.com/VectorSpaceLab/MegaPairs>`_
|
||||
|
||||
BGE-VL contains light weight CLIP based models as well as more powerful LLAVA-NeXT based MLLM models:
|
||||
|
||||
+----------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| Model | Language | Parameters | Model Size | Description |
|
||||
+======================================================================+===========+============+==============+=======================================================================+
|
||||
| `BAAI/bge-vl-base <https://huggingface.co/BAAI/BGE-VL-base>`_ | English | 150M | 299 MB | Light weight multimodel embedder among image and text |
|
||||
+----------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-vl-large <https://huggingface.co/BAAI/BGE-VL-large>`_ | English | 428M | 855 MB | Large scale multimodel embedder among image and text |
|
||||
+----------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-vl-MLLM-S1 <https://huggingface.co/BAAI/BGE-VL-MLLM-S1>`_ | English | 7.57B | 15.14 GB | SOTA in composed image retrieval, trained on MegaPairs dataset |
|
||||
+----------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/bge-vl-MLLM-S2 <https://huggingface.co/BAAI/BGE-VL-MLLM-S2>`_ | English | 7.57B | 15.14 GB | Finetune BGE-VL-MLLM-S1 with one epoch on MMEB training set |
|
||||
+----------------------------------------------------------------------+-----------+------------+--------------+-----------------------------------------------------------------------+
|
||||
| `BAAI/BGE-VL-v1.5-zs <https://huggingface.co/BAAI/BGE-VL-v1.5-zs>`_ | English | 7.57B | 15.14 GB | Better multi-modal retrieval model with performs well in all kinds of tasks |
|
||||
| `BAAI/BGE-VL-v1.5-mmeb <https://huggingface.co/BAAI/BGE-VL-v1.5-mmeb>`_ | English | 7.57B | 15.14 GB | Better multi-modal retrieval model, additionally fine-tuned on MMEB training set |
|
||||
|
||||
|
||||
BGE-VL-CLIP
|
||||
-----------
|
||||
|
||||
The base and large model are trained based on CLIP-vit-base-patch16 and CLIP-vit-large-patch14.
|
||||
For composed image-text data, the model directly use score-fusion to sum up the outputs of visual encoder and text encoder and get the final embedding.
|
||||
|
||||
.. tip::
|
||||
|
||||
Our code works well on transformers==4.45.2, and we recommend using this version.
|
||||
|
||||
You can easily use BGE-VL-CLIP models based on transformers:
|
||||
|
||||
.. code:: python
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
MODEL_NAME = "BAAI/BGE-VL-base" # or "BAAI/BGE-VL-large"
|
||||
model = AutoModel.from_pretrained(MODEL_NAME, trust_remote_code=True) # You must set trust_remote_code=True
|
||||
model.set_processor(MODEL_NAME)
|
||||
model.eval()
|
||||
|
||||
with torch.no_grad():
|
||||
query = model.encode(
|
||||
images = "./assets/cir_query.png",
|
||||
text = "Make the background dark, as if the camera has taken the photo at night"
|
||||
)
|
||||
candidates = model.encode(
|
||||
images = ["./assets/cir_candi_1.png", "./assets/cir_candi_2.png"]
|
||||
)
|
||||
|
||||
scores = query @ candidates.T
|
||||
print(scores)
|
||||
|
||||
|
||||
BGE-VL-MLLM
|
||||
-----------
|
||||
|
||||
The multimodal large language models (MLLMs) incorporate a visual encoder, typically based on a vision transformer, into a large language model (LLM).
|
||||
This integration allows image tokens to be directly processed by the LLM.
|
||||
Consequently, MLLMs can effectively handle diverse multimodal inputs by converting any type of input into a sequence of tokens.
|
||||
|
||||
BGE-VL-MLLM builds upon the LLaVA1.6. In both training and inference stages, MMRet uses task-specific instructions for query inputs to improve generalization, aligning
|
||||
with standard practices in LLM-based embedding models.
|
||||
A typical multimodal query input is structured as follows:
|
||||
|
||||
.. math::
|
||||
|
||||
⟨\text{instruct}⟩{\{task\_ inst\}} \space⟨\text{query}⟩\{q_t\} \{q_i\}\space[\text{EOS}]
|
||||
|
||||
where :math:`{task_inst}` represents the task-specific instruction, :math:`{qt}` denotes the input query text, and
|
||||
:math:`{qi}` is the input query image.
|
||||
The normalized last hidden state of the [EOS] token in the MLLM is used as the embedding of any given input sequence.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
from PIL import Image
|
||||
|
||||
MODEL_NAME= "BAAI/BGE-VL-MLLM-S1"
|
||||
model = AutoModel.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
model.eval()
|
||||
model.cuda()
|
||||
|
||||
with torch.no_grad():
|
||||
model.set_processor(MODEL_NAME)
|
||||
|
||||
query_inputs = model.data_process(
|
||||
text="Make the background dark, as if the camera has taken the photo at night",
|
||||
images="./assets/cir_query.png",
|
||||
q_or_c="q",
|
||||
task_instruction="Retrieve the target image that best meets the combined criteria by using both the provided image and the image retrieval instructions: "
|
||||
)
|
||||
candidate_inputs = model.data_process(
|
||||
images=["./assets/cir_candi_1.png", "./assets/cir_candi_2.png"],
|
||||
q_or_c="c",
|
||||
)
|
||||
|
||||
query_embs = model(**query_inputs, output_hidden_states=True)[:, -1, :]
|
||||
candi_embs = model(**candidate_inputs, output_hidden_states=True)[:, -1, :]
|
||||
|
||||
query_embs = torch.nn.functional.normalize(query_embs, dim=-1)
|
||||
candi_embs = torch.nn.functional.normalize(candi_embs, dim=-1)
|
||||
|
||||
scores = torch.matmul(query_embs, candi_embs.T)
|
||||
print(scores)
|
||||
|
||||
|
||||
BGE-VL-v1.5
|
||||
-----------
|
||||
|
||||
BGE-VL-v1.5 series is the updated version of BGE-VL, bringing better performance on both retrieval and multi-modal understanding. The models were trained on 30M MegaPairs data and extra 10M natural and synthetic data.
|
||||
|
||||
`bge-vl-v1.5-zs` is a zero-shot model, only trained on the data mentioned above. `bge-vl-v1.5-mmeb` is the fine-tuned version on MMEB training set.
|
||||
|
||||
|
||||
.. code:: python
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
from PIL import Image
|
||||
|
||||
MODEL_NAME= "BAAI/BGE-VL-v1.5-mmeb" # "BAAI/BGE-VL-v1.5-zs"
|
||||
|
||||
model = AutoModel.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
model.eval()
|
||||
model.cuda()
|
||||
|
||||
with torch.no_grad():
|
||||
model.set_processor(MODEL_NAME)
|
||||
|
||||
query_inputs = model.data_process(
|
||||
text="Make the background dark, as if the camera has taken the photo at night",
|
||||
images="../../imgs/cir_query.png",
|
||||
q_or_c="q",
|
||||
task_instruction="Retrieve the target image that best meets the combined criteria by using both the provided image and the image retrieval instructions: "
|
||||
)
|
||||
|
||||
candidate_inputs = model.data_process(
|
||||
images=["../../imgs/cir_candi_1.png", "../../imgs/cir_candi_2.png"],
|
||||
q_or_c="c",
|
||||
)
|
||||
|
||||
query_embs = model(**query_inputs, output_hidden_states=True)[:, -1, :]
|
||||
candi_embs = model(**candidate_inputs, output_hidden_states=True)[:, -1, :]
|
||||
|
||||
query_embs = torch.nn.functional.normalize(query_embs, dim=-1)
|
||||
candi_embs = torch.nn.functional.normalize(candi_embs, dim=-1)
|
||||
|
||||
scores = torch.matmul(query_embs, candi_embs.T)
|
||||
print(scores)
|
||||
|
||||
|
||||
|
||||
For more details, check out the repo of `MegaPairs <https://github.com/VectorSpaceLab/MegaPairs>`_
|
||||
@@ -0,0 +1,25 @@
|
||||
BGE
|
||||
===
|
||||
|
||||
.. figure:: ../_static/img/bge_logo.jpeg
|
||||
:width: 250
|
||||
:align: center
|
||||
|
||||
**BGE** stands for **BAAI General Embeddings**, which is a series of embedding models released by BAAI.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Embedder
|
||||
|
||||
bge_v1_v1.5
|
||||
bge_m3
|
||||
bge_icl
|
||||
bge_vl
|
||||
bge_code
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Reranker
|
||||
|
||||
bge_reranker
|
||||
bge_reranker_v2
|
||||
Reference in New Issue
Block a user