commit fd87f9797d0fae1a314cd11bab38d1bfd654aaaa Author: wehub-resource-sync Date: Mon Jul 13 12:37:02 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.env b/.env new file mode 100644 index 0000000..1b41a00 --- /dev/null +++ b/.env @@ -0,0 +1,12 @@ +fastapi_host=localhost +fastapi_port=8000 + +milvus_host=localhost +milvus_port=19530 +milvus_collection_name=knowledge_collection + +openai_base_url=[YOUR_BASE_URL] +openai_api_key=[YOUR_API_KEY] +openai_llm_model_name=gpt-4o-mini + +text_embeddings_model_path=jinaai/jina-embeddings-v3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..49ee88a --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# Knowledge_QA_RAG + +## Overview +本系统是一种基于 RAG 的知识库问答系统简单示例，采用前后端分离的架构设计，融合了多种技术和方法。具体详情可参考：[https://blog.csdn.net/weixin_47936614/article/details/143932997](https://blog.csdn.net/weixin_47936614/article/details/143932997) + +## Milvus Installation +本系统是在 Windows 11 上进行部署和运行的，关于milvus向量数据库的安装和启动可以参考以下步骤： + +1.勾选 `适用于Linux的Windows子系统` 和 `虚拟机平台` + +![09afdd4c8869124fd606715f57cae4cb](https://github.com/user-attachments/assets/67c0e85d-cdbc-4a9e-b724-df3f4ea1799a) + + +2.点击 `确定` 并重新启动计算机 + +3.以管理员身份打开命令提示符，输入以下命令安装WSL： +- 在 PowerShell 中设置 WSL 2 为默认版本： +``` +wsl --set-default-version 2 +``` +- 更新 WSL 内核，使用国内网络建议添加`--web-download`： +``` +wsl --update --web-download +``` +安装成功后的结果如下： + +![abe944eb639649fae89175ed09af9fdd](https://github.com/user-attachments/assets/74fc0c97-cf3c-47c6-b3a2-240ba97a44ff) + +4.下载安装docker-desktop +进入官网下载对应的版本安装即可，官网链接：[https://www.docker.com/products/docker-desktop/](https://www.docker.com/products/docker-desktop/) + +5.验证是否安装成功： +``` +docker --version +docker-compose --version +``` + +![a263610663c19d4046f28773a1126369](https://github.com/user-attachments/assets/1f8e801b-b2db-4644-83d7-5da10f4e5764) + +6.milvus向量数据库安装 +- 创建milvus文件夹，并在该文件夹下创建多个子文件夹，如下： + +![image](https://github.com/user-attachments/assets/27599318-2a9a-4358-a1ec-226ca8363921) + +- 下载milvus +进入下载页面：[https://github.com/milvus-io/milvus/releases](https://github.com/milvus-io/milvus/releases) +选择milvus版本及其对应的yml文件，点击下载即可，如下： + +![image](https://github.com/user-attachments/assets/5b8804ca-5c6b-4102-82ac-d47eee31c77b) + +- 将下载好的 `milvus-standalone-docker-compose.yml` 重命名为 `docker-compose.yml` ，并放入milvus文件中，如下： + +![bf7215948670e96f9b50f90a67463950](https://github.com/user-attachments/assets/177d4889-49ff-4af4-8515-7722dc65a504) + +- 在milvus文件夹中启动cmd命令，输入以下命令： +``` +docker compose up -d +docker compose ps +docker port milvus-standalone 19530/tcp +``` +运行结果如下： + +![694be819cd99abc328136139f2e8e8d2](https://github.com/user-attachments/assets/2506153f-9625-42dd-b09c-db01e5ded5d6) + +![1a3086ad4519524717991dffa47cce29](https://github.com/user-attachments/assets/a8e9a22c-2b1c-4c95-9e15-fe8d6cb0c3ee) + +至此，milvus数据库部署成功！ + +7.Attu图形化界面安装 +下载地址：[https://github.com/zilliztech/attu/releases](https://github.com/zilliztech/attu/releases) +选择对应的版本直接下载安装即可： + +![image](https://github.com/user-attachments/assets/b4e96c46-e5e8-42e0-aae9-670008d3ff53) + +## Environment Installation +``` +conda create --name rag-env python=3.10 +cd Knowledge_QA_RAG +conda activate rag-env +pip install -r requirements.txt +``` +## Quick Start +1.启动milvus数据库 + +![image](https://github.com/user-attachments/assets/4d2a37ad-3f74-4b72-969c-685c6b023e96) + +2.启动系统服务 +``` +python main.py +``` +或 +``` +uvicorn server:app --reload --host 127.0.0.1 --port 8000 +``` + +3.访问页面: 在浏览器中输入下面url地址即可访问 +``` +http://127.0.0.1:8000/ +``` + +## 项目演示示例： +📺 [点击观看项目演示视频](https://www.bilibili.com/video/BV1a49KBqE7n/) diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..e6df79f --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`AI-Meet/Knowledge_QA_RAG` +- 原始仓库：https://github.com/AI-Meet/Knowledge_QA_RAG +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/config.py b/config.py new file mode 100644 index 0000000..6188478 --- /dev/null +++ b/config.py @@ -0,0 +1,36 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/10/31 14:30 +# @blog: https://blog.csdn.net/weixin_47936614 + +import os + +import torch +from dotenv import load_dotenv + +# 从 .env 文件加载环境变量 +load_dotenv() + + +class RagConfig: + # FastAPI 服务配置 + fastapi_host = os.getenv("fastapi_host") + fastapi_port = os.getenv("fastapi_port") + + # Milvus 配置 + milvus_host = os.getenv("milvus_host") + milvus_port = os.getenv("milvus_port") + milvus_collection_name = os.getenv("milvus_collection_name") + + # OpenAI 设置 + base_url = os.getenv("openai_base_url") + api_key = os.getenv("openai_api_key") + llm_model_name = os.getenv("openai_llm_model_name") + + # 嵌入模型路径 + text_embeddings_model_path = os.getenv("text_embeddings_model_path") + + # 设备设置 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + diff --git a/file_process.py b/file_process.py new file mode 100644 index 0000000..225151e --- /dev/null +++ b/file_process.py @@ -0,0 +1,45 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/10/31 13:59 +# @blog: https://blog.csdn.net/weixin_47936614 + +import os + +from langchain_community.document_loaders import TextLoader, UnstructuredWordDocumentLoader, PyPDFLoader + +from text_utils.text_split import RagTextSplitter + + +class RagFileProcessor(object): + def __init__(self, chunk_size: int = 512): + self.text_splitter = RagTextSplitter(chunk_size=chunk_size) + + def file_process(self, file_path: str): + if not os.path.exists(file_path): + raise FileNotFoundError(f"文件 {file_path} 不存在！") + + if file_path.lower().endswith(".txt"): + txt_loader = TextLoader(file_path, autodetect_encoding=True) + txt_docs = txt_loader.load_and_split(text_splitter=self.text_splitter) + return txt_docs + elif file_path.lower().endswith(".docx"): + docx_loader = UnstructuredWordDocumentLoader(file_path, mode="single") + docx_docs = docx_loader.load_and_split(text_splitter=self.text_splitter) + return docx_docs + elif file_path.lower().endswith(".pdf"): + pdf_loader = PyPDFLoader(file_path) + pdf_docs = pdf_loader.load_and_split(text_splitter=self.text_splitter) + return pdf_docs + else: + raise TypeError("文件类型不支持，目前仅支持：txt/docx/pdf") + + def get_data(self, file_path: str): + docs = self.file_process(file_path) + passage_docs = [doc.page_content.strip() for doc in docs] + file_name = {"source": os.path.basename(docs[0].metadata['source'])} + ids = [str(i) for i in range(len(passage_docs))] # 确保每个文档有唯一的 ID + meta_datas = [file_name for _ in range(len(passage_docs))] # 定义元数据信息，包括文件名等 + dict_data = {"texts": passage_docs, "ids": ids, "meta_datas": meta_datas} + return dict_data + diff --git a/main.py b/main.py new file mode 100644 index 0000000..97cd03b --- /dev/null +++ b/main.py @@ -0,0 +1,18 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/11/26 10:14 +# @blog: https://blog.csdn.net/weixin_47936614 + +import uvicorn + +from milvus_vector import config + +if __name__ == '__main__': + # 主函数启动方式 + uvicorn.run( + "server:app", # 指定模块名和应用实例 + host=config.fastapi_host, # 本地地址 + port=int(config.fastapi_port), # 端口 + reload=True # 开启热重载 + ) diff --git a/milvus_vector.py b/milvus_vector.py new file mode 100644 index 0000000..c87d982 --- /dev/null +++ b/milvus_vector.py @@ -0,0 +1,37 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/11/22 18:53 +# @blog: https://blog.csdn.net/weixin_47936614 + +from langchain_milvus import Milvus + +from config import RagConfig +from text_utils.text_embeddings import RagTextEmbeddings + +# 加载配置 +config = RagConfig() + +# 配置索引参数和搜索参数 +index_params = { + "index_type": "IVF_FLAT", + "metric_type": "L2", + "params": {"nlist": 100} +} + +search_params = { + "metric_type": "L2", + "params": {"nprobe": 10} +} + +# 初始化 Milvus 向量存储 +vector_store = Milvus( + embedding_function=RagTextEmbeddings(embed_model_path=config.text_embeddings_model_path, + batch_size=32, + device=config.device), + collection_name=config.milvus_collection_name, + consistency_level="Bounded", + connection_args={"host": config.milvus_host, "port": config.milvus_port}, + index_params=index_params, + search_params=search_params +) diff --git a/protocol/__init__.py b/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/protocol/mode.py b/protocol/mode.py new file mode 100644 index 0000000..8b745f8 --- /dev/null +++ b/protocol/mode.py @@ -0,0 +1,12 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/11/20 16:33 +# @blog: https://blog.csdn.net/weixin_47936614 + +from pydantic import BaseModel + + +class ChatRequest(BaseModel): + question: str + diff --git a/protocol/prompts.py b/protocol/prompts.py new file mode 100644 index 0000000..71b73d4 --- /dev/null +++ b/protocol/prompts.py @@ -0,0 +1,13 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/11/20 16:17 +# @blog: https://blog.csdn.net/weixin_47936614 + +prompt_template = """ +Use the following pieces of context to answer the question at the end. +If you don't know the answer, just say "sorry, I can't answer this question.", don't try to make up an answer. +{context} +Question: {question} +Answer in Chinese: +""" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8407f52 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,153 @@ +accelerate==1.1.1 +aiohappyeyeballs==2.4.3 +aiohttp==3.10.10 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.6.2.post1 +async-timeout==4.0.3 +attrs==24.2.0 +backoff==2.2.1 +beautifulsoup4==4.12.3 +cbor==1.0.0 +certifi==2024.8.30 +cffi==1.17.1 +chardet==5.2.0 +charset-normalizer==3.4.0 +click==8.1.7 +colorama==0.4.6 +cryptography==43.0.3 +dataclasses-json==0.6.7 +datasets==2.19.0 +dill==0.3.8 +distro==1.9.0 +dnspython==2.7.0 +einops==0.8.0 +email_validator==2.2.0 +emoji==2.14.0 +environs==9.5.0 +eval_type_backport==0.2.0 +exceptiongroup==1.2.2 +fastapi==0.115.5 +fastapi-cli==0.0.5 +filelock==3.16.1 +filetype==1.2.0 +FlagEmbedding==1.3.2 +frozenlist==1.5.0 +fsspec==2024.3.1 +greenlet==3.1.1 +grpcio==1.67.1 +h11==0.14.0 +html5lib==1.1 +httpcore==1.0.6 +httptools==0.6.4 +httpx==0.27.2 +httpx-sse==0.4.0 +huggingface-hub==0.26.2 +idna==3.10 +ijson==3.3.0 +inscriptis==2.5.0 +ir_datasets==0.5.9 +itsdangerous==2.2.0 +Jinja2==3.1.4 +jiter==0.7.0 +joblib==1.4.2 +jsonpatch==1.33 +jsonpath-python==1.0.6 +jsonpointer==3.0.0 +langchain==0.3.7 +langchain-community==0.3.7 +langchain-core==0.3.18 +langchain-milvus==0.1.7 +langchain-openai==0.2.6 +langchain-text-splitters==0.3.0 +langdetect==1.0.9 +langsmith==0.1.137 +lxml==5.3.0 +lz4==4.3.3 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +marshmallow==3.23.0 +mdurl==0.1.2 +mpmath==1.3.0 +multidict==6.1.0 +multiprocess==0.70.16 +mypy-extensions==1.0.0 +nest-asyncio==1.6.0 +networkx==3.4.2 +nltk==3.9.1 +numpy==1.26.4 +olefile==0.47 +openai==1.54.3 +orjson==3.10.10 +packaging==24.1 +pandas==2.2.3 +peft==0.13.2 +pillow==11.0.0 +propcache==0.2.0 +protobuf==5.28.3 +psutil==6.1.0 +pyarrow==18.0.0 +pyarrow-hotfix==0.6 +pycparser==2.22 +pydantic==2.9.2 +pydantic-extra-types==2.10.0 +pydantic-settings==2.6.0 +pydantic_core==2.23.4 +Pygments==2.18.0 +pymilvus==2.4.8 +pypdf==5.1.0 +python-dateutil==2.8.2 +python-docx==1.1.2 +python-dotenv==1.0.1 +python-iso639==2024.10.22 +python-magic==0.4.27 +python-multipart==0.0.17 +python-oxmsg==0.0.1 +pytz==2024.2 +PyYAML==6.0.2 +RapidFuzz==3.10.1 +regex==2024.9.11 +requests==2.32.3 +requests-toolbelt==1.0.0 +rich==13.9.4 +safetensors==0.4.5 +scikit-learn==1.5.2 +scipy==1.14.1 +sentence-transformers==3.1.0 +sentencepiece==0.2.0 +shellingham==1.5.4 +six==1.16.0 +sniffio==1.3.1 +soupsieve==2.6 +SQLAlchemy==2.0.35 +starlette==0.41.3 +sympy==1.13.1 +tenacity==8.5.0 +threadpoolctl==3.5.0 +tiktoken==0.8.0 +tokenizers==0.19.1 +torch==2.5.1 +torchaudio==2.5.1 +torchvision==0.20.1 +tqdm==4.66.6 +transformers==4.44.2 +trec-car-tools==2.6 +typer==0.13.1 +typing-inspect==0.9.0 +typing_extensions==4.12.2 +tzdata==2024.2 +ujson==5.10.0 +unlzw3==0.2.2 +unstructured==0.16.3 +unstructured-client==0.26.2 +urllib3==2.2.3 +uvicorn==0.32.0 +warc3-wet==0.2.5 +warc3-wet-clueweb09==0.2.5 +watchfiles==0.24.0 +webencodings==0.5.1 +websockets==14.1 +wrapt==1.16.0 +xxhash==3.5.0 +yarl==1.17.0 +zlib-state==0.1.9 diff --git a/server.py b/server.py new file mode 100644 index 0000000..25e24ce --- /dev/null +++ b/server.py @@ -0,0 +1,141 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/11/20 16:32 +# @blog: https://blog.csdn.net/weixin_47936614 + +import os +import shutil + +from fastapi import FastAPI, Request, File, UploadFile +from fastapi.staticfiles import StaticFiles +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates +from langchain.chains.retrieval_qa.base import RetrievalQA +from langchain_core.prompts import PromptTemplate +from langchain_openai import ChatOpenAI +from pymilvus import Collection, connections +from pymilvus.orm import utility + +from milvus_vector import vector_store, config +from file_process import RagFileProcessor +from protocol.prompts import prompt_template +from protocol.mode import ChatRequest + +# 初始化 FastAPI 应用 +app = FastAPI(title="Knowledge_QA_RAG API", description="API for data process and retrieval using Milvus and LangChain.") + +# 挂载静态文件目录 +app.mount("/static", StaticFiles(directory="static"), name="static") + +# 配置模板目录 +templates = Jinja2Templates(directory="templates") + + +# 渲染主页 +@app.get("/", response_class=HTMLResponse) +async def read_home(request: Request): + return templates.TemplateResponse("qa.html", {"request": request}) + + +@app.post("/rag/chat/") +async def chat(request: ChatRequest): + """ + 根据用户问题，从向量库检索并返回回答。 + """ + print(f"Q: {request.question}") + try: + # 初始化 OpenAI Chat 模型 + llm = ChatOpenAI(model=config.llm_model_name, api_key=config.api_key, base_url=config.base_url) + # 定义 Prompt 模板 + qa_prompt = PromptTemplate(template=prompt_template, input_variables=["context", "question"]) + + # 定义搜索参数 + search_kwargs = {"score_threshold": 0.3, "k": 5} + retriever = vector_store.as_retriever(search_type="similarity_score_threshold", search_kwargs=search_kwargs) + qa_chain = RetrievalQA.from_chain_type( + llm=llm, + chain_type="stuff", + retriever=retriever, + chain_type_kwargs={"prompt": qa_prompt}, + return_source_documents=True + ) + result = qa_chain.invoke({"query": request.question}) + answer_result = result.get("result", "") + print(f"A: {answer_result}") + + source = {"source_documents": [{"content": doc.page_content, "metadata": doc.metadata} for doc in + result.get("source_documents", [])]} + print(f"source: {source}") + + return {"answer": answer_result} + except Exception as e: + return {"status": "error", "message": str(e)} + + +@app.post("/rag/clear/") +async def clear_knowledge(collection_name: str = config.milvus_collection_name, + host: str = config.milvus_host, + port: int = config.milvus_port): + """ + 清空 Milvus 知识库集合，并删除指定目录中的文件 + """ + folder = "./upload_files" + try: + connections.connect("default", host=host, port=port) + if utility.has_collection(collection_name): + collection = Collection(name=collection_name) + collection.drop() + print(f"Collection '{collection_name}' 成功删除.") + else: + print(f"Collection '{collection_name}' 不存在.") + connections.disconnect("default") + + for filename in os.listdir(folder): + file_path = os.path.join(folder, filename) + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + return {"message": f"知识库清空: Collection '{collection_name}' 删除, 文件夹 '{folder}' 清空."} + except Exception as e: + return {"error": f"知识库清空失败. 原因: {str(e)}"} + + +@app.post("/rag/create/") +async def create_knowledge(file: UploadFile = File(...)): + """ + 上传文件到指定目录后，处理文件内容并添加到向量库。 + """ + folder = './upload_files' # 文件存储目录 + os.makedirs(folder, exist_ok=True) # 确保目录存在 + file_path = os.path.join(folder, file.filename) + + try: + # 保存文件 + with open(file_path, "wb") as f: + content = await file.read() + f.write(content) + + folder: str = './upload_files' + file_path = os.path.join(folder, file.filename) + + # 初始化文件处理器 + file_processor = RagFileProcessor(chunk_size=64) + # 处理文件内容并插入到向量库 + text_datas = file_processor.get_data(file_path=file_path) + + # 连接到 Milvus + vector_store.add_texts(**text_datas) + return { + "status": "success", + "message": f"文件 '{file.filename}' 上传成功并添加至向量数据库.", + } + except Exception as e: + return { + "status": "error", + "message": f"文件 '{file.filename}'处理失败. 原因: {str(e)}", + } + + diff --git a/static/css/qa_style.css b/static/css/qa_style.css new file mode 100644 index 0000000..2a920ba --- /dev/null +++ b/static/css/qa_style.css @@ -0,0 +1,151 @@ +/*qa_css*/ +body { + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; + margin: 0; + padding: 0; + display: flex; + justify-content: center; + align-items: center; + height: 100vh; + background: linear-gradient(135deg, #696971FF 0%, #788090FF 100%); /* 渐变背景 */ + background-size: cover; + color: #fff; +} + +#chat-container { + width: 50%; /* 宽度为页面的一半 */ + height: 80%; /* 高度为页面的 80% */ + display: flex; + flex-direction: column; + border-radius: 15px; /* 边角圆滑 */ + background-color: rgba(255, 255, 255, 0.9); /* 半透明背景 */ + box-shadow: 0 4px 10px rgba(0, 0, 0, 0.2); /* 阴影效果 */ + overflow: hidden; +} + +#chat-box { + flex: 1; + padding: 20px; + overflow-y: auto; + display: flex; + flex-direction: column; + align-items: flex-start; + background: #f8f8f8; /* 聊天框背景 */ + border-radius: 10px; + margin: 10px; +} + +.message { + display: flex; + margin: 10px 0; + padding: 10px 15px; + border-radius: 10px; + max-width: 75%; + align-items: center; + font-size: 16px; +} + +.user-message { + align-self: flex-end; + background-color: #0084ff; + color: white; + flex-direction: row-reverse; +} + +.bot-message { + align-self: flex-start; + background-color: #e5e5e5; + color: black; + flex-direction: row; +} + +.avatar { + width: 45px; + height: 45px; + border-radius: 50%; + margin: 0 10px; +} + +#input-container { + display: flex; + padding: 15px; + border-top: 1px solid #ddd; + background-color: #fff; + justify-content: space-between; + align-items: center; +} + +#user-input { + flex: 1; + padding: 10px; + font-size: 16px; + border: 1px solid #ccc; + border-radius: 20px; + margin-right: 15px; + outline: none; + transition: all 0.3s ease; +} + +#user-input:focus { + border-color: #0084ff; /* 聚焦时边框颜色变化 */ + box-shadow: 0 0 5px rgba(0, 132, 255, 0.5); /* 聚焦时增加阴影效果 */ +} + +#send-button { + padding: 10px 20px; + font-size: 16px; + background-color: #0084ff; + color: white; + border: none; + border-radius: 30px; + cursor: pointer; + transition: all 0.3s ease; +} + +#send-button:hover { + background-color: #005bb5; /* 按钮悬停时的颜色 */ + transform: scale(1.05); /* 悬停时轻微放大 */ +} + +.message-container { + display: flex; + justify-content: flex-start; + align-items: center; +} + +.message-container-right { + justify-content: flex-end; +} + + +/*新添加的知识库按钮*/ +#kb-actions { + display: flex; + justify-content: center; + margin-top: 10px; + margin-bottom: 5px; +} + +#kb-actions button { + padding: 10px 20px; + font-size: 16px; + background-color: #56a750; /* 默认绿色 */ + color: white; + border: none; + border-radius: 30px; + cursor: pointer; + margin: 0 10px; + transition: all 0.3s ease; +} + +#kb-actions button:hover { + transform: scale(1.05); /* 悬停时轻微放大 */ +} + +#clear-knowledge { + background-color: #dc3545; /* 红色清空按钮 */ +} + +#clear-knowledge:hover { + background-color: #b22b37; /* 悬停时红色更深 */ +} diff --git a/static/images/bot.jpg b/static/images/bot.jpg new file mode 100644 index 0000000..bb867a4 Binary files /dev/null and b/static/images/bot.jpg differ diff --git a/static/images/user.jpg b/static/images/user.jpg new file mode 100644 index 0000000..7ee6721 Binary files /dev/null and b/static/images/user.jpg differ diff --git a/static/js/kb_manager.js b/static/js/kb_manager.js new file mode 100644 index 0000000..a87c897 --- /dev/null +++ b/static/js/kb_manager.js @@ -0,0 +1,73 @@ +// 获取添加知识库按钮 +const addKbButton = document.getElementById("create-knowledge"); + +// 监听添加知识库按钮的点击事件 +addKbButton.addEventListener("click", () => { + // 创建一个隐藏的文件输入框 + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.accept = ".txt,.pdf,.docx"; // 根据需求限制文件类型 + fileInput.style.display = "none"; + + // 将文件输入框添加到页面 + document.body.appendChild(fileInput); + + // 监听文件选择事件 + fileInput.addEventListener("change", async () => { + const file = fileInput.files[0]; // 获取用户选择的文件 + if (!file) { + alert("请选择一个文件！"); + return; + } + + const formData = new FormData(); + formData.append("file", file); // 将文件添加到 FormData 对象 + + try { + // 发起 POST 请求到 /rag/add/ 接口 + const response = await fetch("/rag/create/", { + method: "POST", + body: formData, + }); + + const result = await response.json(); // 解析响应数据 + if (response.ok) { + alert(result.message || "文件上传并添加到知识库成功！"); + } else { + alert(result.message || "上传失败，请重试！"); + } + } catch (error) { + console.error("Error uploading file:", error); + alert("上传操作失败，请检查后端服务！"); + } finally { + // 从 DOM 中移除文件输入框 + document.body.removeChild(fileInput); + } + }); + + // 模拟点击文件输入框，触发文件选择对话框 + fileInput.click(); +}); + +// 监听清空知识库按钮 +const clearKnowledge = async () => { + try { + const response = await fetch("/rag/clear/", { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ + collection_name: "knowledge_collection", + host: "localhost", + port: 19530, + }), + }); + const result = await response.json(); + alert(result.message || result.error); + } catch (error) { + console.error("Error clearing knowledge:", error); + alert("Failed to clear knowledge."); + } +}; + +document.getElementById("clear-knowledge").addEventListener("click", clearKnowledge); + diff --git a/static/js/qa_script.js b/static/js/qa_script.js new file mode 100644 index 0000000..7cd122d --- /dev/null +++ b/static/js/qa_script.js @@ -0,0 +1,97 @@ +// qa_script.js +const chatBox = document.getElementById("chat-box"); +const userInput = document.getElementById("user-input"); +const sendButton = document.getElementById("send-button"); + +// 用户头像与机器人头像的路径，从静态资源目录加载 +const userAvatar = "./static/images/user.jpg"; +const botAvatar = "./static/images/bot.jpg"; + +async function sendMessage() { + const message = userInput.value.trim(); + if (!message) return; + + // 显示用户消息 + const userMessage = document.createElement("div"); + userMessage.textContent = message; + userMessage.className = "message user-message"; + + // 创建用户头像 + const userImage = document.createElement("img"); + userImage.src = userAvatar; + userImage.className = "avatar"; + + // 将用户头像和消息一起放到 message-container 中 + const userMessageContainer = document.createElement("div"); + userMessageContainer.className = "message-container message-container-right"; + userMessageContainer.appendChild(userImage); + userMessageContainer.appendChild(userMessage); + chatBox.appendChild(userMessageContainer); + chatBox.scrollTop = chatBox.scrollHeight; // 滚动到最底部 + + userInput.value = ""; // 清空输入框 + + // 发送问题到后端 + try { + const response = await fetch("/rag/chat/", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ question: message }), + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = await response.json(); + + // 显示机器人回复 + const botMessage = document.createElement("div"); + botMessage.textContent = data.answer || "No answer available."; + botMessage.className = "message bot-message"; + + // 创建机器人头像 + const botImage = document.createElement("img"); + botImage.src = botAvatar; + botImage.className = "avatar"; + + // 将机器人头像和消息一起放到 message-container 中 + const botMessageContainer = document.createElement("div"); + botMessageContainer.className = "message-container"; + botMessageContainer.appendChild(botImage); + botMessageContainer.appendChild(botMessage); + chatBox.appendChild(botMessageContainer); + chatBox.scrollTop = chatBox.scrollHeight; // 滚动到最底部 + } catch (error) { + console.error("Error:", error); + + const botMessage = document.createElement("div"); + botMessage.textContent = "An error occurred. Please try again."; + botMessage.className = "message bot-message"; + + // 创建机器人头像 + const botImage = document.createElement("img"); + botImage.src = botAvatar; + botImage.className = "avatar"; + + // 将头像和消息内容添加到消息容器 + const botMessageContainer = document.createElement("div"); + botMessageContainer.className = "message-container"; + botMessageContainer.appendChild(botImage); + botMessageContainer.appendChild(botMessage); + chatBox.appendChild(botMessageContainer); + chatBox.scrollTop = chatBox.scrollHeight; // 滚动到最底部 + } +} + +// 点击发送按钮时发送消息 +sendButton.addEventListener("click", sendMessage); + +// 按下 Enter 键发送消息 +userInput.addEventListener("keypress", (event) => { + if (event.key === "Enter") { + sendMessage(); + } +}); diff --git a/templates/qa.html b/templates/qa.html new file mode 100644 index 0000000..322dfd0 --- /dev/null +++ b/templates/qa.html @@ -0,0 +1,29 @@ + + + + + + 基于知识库的文档问答系统 + + + + + +
+
+
+ + +
+
+
+ + +
+
+
+ + + + + diff --git a/text_utils/__init__.py b/text_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/text_utils/text_embeddings.py b/text_utils/text_embeddings.py new file mode 100644 index 0000000..a1dc33a --- /dev/null +++ b/text_utils/text_embeddings.py @@ -0,0 +1,30 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/10/31 14:33 +# @blog: https://blog.csdn.net/weixin_47936614 + +from langchain_core.embeddings import Embeddings +from sentence_transformers import SentenceTransformer + + +class RagTextEmbeddings(Embeddings): + def __init__(self, embed_model_path: str, **kwargs): + self.batch_size = kwargs['batch_size'] + self.device = kwargs['device'] + self.embed_model = SentenceTransformer(embed_model_path, trust_remote_code=True, device=self.device) + + def embed_documents(self, texts: list[str]) -> list[list[float]]: + docs_embeddings = self.embed_model.encode(texts, + task="retrieval.passage", + batch_size=self.batch_size, + device=self.device, + show_progress_bar=True) + return docs_embeddings.tolist() + + def embed_query(self, text: str) -> list[float]: + query_embeddings = self.embed_model.encode([text], + task="retrieval.query", + device=self.device) + return query_embeddings.tolist()[0] + diff --git a/text_utils/text_split.py b/text_utils/text_split.py new file mode 100644 index 0000000..7e00bdf --- /dev/null +++ b/text_utils/text_split.py @@ -0,0 +1,76 @@ +# !/usr/bin/env python3 +# -*- coding: utf-8 -*- +# @author: CS_木成河 +# @time: 2024/10/31 11:17 +# @blog: https://blog.csdn.net/weixin_47936614 + +import re +from typing import List + +from langchain.text_splitter import CharacterTextSplitter + + +class RagTextSplitter(CharacterTextSplitter): + def __init__(self, chunk_size: int = 1024): + super().__init__() + self.chunk_size = chunk_size + + def split_text(self, text: str) -> List[str]: + text = re.sub(r"\n{3,}", "\n", text) # 移除三个或更多的连续换行符，用一个换行符代替 + text = re.sub(r'\s+', ' ', text) # 替换所有的空白字符为单个空格 + text = text.replace("\n\n", "") # 移除双换行符 + + sent_sep_pattern = re.compile(r'([﹒﹔﹖﹗．。！？]["’”」』]{0,2})') # 用于匹配中文句子结束标点符号以及紧随其后的引号 + sentences = [] + current_chunk = "" + + start = 0 + for match in sent_sep_pattern.finditer(text): + end = match.end() + sentence = text[start:end] + start = end + + # 检查当前块是否能容纳新句子 + if len(current_chunk) + len(sentence) > self.chunk_size: # 不能容纳 + if current_chunk: + sentences.append(current_chunk) + current_chunk = sentence + else: # 可以容纳 + current_chunk += sentence + + if len(sentences) == 0: + sentences.append(text.strip()) + + final_sentences = [] + for line in sentences: + if len(line) <= self.chunk_size: + final_sentences.append(line) + else: + final_sentences.extend(self.split_string(line, self.chunk_size)) + return final_sentences + + @staticmethod + def split_string(text: str, size: int) -> List[str]: + """ + Split the input string into chunks of specified size, splitting at the last space if needed. + + Parameters: + text (str): The input string to be split. + size (int): The size of each chunk. + + Returns: + list: A list containing the chunks of the input string. + """ + # 定义句子或标记分割符号列表 + SENTENCE_BREAK_SYMBOLS = [' ', '.', '!', '?', ',', ';', ':', '。', '？', '！', '，', '；', '：'] + chunks = [] + start = 0 + while start < len(text): + end = start + size + if end < len(text): + # 在最后一个空格处进行切分 + while end > start and text[end - 1] not in SENTENCE_BREAK_SYMBOLS: + end -= 1 + chunks.append(text[start:end]) + start = end + return chunks