chore: import upstream snapshot with attribution

sdk/python/test/conftest.py

@@ -0,0 +1,152 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+
+import pytest
+import requests
+
+HOST_ADDRESS = os.getenv("HOST_ADDRESS", "http://127.0.0.1:9380")
+ZHIPU_AI_API_KEY = os.getenv("ZHIPU_AI_API_KEY")
+if ZHIPU_AI_API_KEY is None:
+    pytest.exit("Error: Environment variable ZHIPU_AI_API_KEY must be set")
+
+# def generate_random_email():
+#     return 'user_' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))+'@1.com'
+
+
+def generate_email():
+    return "user_123@1.com"
+
+
+EMAIL = generate_email()
+# password is "123"
+PASSWORD = """ctAseGvejiaSWWZ88T/m4FQVOpQyUvP+x7sXtdv3feqZACiQleuewkUi35E16wSd5C5QcnkkcV9cYc8TKPTRZlxappDuirxghxoOvFcJxFU4ixLsD
+fN33jCHRoDUW81IH9zjij/vaw8IbVyb6vuwg6MX6inOEBRRzVbRYxXOu1wkWY6SsI8X70oF9aeLFp/PzQpjoe/YbSqpTq8qqrmHzn9vO+yvyYyvmDsphXe
+X8f7fp9c7vUsfOCkM+gHY3PadG+QHa7KI7mzTKgUTZImK6BZtfRBATDTthEUbbaTewY4H0MnWiCeeDhcbeQao6cFy1To8pE3RpmxnGnS8BsBn8w=="""
+
+
+def register():
+    url = HOST_ADDRESS + "/api/v1/users"
+    name = "user"
+    register_data = {"email": EMAIL, "nickname": name, "password": PASSWORD}
+    res = requests.post(url=url, json=register_data)
+    res = res.json()
+    if res.get("code") != 0:
+        raise Exception(res.get("message"))
+
+
+def login():
+    url = HOST_ADDRESS + "/api/v1/auth/login"
+    login_data = {"email": EMAIL, "password": PASSWORD}
+    response = requests.post(url=url, json=login_data)
+    res = response.json()
+    if res.get("code") != 0:
+        raise Exception(res.get("message"))
+    auth = response.headers["Authorization"]
+    return auth
+
+
+@pytest.fixture(scope="session")
+def get_api_key_fixture():
+    try:
+        register()
+    except Exception as e:
+        print(e)
+    auth = login()
+    url = HOST_ADDRESS + "/v1/system/tokens"
+    auth = {"Authorization": auth}
+    response = requests.post(url=url, headers=auth)
+    res = response.json()
+    if res.get("code") != 0:
+        raise Exception(res.get("message"))
+    return res["data"].get("token")
+
+
+@pytest.fixture(scope="session")
+def get_auth():
+    try:
+        register()
+    except Exception as e:
+        print(e)
+    auth = login()
+    return auth
+
+
+@pytest.fixture(scope="session")
+def get_email():
+    return EMAIL
+
+
+def get_added_models(auth, factory_name):
+    url = HOST_ADDRESS + "/api/v1/models"
+    authorization = {"Authorization": auth}
+    response = requests.get(url=url, headers=authorization)
+    res = response.json()
+    if res.get("code") != 0:
+        raise Exception(res.get("message"))
+    added_factory = {model["provider_name"] for model in res.get("data", [])}
+    if factory_name in added_factory:
+        return True
+    return False
+
+
+def add_model_instance(auth):
+    add_provider_api = HOST_ADDRESS + "/api/v1/providers"
+    authorization = {"Authorization": auth}
+    add_provider_response = requests.put(url=add_provider_api, headers=authorization, json={"provider_name": "ZHIPU-AI"})
+    add_provider_res = add_provider_response.json()
+    if add_provider_res.get("code") != 0:
+        pytest.exit(f"Critical error in add model provider: {add_provider_res.get('message')}")
+
+    add_instance_api = HOST_ADDRESS + "/api/v1/providers/ZHIPU-AI/instances"
+    add_instance_response = requests.post(url=add_instance_api, headers=authorization, json={"instance_name": "CI", "api_key": ZHIPU_AI_API_KEY, "region": "default", "base_url": ""})
+    add_instance_res = add_instance_response.json()
+    if add_instance_res.get("code") != 0:
+        pytest.exit(f"Critical error in add model instance: {add_instance_res.get('message')}")
+
+    add_success = get_added_models(auth, "ZHIPU-AI")
+    if not add_success:
+        pytest.exit("Critical error in check added model: add model failed")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def set_tenant_info(get_auth):
+    auth = get_auth
+    if not get_added_models(auth, "ZHIPU-AI"):
+        try:
+            add_model_instance(auth)
+        except Exception as e:
+            pytest.exit(f"Error in set_tenant_info: {str(e)}")
+    url = HOST_ADDRESS + "/api/v1/models/default"
+    authorization = {"Authorization": get_auth}
+    # set chat model
+    set_default_llm_response = requests.patch(url=url, headers=authorization, json={"model_provider": "ZHIPU-AI", "model_instance": "CI", "model_type": "chat", "model_name": "glm-4-flash"})
+    llm_res = set_default_llm_response.json()
+    if llm_res.get("code") != 0:
+        raise Exception(llm_res.get("message"))
+    # set embedding model
+    set_default_embedding_response = requests.patch(
+        url=url, headers=authorization, json={"model_provider": "Builtin", "model_instance": "Local", "model_type": "embedding", "model_name": "BAAI/bge-small-en-v1.5"}
+    )
+    embd_res = set_default_embedding_response.json()
+    if embd_res.get("code") != 0:
+        raise Exception(embd_res.get("message"))
+    # set image to text model
+    set_default_img2txt_response = requests.patch(url=url, headers=authorization, json={"model_provider": "ZHIPU-AI", "model_instance": "CI", "model_type": "vision", "model_name": "glm-4v"})
+    img2txt_res = set_default_img2txt_response.json()
+    if img2txt_res.get("code") != 0:
+        raise Exception(img2txt_res.get("message"))

sdk/python/test/test_frontend_api/common.py

@@ -0,0 +1,115 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+
+import requests
+
+HOST_ADDRESS = os.getenv("HOST_ADDRESS", "http://127.0.0.1:9380")
+API_VERSION = "v1"
+DATASETS_API_URL = f"/api/{API_VERSION}/datasets"
+
+DATASET_NAME_LIMIT = 128
+
+
+def create_dataset(auth, payload=None):
+    url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
+    res = requests.post(url=url, headers={"Content-Type": "application/json"}, auth=auth, json=payload)
+    return res.json()
+
+
+def list_dataset(auth, params=None):
+    url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
+    res = requests.get(url=url, headers={"Content-Type": "application/json"}, auth=auth, params=params)
+    return res.json()
+
+
+def rm_dataset(auth, dataset_ids):
+    url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
+    res = requests.delete(url=url, headers={"Content-Type": "application/json"}, auth=auth, json={"ids": dataset_ids})
+    return res.json()
+
+
+def update_dataset(auth, dataset_id, payload=None):
+    url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}"
+    res = requests.put(url=url, headers={"Content-Type": "application/json"}, auth=auth, json=payload)
+    return res.json()
+
+
+def upload_file(auth, dataset_id, path):
+    authorization = {"Authorization": auth}
+    url = f"{HOST_ADDRESS}/v1/document/upload"
+    json_req = {
+        "kb_id": dataset_id,
+    }
+
+    file = {"file": open(f"{path}", "rb")}
+
+    res = requests.post(url=url, headers=authorization, files=file, data=json_req)
+    return res.json()
+
+
+def list_document(auth, dataset_id):
+    authorization = {"Authorization": auth}
+    url = f"{HOST_ADDRESS}/v1/document/list?id={dataset_id}"
+    json = {}
+    res = requests.post(url=url, headers=authorization, json=json)
+    return res.json()
+
+
+def get_docs_info(auth, dataset_id, doc_ids=None, doc_id=None):
+    """
+    Get document information by IDs.
+
+    Args:
+        auth: Authorization header
+        dataset_id: Dataset ID
+        doc_ids: List of document IDs (use for multiple) - exclusive with doc_id
+        doc_id: Single document ID (use for one) - exclusive with doc_ids
+
+    Raises:
+        ValueError: If both doc_id and doc_ids are provided
+    """
+    # Validate that id and ids are not used together
+    if doc_id and doc_ids:
+        raise ValueError("Cannot use both 'id' and 'ids' parameters at the same time.")
+
+    authorization = {"Authorization": auth}
+    params = {}
+    if doc_ids:
+        # Multiple IDs
+        for id in doc_ids:
+            params.append(("ids", id))
+    elif doc_id:
+        # Single ID
+        params["id"] = doc_id
+
+    # Use /api/v1 prefix for dataset API
+    url = f"{HOST_ADDRESS}/api/v1/datasets/{dataset_id}/documents"
+    res = requests.get(url=url, headers=authorization, params=params)
+    return res.json()
+
+
+def parse_docs(auth, doc_ids):
+    authorization = {"Authorization": auth}
+    json_req = {"doc_ids": doc_ids, "run": 1}
+    url = f"{HOST_ADDRESS}/api/v1/documents/ingest"
+    res = requests.post(url=url, headers=authorization, json=json_req)
+    return res.json()
+
+
+def parse_file(auth, document_id):
+    pass

sdk/python/test/test_frontend_api/get_email.py

@@ -0,0 +1,20 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+def test_get_email(get_email):
+    print("\nEmail account:", flush=True)
+    print(f"{get_email}\n", flush=True)

sdk/python/test/test_frontend_api/test_chunk.py

@@ -0,0 +1,74 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from common import create_dataset, list_dataset, rm_dataset, upload_file
+from common import list_document, get_docs_info, parse_docs
+from time import sleep
+from timeit import default_timer as timer
+
+
+def test_parse_txt_document(get_auth):
+    # create dataset
+    res = create_dataset(get_auth, {"name": "test_parse_txt_document"})
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    page_number = 1
+    dataset_list = []
+    dataset_id = None
+    while True:
+        res = list_dataset(get_auth, {"page": page_number, "page_size": 150})
+        data = res.get("data")
+        if isinstance(data, dict):
+            data = data.get("kbs", [])
+        for item in data:
+            dataset_id = item.get("id")
+            dataset_list.append(dataset_id)
+        if len(dataset_list) < page_number * 150:
+            break
+        page_number += 1
+
+    filename = "ragflow_test.txt"
+    res = upload_file(get_auth, dataset_id, f"../test_sdk_api/test_data/{filename}")
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    res = list_document(get_auth, dataset_id)
+
+    doc_id_list = []
+    for doc in res["data"]["docs"]:
+        doc_id_list.append(doc["id"])
+
+    res = get_docs_info(get_auth, dataset_id, doc_ids=doc_id_list)
+    print(doc_id_list)
+    doc_count = len(doc_id_list)
+    res = parse_docs(get_auth, doc_id_list)
+
+    start_ts = timer()
+    while True:
+        res = get_docs_info(get_auth, dataset_id, doc_ids=doc_id_list)
+        finished_count = 0
+        for doc_info in res["data"]:
+            if doc_info["progress"] == 1:
+                finished_count += 1
+        if finished_count == doc_count:
+            break
+        sleep(1)
+    print("time cost {:.1f}s".format(timer() - start_ts))
+
+    # delete dataset
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")

sdk/python/test/test_frontend_api/test_dataset.py

@@ -0,0 +1,198 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from common import create_dataset, list_dataset, rm_dataset, update_dataset, DATASET_NAME_LIMIT
+import re
+import random
+import string
+
+
+def test_dataset(get_auth):
+    # create dataset
+    res = create_dataset(get_auth, {"name": "test_create_dataset"})
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    page_number = 1
+    dataset_list = []
+    while True:
+        res = list_dataset(get_auth, {"page": page_number, "page_size": 150})
+        data = res.get("data")
+        if isinstance(data, dict):
+            data = data.get("kbs", [])
+        for item in data:
+            dataset_id = item.get("id")
+            dataset_list.append(dataset_id)
+        if len(dataset_list) < page_number * 150:
+            break
+        page_number += 1
+
+    print(f"found {len(dataset_list)} datasets")
+    # delete dataset
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")
+
+
+def test_dataset_1k_dataset(get_auth):
+    # create dataset
+    for i in range(1000):
+        res = create_dataset(get_auth, {"name": f"test_create_dataset_{i}"})
+        assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    page_number = 1
+    dataset_list = []
+    while True:
+        res = list_dataset(get_auth, {"page": page_number, "page_size": 150})
+        data = res.get("data")
+        if isinstance(data, dict):
+            data = data.get("kbs", [])
+        for item in data:
+            dataset_id = item.get("id")
+            dataset_list.append(dataset_id)
+        if len(dataset_list) < page_number * 150:
+            break
+        page_number += 1
+
+    print(f"found {len(dataset_list)} datasets")
+    # delete dataset
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")
+
+
+def test_duplicated_name_dataset(get_auth):
+    # create dataset
+    for i in range(20):
+        res = create_dataset(get_auth, {"name": "test_create_dataset"})
+        assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    res = list_dataset(get_auth, {"page": 1})
+    data = res.get("data")
+    if isinstance(data, dict):
+        data = data.get("kbs", [])
+    dataset_list = []
+    pattern = r"^test_create_dataset.*"
+    for item in data:
+        dataset_name = item.get("name")
+        dataset_id = item.get("id")
+        dataset_list.append(dataset_id)
+        match = re.match(pattern, dataset_name)
+        assert match is not None
+
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")
+
+
+def test_invalid_name_dataset(get_auth):
+    # create dataset
+    res = create_dataset(get_auth, {"name": 0})
+    assert res["code"] != 0
+
+    res = create_dataset(get_auth, {"name": ""})
+    assert res["code"] != 0
+
+    long_string = ""
+
+    while len(long_string.encode("utf-8")) <= DATASET_NAME_LIMIT:
+        long_string += random.choice(string.ascii_letters + string.digits)
+
+    res = create_dataset(get_auth, {"name": long_string})
+    assert res["code"] != 0
+    print(res)
+
+
+def test_update_different_params_dataset_success(get_auth):
+    # create dataset
+    res = create_dataset(get_auth, {"name": "test_create_dataset"})
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    page_number = 1
+    dataset_list = []
+    while True:
+        res = list_dataset(get_auth, {"page": page_number, "page_size": 150})
+        data = res.get("data")
+        if isinstance(data, dict):
+            data = data.get("kbs", [])
+        for item in data:
+            dataset_id = item.get("id")
+            dataset_list.append(dataset_id)
+        if len(dataset_list) < page_number * 150:
+            break
+        page_number += 1
+
+    print(f"found {len(dataset_list)} datasets")
+    dataset_id = dataset_list[0]
+
+    res = update_dataset(
+        get_auth,
+        dataset_id,
+        {
+            "name": "test_update_dataset",
+            "description": "test",
+            "permission": "me",
+            "chunk_method": "presentation",
+            "language": "spanish",
+        },
+    )
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    # delete dataset
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")
+
+
+# update dataset with different parameters
+def test_update_different_params_dataset_fail(get_auth):
+    # create dataset
+    res = create_dataset(get_auth, {"name": "test_create_dataset"})
+    assert res.get("code") == 0, f"{res.get('message')}"
+
+    # list dataset
+    page_number = 1
+    dataset_list = []
+    while True:
+        res = list_dataset(get_auth, {"page": page_number, "page_size": 150})
+        data = res.get("data")
+        if isinstance(data, dict):
+            data = data.get("kbs", [])
+        for item in data:
+            dataset_id = item.get("id")
+            dataset_list.append(dataset_id)
+        if len(dataset_list) < page_number * 150:
+            break
+        page_number += 1
+
+    print(f"found {len(dataset_list)} datasets")
+    dataset_id = dataset_list[0]
+
+    res = update_dataset(get_auth, dataset_id, {"id": "xxx"})
+    assert res.get("code") == 101
+
+    # delete dataset
+    if dataset_list:
+        res = rm_dataset(get_auth, dataset_list)
+        assert res.get("code") == 0, f"{res.get('message')}"
+    print(f"{len(dataset_list)} datasets are deleted")