chore: import upstream snapshot with attribution

examples/__main__.py

@@ -0,0 +1,31 @@
+# TODO add example run code here
+
+import asyncio
+
+# Agents examples
+from .agents.auto_plan_agent_dialogue_example import main as auto_plan_main
+from .agents.awel_layout_agents_chat_examples import main as awel_layout_main
+from .agents.custom_tool_agent_example import main as custom_tool_main
+from .agents.plugin_agent_dialogue_example import main as plugin_main
+from .agents.retrieve_summary_agent_dialogue_example import (
+    main as retrieve_summary_main,
+)
+from .agents.sandbox_code_agent_example import main as sandbox_code_main
+from .agents.single_agent_dialogue_example import main as single_agent_main
+from .agents.sql_agent_dialogue_example import main as sql_main
+
+if __name__ == "__main__":
+    # Run the examples
+
+    ## Agent examples
+    asyncio.run(auto_plan_main())
+    asyncio.run(awel_layout_main())
+    asyncio.run(custom_tool_main())
+    asyncio.run(retrieve_summary_main())
+    asyncio.run(plugin_main())
+    asyncio.run(sandbox_code_main())
+    asyncio.run(single_agent_main())
+    asyncio.run(sql_main())
+
+    ## awel examples
+    print("hello world!")

examples/agents/README_SKILL_AGENT.md

@@ -0,0 +1,8 @@
+Skill integration notes and quick run for the skill-enabled agent example
+
+Run the example:
+
+python examples/agents/skill_agent_example.py
+
+Ensure you have the environment variables needed by the local LLM client, or
+replace the LLM client construction with a mock for testing.

examples/agents/auto_plan_agent_dialogue_example.py

@@ -0,0 +1,82 @@
+"""Agents: auto plan agents example?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export SILICONFLOW_API_KEY=sk-xx
+        export SILICONFLOW_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/auto_plan_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    AutoPlanChatManager,
+    LLMConfig,
+    UserProxyAgent,
+)
+from dbgpt.agent.expand.code_assistant_agent import CodeAssistantAgent
+from dbgpt.util.tracer import initialize_tracer
+
+initialize_tracer(
+    "/tmp/agent_auto_plan_agent_dialogue_example_trace.jsonl", create_system_app=True
+)
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+
+    context: AgentContext = AgentContext(
+        conv_id="test456", gpts_app_name="代码分析助手", max_new_tokens=2048
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+    try:
+        coder = (
+            await CodeAssistantAgent()
+            .bind(context)
+            .bind(LLMConfig(llm_client=llm_client))
+            .bind(agent_memory)
+            .build()
+        )
+
+        manager = (
+            await AutoPlanChatManager()
+            .bind(context)
+            .bind(agent_memory)
+            .bind(LLMConfig(llm_client=llm_client))
+            .build()
+        )
+        manager.hire([coder])
+
+        user_proxy = await UserProxyAgent().bind(context).bind(agent_memory).build()
+
+        await user_proxy.initiate_chat(
+            recipient=manager,
+            reviewer=user_proxy,
+            message="Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.",
+            # message="Find papers on gpt-4 in the past three weeks on arxiv, and organize their titles, authors, and links into a markdown table",
+            # message="find papers on LLM applications from arxiv in the last month, create a markdown table of different domains.",
+        )
+    finally:
+        agent_memory.gpts_memory.clear(conv_id="test456")
+
+
+if __name__ == "__main__":
+    ## dbgpt-vis message infos
+    asyncio.run(main())

examples/agents/auto_search_agent_example.py

@@ -0,0 +1,109 @@
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.data_scientist_agent import DataScientistAgent
+from dbgpt.agent.expand.web_assistant_agent import WebSearchAgent
+from dbgpt.agent.resource import RDBMSConnectorResource
+from dbgpt.model.proxy import TongyiLLMClient
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteConnector
+
+api_base = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+api_key = "sk-xxx"
+model = "qwen3-32b"
+
+
+def read_excel_headers_and_data(
+    file_path: str,
+) -> Tuple[List[str], List[Dict[str, Any]]]:
+    """
+    读取Excel文件，返回表头信息和结构化数据
+
+    参数:
+        file_path: Excel文件路径（.xlsx格式）
+
+    返回:
+        Tuple[表头列表, 数据列表]
+        - 表头列表: 从Excel第一行读取的列名
+        - 数据列表: 每个元素是一个字典，键为表头，值为对应单元格数据（空单元格转为None）
+    """
+    if not Path(file_path).exists():
+        raise FileNotFoundError(f"文件不存在: {file_path}")
+
+    if Path(file_path).suffix.lower() != ".xlsx":
+        raise ValueError(f"不支持的文件格式: {Path(file_path).suffix}，仅支持.xlsx")
+
+    try:
+        df = pd.read_excel(
+            file_path, sheet_name=0, engine="openpyxl", keep_default_na=False
+        )
+    except Exception as e:
+        raise RuntimeError(f"读取Excel失败: {str(e)}")
+
+    headers = list(df.columns)
+    if not headers:
+        raise ValueError("Excel文件没有表头信息（第一行为空）")
+
+    data = []
+    for _, row in df.iterrows():
+        row_data = {}
+        for header in headers:
+            value = row[header]
+            row_data[header] = value if value != "" else None
+        data.append(row_data)
+
+    return headers, data
+
+
+def data2md(headers, table_data):
+    md_lines = []
+
+    md_lines.append("| " + " | ".join(headers) + " |")
+    md_lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+    for row in table_data:
+        values = []
+        for h in headers:
+            val = row.get(h, "")
+            if hasattr(val, "strftime"):
+                values.append(val.strftime("%Y-%m-%d"))
+            else:
+                values.append(str(val))
+        md_lines.append("| " + " | ".join(values) + " |")
+
+    markdown_table = "\n".join(md_lines)
+    return markdown_table
+
+
+async def main():
+    llm_client = TongyiLLMClient(api_base=api_base, api_key=api_key, model=model)
+    context: AgentContext = AgentContext(
+        conv_id="test123", language="zh", temperature=0.5, max_new_tokens=2048
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test123")
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    sql_boy = (
+        await WebSearchAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=sql_boy, reviewer=user_proxy, message=f"今年的中秋节是多久？"
+    )
+    print(await agent_memory.gpts_memory.app_link_chat_message("test123"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/awel_layout_agents_chat_examples.py

@@ -0,0 +1,95 @@
+"""Agents: auto plan agents example?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export SILICONFLOW_API_KEY=sk-xx
+        export SILICONFLOW_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/awel_layout_agents_chat_examples.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    LLMConfig,
+    UserProxyAgent,
+    WrappedAWELLayoutManager,
+)
+from dbgpt.agent.expand.resources.search_tool import baidu_search
+from dbgpt.agent.expand.summary_assistant_agent import SummaryAssistantAgent
+from dbgpt.agent.expand.tool_assistant_agent import ToolAssistantAgent
+from dbgpt.agent.resource import ToolPack
+from dbgpt.util.tracer import initialize_tracer
+
+initialize_tracer("/tmp/agent_trace.jsonl", create_system_app=True)
+
+
+async def main():
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+    try:
+        from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+        llm_client = SiliconFlowLLMClient(
+            model_alias=os.getenv(
+                "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+            ),
+        )
+
+        context: AgentContext = AgentContext(
+            conv_id="test456", gpts_app_name="信息析助手"
+        )
+
+        tools = ToolPack([baidu_search])
+        tool_engineer = (
+            await ToolAssistantAgent()
+            .bind(context)
+            .bind(LLMConfig(llm_client=llm_client))
+            .bind(agent_memory)
+            .bind(tools)
+            .build()
+        )
+        summarizer = (
+            await SummaryAssistantAgent()
+            .bind(context)
+            .bind(agent_memory)
+            .bind(LLMConfig(llm_client=llm_client))
+            .build()
+        )
+
+        manager = (
+            await WrappedAWELLayoutManager()
+            .bind(context)
+            .bind(agent_memory)
+            .bind(LLMConfig(llm_client=llm_client))
+            .build()
+        )
+        manager.hire([tool_engineer, summarizer])
+
+        user_proxy = await UserProxyAgent().bind(context).bind(agent_memory).build()
+
+        await user_proxy.initiate_chat(
+            recipient=manager,
+            reviewer=user_proxy,
+            message="查询北京今天天气",
+            # message="查询今天的最新热点财经新闻",
+            # message="Find papers on gpt-4 in the past three weeks on arxiv, and organize their titles, authors, and links into a markdown table",
+            # message="find papers on LLM applications from arxiv in the last month, create a markdown table of different domains.",
+        )
+
+    finally:
+        agent_memory.gpts_memory.clear(conv_id="test456")
+
+
+if __name__ == "__main__":
+    ## dbgpt-vis message infos
+    asyncio.run(main())

examples/agents/claude_skill_example.py

@@ -0,0 +1,102 @@
+"""
+Example: Using Claude-style SKILL files with DB-GPT agents.
+
+This demonstrates how to use the Claude SKILL mechanism
+where skills are defined in Markdown files.
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig
+from dbgpt.agent.claude_skill import (
+    ClaudeSkillAgent,
+    get_registry,
+    load_skills_from_dir,
+)
+from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+
+async def main():
+    """Main function demonstrating Claude SKILL usage."""
+
+    # Load skills from directory
+    skill_dir = os.path.join(os.path.dirname(__file__), "../skills/claude")
+
+    load_skills_from_dir(skill_dir, recursive=True)
+
+    # List available skills
+    registry = get_registry()
+    print("Loaded skills:")
+    for skill_metadata in registry.list_skills():
+        print(f"  - {skill_metadata.name}: {skill_metadata.description}")
+
+    # Create LLM client
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+
+    # Create agent context
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="claude_skill_test")
+
+    context: AgentContext = AgentContext(
+        conv_id="claude_skill_test",
+        gpts_app_name="Claude Skill Agent",
+    )
+
+    # Create Claude Skill Agent
+    agent = ClaudeSkillAgent()
+
+    # Bind necessary components
+    await (
+        agent.bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    print("\n" + "=" * 60)
+    print("Claude Skill Agent Ready!")
+    print("=" * 60)
+
+    # Example interactions
+    test_inputs = [
+        "Can you explain how this code works?",
+        "My code isn't working, can you help debug it?",
+        "Write a function to sort a list",
+        "Simplify this complex code for me",
+    ]
+
+    print("\nTest inputs and skill detection:")
+    print("-" * 60)
+
+    for user_input in test_inputs:
+        agent.detect_and_apply_skill(user_input)
+
+        if agent.current_skill:
+            print(f"Input: {user_input}")
+            print(f"Matched Skill: {agent.current_skill.metadata.name}")
+            print(f"Description: {agent.current_skill.metadata.description}")
+            print(f"Instructions preview: {agent.current_skill.instructions[:100]}...")
+            print()
+
+    # Show available skills
+    print("\nAvailable skills:")
+    for skill_name in agent.get_available_skills():
+        print(f"  - {skill_name}")
+
+    # Manual skill selection example
+    print("\n" + "-" * 60)
+    print("Manual skill selection:")
+    agent.set_skill("explain-code")
+    print(f"Current skill: {agent.current_skill.metadata.name}")
+
+    agent.clear_skill()
+    print(f"After clearing: {agent.current_skill}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/custom_tool_agent_example.py

@@ -0,0 +1,89 @@
+import asyncio
+import logging
+import os
+import sys
+
+from typing_extensions import Annotated, Doc
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.tool_assistant_agent import ToolAssistantAgent
+from dbgpt.agent.resource import ToolPack, tool
+
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+)
+
+
+@tool
+def simple_calculator(first_number: int, second_number: int, operator: str) -> float:
+    """Simple calculator tool. Just support +, -, *, /."""
+    if isinstance(first_number, str):
+        first_number = int(first_number)
+    if isinstance(second_number, str):
+        second_number = int(second_number)
+    if operator == "+":
+        return first_number + second_number
+    elif operator == "-":
+        return first_number - second_number
+    elif operator == "*":
+        return first_number * second_number
+    elif operator == "/":
+        return first_number / second_number
+    else:
+        raise ValueError(f"Invalid operator: {operator}")
+
+
+@tool
+def count_directory_files(path: Annotated[str, Doc("The directory path")]) -> int:
+    """Count the number of files in a directory."""
+    if not os.path.isdir(path):
+        raise ValueError(f"Invalid directory path: {path}")
+    return len(os.listdir(path))
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+
+    context: AgentContext = AgentContext(conv_id="test456", gpts_app_name="工具助手")
+
+    tools = ToolPack([simple_calculator, count_directory_files])
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    tool_engineer = (
+        await ToolAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tools)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="Calculate the product of 10 and 99",
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="Count the number of files in /tmp",
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("test456"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/data_manus_example.py

@@ -0,0 +1,180 @@
+import asyncio
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    AutoPlanChatManager,
+    LLMConfig,
+    UserProxyAgent,
+)
+from dbgpt.agent.expand.actions.insert_action import Excel2TableAction
+from dbgpt.agent.expand.data_scientist_agent import DataScientistAgent
+from dbgpt.agent.expand.excel_table_agent import Excel2TableAgent, excel_files
+from dbgpt.agent.expand.web_assistant_agent import WebSearchAgent
+from dbgpt.agent.resource import RDBMSConnectorResource
+from dbgpt.model.proxy import OpenAILLMClient, TongyiLLMClient
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteConnector
+
+connector = SQLiteConnector.from_file_path("../test_files/datamanus_test.db")
+db_resource = RDBMSConnectorResource("user_manager", connector=connector)
+api_base = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+api_key = "sk-xxx"
+model = "qwq-32b"
+
+
+def read_excel_headers_and_data(
+    file_path: str, read_rows: Optional[int] = 3
+) -> Tuple[List[str], List[Dict[str, Any]]]:
+    """
+    读取Excel文件，返回表头信息和结构化数据（支持指定读取行数）
+
+    参数:
+        file_path: Excel文件路径（.xlsx格式）
+        read_rows: 可选，指定读取的数据行数（不含表头）。
+                   - 默认为5：仅读取前5行数据
+                   - 设为None或0：读取全部数据
+                   - 设为正整数N：读取前N行数据（若数据总行数不足N，则读取实际所有行）
+
+    返回:
+        Tuple[表头列表, 数据列表]
+        - 表头列表: 从Excel第一行读取的列名
+        - 数据列表: 每个元素是一个字典，键为表头，值为对应单元格数据（空单元格转为None）
+    """
+
+    if not Path(file_path).exists():
+        raise FileNotFoundError(f"文件不存在: {file_path}")
+    if Path(file_path).suffix.lower() != ".xlsx":
+        raise ValueError(f"不支持的文件格式: {Path(file_path).suffix}，仅支持.xlsx")
+
+    try:
+        df = pd.read_excel(
+            file_path, sheet_name=0, engine="openpyxl", keep_default_na=False
+        )
+    except Exception as e:
+        raise RuntimeError(f"读取Excel失败: {str(e)}")
+
+    headers = list(df.columns)
+    if not headers:
+        raise ValueError("Excel文件没有表头信息（第一行为空）")
+
+    total_data_rows = len(df)
+    if read_rows in (None, 0):
+        target_rows = total_data_rows
+    elif isinstance(read_rows, int) and read_rows > 0:
+        target_rows = min(read_rows, total_data_rows)
+    else:
+        raise ValueError(f"参数read_rows无效：{read_rows}，仅支持正整数、None或0")
+
+    df_target = df.head(target_rows)
+
+    data = []
+    for _, row in df_target.iterrows():
+        row_data = {
+            header: (row[header] if row[header] != "" else None) for header in headers
+        }
+        data.append(row_data)
+
+    return headers, data
+
+
+def data2md(headers, table_data):
+    md_lines = []
+
+    md_lines.append("| " + " | ".join(headers) + " |")
+    md_lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+    for row in table_data:
+        values = []
+        for h in headers:
+            val = row.get(h, "")
+            if hasattr(val, "strftime"):
+                values.append(val.strftime("%Y-%m-%d"))
+            else:
+                values.append(str(val))
+        md_lines.append("| " + " | ".join(values) + " |")
+
+    markdown_table = "\n".join(md_lines)
+    return markdown_table
+
+
+async def main():
+    all_file_data = []
+    # To read some data from Excel files, you can go to excel_table_agent.py
+    # by yourself and replace the excel_file variable
+    # as the default directory where the excel file is located
+    for excel_file in excel_files:
+        filename_with_ext = os.path.basename(excel_file)
+        headers, table_data = read_excel_headers_and_data(excel_file)
+        mdstr = data2md(headers, table_data)
+        all_file_data.append((filename_with_ext, mdstr))
+
+    llm_client = TongyiLLMClient(api_base=api_base, api_key=api_key, model=model)
+    context: AgentContext = AgentContext(
+        conv_id="test123", language="zh", temperature=0.5, max_new_tokens=2048
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test123")
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    excel_boy = (
+        await Excel2TableAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(db_resource)
+        .bind(agent_memory)
+        .build()
+    )
+
+    sql_boy = (
+        await DataScientistAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(db_resource)
+        .bind(agent_memory)
+        .build()
+    )
+
+    web_boy = (
+        await WebSearchAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    manager = (
+        await AutoPlanChatManager()
+        .bind(context)
+        .bind(agent_memory)
+        .bind(LLMConfig(llm_client=llm_client))
+        .build()
+    )
+    manager.hire([sql_boy])
+    manager.hire([excel_boy])
+    manager.hire([web_boy])
+
+    message_parts = ["我读取到以下Excel文件的数据："]
+    for i, (filename, mdstr) in enumerate(all_file_data, 1):
+        message_parts.append(f"\n文件 {i}: {filename}")
+        message_parts.append(f"数据内容：\n{mdstr}")
+    full_message = (
+        "\n".join(message_parts) + "\n\n\n截止今年中秋节之前哪些员工还有项目没有结项？"
+    )
+    print("完整消息内容：" + full_message)
+    await user_proxy.initiate_chat(
+        recipient=manager,
+        reviewer=user_proxy,
+        message=full_message,
+    )
+
+    print(await agent_memory.gpts_memory.app_link_chat_message("test123"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/db_create_example.py

@@ -0,0 +1,125 @@
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.actions.insert_action import Excel2TableAction
+from dbgpt.agent.expand.data_scientist_agent import DataScientistAgent
+from dbgpt.agent.expand.excel_table_agent import Excel2TableAgent, excel_files
+from dbgpt.agent.resource import RDBMSConnectorResource
+from dbgpt.model.proxy import TongyiLLMClient
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteConnector
+
+connector = SQLiteConnector.from_file_path("../test_files/datamanus_test.db")
+db_resource = RDBMSConnectorResource("user_manager", connector=connector)
+api_base = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+api_key = "sk-xxx"
+model = "qwen3-32b"
+
+
+def read_excel_headers_and_data(
+    file_path: str,
+) -> Tuple[List[str], List[Dict[str, Any]]]:
+    if not Path(file_path).exists():
+        raise FileNotFoundError(f"文件不存在: {file_path}")
+
+    if Path(file_path).suffix.lower() != ".xlsx":
+        raise ValueError(f"不支持的文件格式: {Path(file_path).suffix}，仅支持.xlsx")
+
+    try:
+        df = pd.read_excel(
+            file_path, sheet_name=0, engine="openpyxl", keep_default_na=False
+        )
+    except Exception as e:
+        raise RuntimeError(f"读取Excel失败: {str(e)}")
+
+    headers = list(df.columns)
+    if not headers:
+        raise ValueError("Excel文件没有表头信息（第一行为空）")
+
+    data = []
+    for _, row in df.iterrows():
+        row_data = {}
+        for header in headers:
+            value = row[header]
+            row_data[header] = value if value != "" else None
+        data.append(row_data)
+
+    return headers, data
+
+
+def data2md(headers, table_data):
+    md_lines = []
+
+    md_lines.append("| " + " | ".join(headers) + " |")
+    md_lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+    for row in table_data:
+        values = []
+        for h in headers:
+            val = row.get(h, "")
+            if hasattr(val, "strftime"):
+                values.append(val.strftime("%Y-%m-%d"))
+            else:
+                values.append(str(val))
+        md_lines.append("| " + " | ".join(values) + " |")
+
+    markdown_table = "\n".join(md_lines)
+    return markdown_table
+
+
+async def main():
+    all_file_data = []
+    # To read some data from Excel files, you can go to excel_table_agent.py
+    # by yourself and replace the excel_file variable
+    # as the default directory where the excel file is located
+    for excel_file in excel_files:
+        filename_with_ext = os.path.basename(excel_file)
+        headers, table_data = read_excel_headers_and_data(excel_file)
+        mdstr = data2md(headers, table_data)
+        all_file_data.append((filename_with_ext, mdstr))
+
+    llm_client = TongyiLLMClient(api_base=api_base, api_key=api_key, model=model)
+
+    context: AgentContext = AgentContext(
+        conv_id="test123", language="zh", temperature=0.5, max_new_tokens=2048
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test123")
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    excel_boy = (
+        await Excel2TableAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(db_resource)
+        .bind(agent_memory)
+        .build()
+    )
+
+    message_parts = ["我读取到以下Excel文件的数据："]
+    for i, (filename, mdstr) in enumerate(all_file_data, 1):
+        message_parts.append(f"\n文件 {i}: {filename}")
+        message_parts.append(f"数据内容：\n{mdstr}")
+    full_message = (
+        "\n".join(message_parts)
+        + "\n请帮我分析这些数据，为每个文件生成对应的建表语句和插入语句，并执行这些SQL语句创建数据表并插入数据。"
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=excel_boy,
+        reviewer=user_proxy,
+        message=full_message,
+    )
+
+    print(await agent_memory.gpts_memory.app_link_chat_message("test123"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/excel_data_agent.py

@@ -0,0 +1,291 @@
+"""Excel Data Analysis Agent Example (ReAct Version with Built-in Skill Metadata).
+
+This example demonstrates a ReAct agent that has pre-loaded metadata about available skills
+in its system prompt. It can choose to load a specific skill to get detailed instructions
+without needing to search/list first.
+"""
+
+import asyncio
+import logging
+import os
+import sys
+
+# Add project root to sys.path
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+if project_root not in sys.path:
+    sys.path.append(project_root)
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    LLMConfig,
+    ProfileConfig,
+    UserProxyAgent,
+)
+from dbgpt.agent.expand.actions.react_action import Terminate
+from dbgpt.agent.expand.react_agent import ReActAgent
+from dbgpt.agent.resource import ToolPack, tool
+from dbgpt.agent.resource.manage import (
+    get_resource_manager,
+    initialize_resource,
+)
+from dbgpt.agent.resource.skill_resource import SkillResource
+from dbgpt.agent.skill import (
+    SkillLoader,
+    initialize_skill,
+)
+from dbgpt.model import AutoLLMClient
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# --- Global Execution Context ---
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+GLOBAL_EXECUTION_CONTEXT = {"pd": pd, "np": np, "plt": plt, "print": print}
+
+
+# --- Helpers ---
+
+
+def scan_skills(skills_dir: str):
+    """Scan the skills directory and return a list of skill metadata dicts."""
+    skills = []
+    if not os.path.exists(skills_dir):
+        return skills
+
+    # Use SkillLoader to properly load metadata if possible,
+    # but for scanning we might just want to peek at files to be fast.
+    # Here we will try to load them to get accurate descriptions.
+    loader = SkillLoader()
+
+    for root, dirs, files in os.walk(skills_dir):
+        if "SKILL.md" in files:
+            skill_path = os.path.join(root, "SKILL.md")
+            try:
+                # We load the skill to get its metadata (name, description)
+                skill = loader.load_skill_from_file(skill_path)
+                if skill and skill.metadata:
+                    skills.append(
+                        {
+                            "name": skill.metadata.name,
+                            "description": skill.metadata.description,
+                            "path": os.path.relpath(skill_path, project_root),
+                        }
+                    )
+            except Exception as e:
+                logger.warning(f"Failed to load skill at {skill_path}: {e}")
+    return skills
+
+
+# --- Tools ---
+
+
+@tool
+def code_interpreter(code: str) -> str:
+    """Execute Python code for data analysis.
+
+    This tool allows you to run Python code to analyze data. You can use pandas, numpy,
+    matplotlib, etc. The environment is persistent between calls.
+
+    Args:
+        code: The Python code to execute.
+
+    Returns:
+        The standard output and any error messages from the execution.
+    """
+    try:
+        import ast
+        import io
+        import sys
+
+        # AST transformation: wrap last expression in print() if needed
+        try:
+            tree = ast.parse(code)
+            if tree.body and isinstance(tree.body[-1], ast.Expr):
+                expr_node = tree.body[-1]
+                print_call = ast.Call(
+                    func=ast.Name(id="print", ctx=ast.Load()),
+                    args=[expr_node.value],
+                    keywords=[],
+                )
+                tree.body[-1] = ast.Expr(value=print_call)
+                ast.fix_missing_locations(tree)
+                compiled_code = compile(tree, filename="<string>", mode="exec")
+        except Exception:
+            compiled_code = code
+
+        old_stdout = sys.stdout
+        redirected_output = sys.stdout = io.StringIO()
+
+        exec(compiled_code, GLOBAL_EXECUTION_CONTEXT)
+
+        sys.stdout = old_stdout
+        return redirected_output.getvalue()
+    except Exception as e:
+        return f"Execution Error: {str(e)}"
+
+
+@tool
+def load_skill(skill_name: str) -> str:
+    """Load a skill to get detailed instructions for a specific task.
+
+    Use this tool when you want to "read" or "activate" a skill from your available list.
+    It returns the full content and instructions of the skill.
+
+    Args:
+        skill_name: The name of the skill to load (must be one of the available skills).
+
+    Returns:
+        The detailed instructions and workflow defined in the skill.
+    """
+    # In a real implementation, we would use the SkillManager or a registry lookup.
+    # For this script, we'll scan again or use a cached lookup to find the path.
+    skills_dir = os.path.join(project_root, "skills")
+    target_skill_path = None
+
+    # Simple search
+    for root, dirs, files in os.walk(skills_dir):
+        if "SKILL.md" in files:
+            # Check if this folder or metadata matches the requested name
+            # We'll try to match loosely by folder name or strict check if we loaded metadata
+            # For robustness in this example, let's load it to check name.
+            try:
+                loader = SkillLoader()
+                path = os.path.join(root, "SKILL.md")
+                skill = loader.load_skill_from_file(path)
+                if skill and skill.metadata.name == skill_name:
+                    target_skill_path = path
+                    break
+            except:
+                continue
+
+    if not target_skill_path:
+        return f"Error: Skill '{skill_name}' not found."
+
+    try:
+        with open(target_skill_path, "r", encoding="utf-8") as f:
+            content = f.read()
+        return f"Successfully loaded skill '{skill_name}'.\n\nSKILL CONTENT:\n{content}"
+    except Exception as e:
+        return f"Error reading skill file: {str(e)}"
+
+
+async def main():
+    """Main execution function."""
+    system_app = SystemApp()
+
+    # 1. Initialize Managers
+    initialize_skill(system_app)
+    initialize_resource(system_app)
+
+    # 2. Setup LLM
+    llm_client = AutoLLMClient(
+        provider=os.getenv("LLM_PROVIDER", "proxy/siliconflow"),
+        name=os.getenv("LLM_MODEL_NAME", "Qwen/Qwen2.5-Coder-32B-Instruct"),
+    )
+
+    # 3. Scan for Skills
+    skills_dir = os.path.join(project_root, "skills")
+    available_skills_list = scan_skills(skills_dir)
+
+    # 4. Construct Skill Prompt Section
+    if not available_skills_list:
+        skill_section = "Load a skill to get detailed instructions for a specific task. No skills are currently available."
+    else:
+        skills_xml = []
+        for s in available_skills_list:
+            skills_xml.append(f"  <skill>")
+            skills_xml.append(f"    <name>{s['name']}</name>")
+            skills_xml.append(f"    <description>{s['description']}</description>")
+            skills_xml.append(f"  </skill>")
+
+        skill_section_str = "\n".join(skills_xml)
+        skill_section = (
+            "Load a skill to get detailed instructions for a specific task.\n"
+            "Skills provide specialized knowledge and step-by-step guidance.\n"
+            "Use this when a task matches an available skill's description.\n"
+            "Only the skills listed here are available:\n"
+            "<available_skills>\n"
+            f"{skill_section_str}\n"
+            "</available_skills>"
+        )
+
+    # 5. Context & Memory
+    context = AgentContext(
+        conv_id="skill_metadata_session", gpts_app_name="Skill Specialist"
+    )
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="skill_metadata_session")
+
+    # 6. Tools (Notice: No list tool, just load/read)
+    tools = ToolPack([load_skill, code_interpreter, Terminate()])
+
+    # 7. Profile
+    profile = ProfileConfig(
+        name="SkillAwareAgent",
+        role="Adaptive Assistant",
+        goal=(
+            "You are an intelligent assistant. "
+            f"{skill_section}\n\n"
+            "WORKFLOW:\n"
+            "1. Analyze the user's request.\n"
+            "2. Identify if an available skill in <available_skills> matches the request.\n"
+            "3. If yes, use the `load_skill` tool with the skill's name to get instructions.\n"
+            "4. Follow the loaded instructions strictly to complete the task using `code_interpreter`."
+        ),
+    )
+
+    # 8. Build ReAct Agent
+    agent = (
+        await ReActAgent(
+            profile=profile,
+            max_retry_count=5,
+        )
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tools)
+        .build()
+    )
+
+    # 9. User Proxy
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    # 10. Test Data Setup
+    test_file = "sales_data_sample.xlsx"
+    df = pd.DataFrame(
+        {
+            "Date": pd.date_range(start="1/1/2023", periods=10),
+            "Product": ["Widget A", "Widget B", "Widget A", "Widget C", "Widget B"] * 2,
+            "Sales": [100, 200, 150, 300, 250, 120, 220, 160, 310, 260],
+            "Region": ["North", "South", "North", "East", "South"] * 2,
+        }
+    )
+    df.to_excel(test_file, index=False)
+    abs_test_file = os.path.abspath(test_file)
+    logger.info(f"Created test file: {abs_test_file}")
+
+    # 11. Start Chat
+    msg = f"I have a file at '{abs_test_file}'. Analyze the sales data."
+
+    logger.info("Starting session...")
+    await user_proxy.initiate_chat(
+        recipient=agent,
+        reviewer=user_proxy,
+        message=msg,
+    )
+
+    # Cleanup
+    if os.path.exists(test_file):
+        os.remove(test_file)
+        logger.info("Test file cleaned up.")
+
+
+from dbgpt.component import SystemApp
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/mcp_agent_dialogue_example.py

@@ -0,0 +1,79 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export SILICONFLOW_API_KEY=sk-xx
+        export SILICONFLOW_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/plugin_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.tool_assistant_agent import ToolAssistantAgent
+from dbgpt.agent.resource import AutoGPTPluginToolPack, MCPToolPack
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt.model.proxy import SiliconFlowLLMClient
+
+test_plugin_dir = os.path.join(ROOT_PATH, "examples/test_files/plugins")
+
+
+async def main():
+    ### Test method
+    # 1.start mcp server as a sse server
+    # Reference https://github.com/supercorp-ai/supergateway
+    # npx -y supergateway --stdio "uvx mcp-server-fetch"
+    # or
+    # npx -y supergateway --stdio "npx -y @modelcontextprotocol/server-filesystem ./"
+    ## ./ 可以替换为你需要代理的目录
+
+    # 2.bind dbgpt resource MCPToolPack use mcp sse server lisk this：
+    # MCPToolPack("http://127.0.0.1:8000/sse")
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+
+    context: AgentContext = AgentContext(
+        conv_id="test456", gpts_app_name="MCP工具对话助手"
+    )
+
+    tools = MCPToolPack("http://127.0.0.1:8000/sse")
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    tool_engineer = (
+        await ToolAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tools)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="看下这个页面: https://www.cnblogs.com/fnng/p/18744210",  ##配合 mcp-server-fetch 使用
+        # message="有多少个文件", ## 配合server-filesystem 这个mcp使用
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("test456"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/plugin_agent_dialogue_example.py

@@ -0,0 +1,68 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export SILICONFLOW_API_KEY=sk-xx
+        export SILICONFLOW_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/plugin_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.tool_assistant_agent import ToolAssistantAgent
+from dbgpt.agent.resource import AutoGPTPluginToolPack
+from dbgpt.configs.model_config import ROOT_PATH
+
+test_plugin_dir = os.path.join(ROOT_PATH, "examples/test_files/plugins")
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+
+    context: AgentContext = AgentContext(
+        conv_id="test456", gpts_app_name="插件对话助手"
+    )
+
+    tools = AutoGPTPluginToolPack(test_plugin_dir)
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    tool_engineer = (
+        await ToolAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tools)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="查询今天成都的天气",
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("test456"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/react_agent_example.py

@@ -0,0 +1,126 @@
+import asyncio
+import logging
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+
+from typing_extensions import Annotated, Doc
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    HybridMemory,
+    LLMConfig,
+    LongTermMemory,
+    SensoryMemory,
+    ShortTermMemory,
+    UserProxyAgent,
+)
+from dbgpt.agent.expand.actions.react_action import ReActAction, Terminate
+from dbgpt.agent.expand.react_agent import ReActAgent
+from dbgpt.agent.resource import ToolPack, tool
+from dbgpt.rag.embedding import OpenAPIEmbeddings
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+)
+
+
+@tool
+def simple_calculator(first_number: int, second_number: int, operator: str) -> float:
+    """Simple calculator tool. Just support +, -, *, /.
+    When users need to do numerical calculations, you must use this tool to calculate, \
+    and you are not allowed to directly infer calculation results from user input or \
+    external observations.
+    """
+    if isinstance(first_number, str):
+        first_number = int(first_number)
+    if isinstance(second_number, str):
+        second_number = int(second_number)
+    if operator == "+":
+        return first_number + second_number
+    elif operator == "-":
+        return first_number - second_number
+    elif operator == "*":
+        return first_number * second_number
+    elif operator == "/":
+        return first_number / second_number
+    else:
+        raise ValueError(f"Invalid operator: {operator}")
+
+
+@tool
+def count_directory_files(path: Annotated[str, Doc("The directory path")]) -> int:
+    """Count the number of files in a directory."""
+    if not os.path.isdir(path):
+        raise ValueError(f"Invalid directory path: {path}")
+    return len(os.listdir(path))
+
+
+async def main():
+    from dbgpt.model import AutoLLMClient
+
+    llm_client = AutoLLMClient(
+        # provider=os.getenv("LLM_PROVIDER", "proxy/deepseek"),
+        # name=os.getenv("LLM_MODEL_NAME", "deepseek-chat"),
+        provider=os.getenv("LLM_PROVIDER", "proxy/siliconflow"),
+        name=os.getenv("LLM_MODEL_NAME", "Qwen/Qwen2.5-Coder-32B-Instruct"),
+    )
+    short_memory = ShortTermMemory(buffer_size=1)
+    sensor_memory = SensoryMemory()
+    embedding_fn = OpenAPIEmbeddings(
+        api_url="https://api.siliconflow.cn/v1/embeddings",
+        api_key=os.getenv("SILICONFLOW_API_KEY"),
+        model_name="BAAI/bge-large-zh-v1.5",
+    )
+    vector_store = ChromaStore(
+        ChromaVectorConfig(persist_path="pilot/data"),
+        name="react_mem",
+        embedding_fn=embedding_fn,
+    )
+    long_memory = LongTermMemory(ThreadPoolExecutor(), vector_store)
+
+    agent_memory = AgentMemory(
+        memory=HybridMemory(datetime.now(), sensor_memory, short_memory, long_memory)
+    )
+    agent_memory.gpts_memory.init(conv_id="test456")
+
+    # It is important to set the temperature to a low value to get a better result
+    context: AgentContext = AgentContext(
+        conv_id="test456", gpts_app_name="ReAct", temperature=0.01
+    )
+
+    tools = ToolPack([simple_calculator, count_directory_files, Terminate()])
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    tool_engineer = (
+        await ReActAgent(max_retry_count=10)
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tools)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="Calculate the product of 10 and 99, and then add 1 to the result, and finally divide the result by 2.",
+    )
+    await user_proxy.initiate_chat(
+        recipient=tool_engineer,
+        reviewer=user_proxy,
+        message="Count the number of files in /tmp",
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("test456"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/retrieve_summary_agent_dialogue_example.py

@@ -0,0 +1,70 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export SILICONFLOW_API_KEY=sk-xx
+        export SILICONFLOW_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/retrieve_summary_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.summary_assistant_agent import SummaryAssistantAgent
+from dbgpt.configs.model_config import ROOT_PATH
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    context: AgentContext = AgentContext(
+        conv_id="retrieve_summarize", gpts_app_name="Summary Assistant"
+    )
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="retrieve_summarize")
+
+    summarizer = (
+        await SummaryAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    user_proxy = UserProxyAgent(memory=agent_memory, agent_context=context)
+
+    paths_urls = [
+        os.path.join(ROOT_PATH, "examples/agents/example_files/Nuclear_power.pdf"),
+        os.path.join(ROOT_PATH, "examples/agents/example_files/Taylor_Swift.pdf"),
+        "https://en.wikipedia.org/wiki/Modern_Family",
+        "https://en.wikipedia.org/wiki/Chernobyl_disaster",
+    ]
+
+    # TODO add a tool to load the pdf and internet files
+    await user_proxy.initiate_chat(
+        recipient=summarizer,
+        reviewer=user_proxy,
+        message=f"I want to summarize advantages of Nuclear Power. You can refer the "
+        f"following file paths and URLs: {paths_urls}",
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("retrieve_summarize"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/sandbox_code_agent_example.py

@@ -0,0 +1,322 @@
+"""Run your code assistant agent in a sandbox environment.
+
+This example demonstrates how to create a code assistant agent that can execute code
+in a sandbox environment. The agent can execute Python and JavaScript code blocks
+and provide the output to the user. The agent can also check the correctness of the
+code execution results and provide feedback to the user.
+
+
+You can limit the memory and file system resources available to the code execution
+environment. The code execution environment is isolated from the host system,
+preventing access to the internet and other external resources.
+"""
+
+import asyncio
+import logging
+import os
+from typing import Optional, Tuple
+
+from dbgpt.agent import (
+    Action,
+    ActionOutput,
+    AgentContext,
+    AgentMemory,
+    AgentMemoryFragment,
+    AgentMessage,
+    AgentResource,
+    ConversableAgent,
+    HybridMemory,
+    LLMConfig,
+    ProfileConfig,
+    UserProxyAgent,
+)
+from dbgpt.agent.expand.code_assistant_agent import CHECK_RESULT_SYSTEM_MESSAGE
+from dbgpt.core import ModelMessageRoleType
+from dbgpt.util.code_utils import UNKNOWN, extract_code, infer_lang
+from dbgpt.util.string_utils import str_to_bool
+from dbgpt.util.utils import colored
+from dbgpt.vis.tags.vis_code import Vis, VisCode
+
+logger = logging.getLogger(__name__)
+
+
+class SandboxCodeAction(Action[None]):
+    """Code Action Module."""
+
+    def __init__(self, **kwargs):
+        """Code action init."""
+        super().__init__(**kwargs)
+        self._render_protocol = VisCode()
+        self._code_execution_config = {}
+
+    @property
+    def render_protocol(self) -> Optional[Vis]:
+        """Return the render protocol."""
+        return self._render_protocol
+
+    async def run(
+        self,
+        ai_message: str,
+        resource: Optional[AgentResource] = None,
+        rely_action_out: Optional[ActionOutput] = None,
+        need_vis_render: bool = True,
+        **kwargs,
+    ) -> ActionOutput:
+        """Perform the action."""
+        try:
+            code_blocks = extract_code(ai_message)
+            if len(code_blocks) < 1:
+                logger.info(
+                    f"No executable code found in answer,{ai_message}",
+                )
+                return ActionOutput(
+                    is_exe_success=False, content="No executable code found in answer."
+                )
+            elif len(code_blocks) > 1 and code_blocks[0][0] == UNKNOWN:
+                # found code blocks, execute code and push "last_n_messages" back
+                logger.info(
+                    f"Missing available code block type, unable to execute code,"
+                    f"{ai_message}",
+                )
+                return ActionOutput(
+                    is_exe_success=False,
+                    content="Missing available code block type, "
+                    "unable to execute code.",
+                )
+            exitcode, logs = await self.execute_code_blocks(code_blocks)
+            exit_success = exitcode == 0
+
+            content = (
+                logs
+                if exit_success
+                else f"exitcode: {exitcode} (execution failed)\n {logs}"
+            )
+
+            param = {
+                "exit_success": exit_success,
+                "language": code_blocks[0][0],
+                "code": code_blocks,
+                "log": logs,
+            }
+            if not self.render_protocol:
+                raise NotImplementedError("The render_protocol should be implemented.")
+            view = await self.render_protocol.display(content=param)
+            return ActionOutput(
+                is_exe_success=exit_success,
+                content=content,
+                view=view,
+                thoughts=ai_message,
+                observations=content,
+            )
+        except Exception as e:
+            logger.exception("Code Action Run Failed！")
+            return ActionOutput(
+                is_exe_success=False, content="Code execution exception，" + str(e)
+            )
+
+    async def execute_code_blocks(self, code_blocks):
+        """Execute the code blocks and return the result."""
+        from lyric import (
+            PyTaskFsConfig,
+            PyTaskMemoryConfig,
+            PyTaskResourceConfig,
+        )
+
+        from dbgpt.util.code.server import get_code_server
+
+        fs = PyTaskFsConfig(
+            preopens=[
+                # Mount the /tmp directory to the /tmp directory in the sandbox
+                # Directory permissions are set to 3 (read and write)
+                # File permissions are set to 3 (read and write)
+                ("/tmp", "/tmp", 3, 3),
+                # Mount the current directory to the /home directory in the sandbox
+                # Directory and file permissions are set to 1 (read)
+                (".", "/home", 1, 1),
+            ]
+        )
+        memory = PyTaskMemoryConfig(memory_limit=50 * 1024 * 1024)  # 50MB in bytes
+        resources = PyTaskResourceConfig(
+            fs=fs,
+            memory=memory,
+            env_vars=[
+                ("TEST_ENV", "hello, im an env var"),
+                ("TEST_ENV2", "hello, im another env var"),
+            ],
+        )
+
+        code_server = await get_code_server()
+        logs_all = ""
+        exitcode = -1
+        for i, code_block in enumerate(code_blocks):
+            lang, code = code_block
+            if not lang:
+                lang = infer_lang(code)
+            print(
+                colored(
+                    f"\n>>>>>>>> EXECUTING CODE BLOCK {i} "
+                    f"(inferred language is {lang})...",
+                    "red",
+                ),
+                flush=True,
+            )
+            if lang in ["python", "Python"]:
+                result = await code_server.exec(code, "python", resources=resources)
+                exitcode = result.exit_code
+                logs = result.logs
+            elif lang in ["javascript", "JavaScript"]:
+                result = await code_server.exec(code, "javascript", resources=resources)
+                exitcode = result.exit_code
+                logs = result.logs
+            else:
+                # In case the language is not supported, we return an error message.
+                exitcode, logs = (
+                    1,
+                    f"unknown language {lang}",
+                )
+
+            logs_all += "\n" + logs
+            if exitcode != 0:
+                return exitcode, logs_all
+        return exitcode, logs_all
+
+
+class SandboxCodeAssistantAgent(ConversableAgent):
+    """Code Assistant Agent."""
+
+    profile: ProfileConfig = ProfileConfig(
+        name="Turing",
+        role="CodeEngineer",
+        goal=(
+            "Solve tasks using your coding and language skills.\n"
+            "In the following cases, suggest python code (in a python coding block) or "
+            "javascript for the user to execute.\n"
+            "    1. When you need to collect info, use the code to output the info you "
+            "need, for example, get the current date/time, check the "
+            "operating system. After sufficient info is printed and the task is ready "
+            "to be solved based on your language skill, you can solve the task by "
+            "yourself.\n"
+            "    2. When you need to perform some task with code, use the code to "
+            "perform the task and output the result. Finish the task smartly."
+        ),
+        constraints=[
+            "The user cannot provide any other feedback or perform any other "
+            "action beyond executing the code you suggest. The user can't modify "
+            "your code. So do not suggest incomplete code which requires users to "
+            "modify. Don't use a code block if it's not intended to be executed "
+            "by the user.Don't ask users to copy and paste results. Instead, "
+            "the 'Print' function must be used for output when relevant.",
+            "When using code, you must indicate the script type in the code block. "
+            "Please don't include multiple code blocks in one response.",
+            "If you receive user input that indicates an error in the code "
+            "execution, fix the error and output the complete code again. It is "
+            "recommended to use the complete code rather than partial code or "
+            "code changes. If the error cannot be fixed, or the task is not "
+            "resolved even after the code executes successfully, analyze the "
+            "problem, revisit your assumptions, gather additional information you "
+            "need from historical conversation records, and consider trying a "
+            "different approach.",
+            "Unless necessary, give priority to solving problems with python code.",
+            "The output content of the 'print' function will be passed to other "
+            "LLM agents as dependent data. Please control the length of the "
+            "output content of the 'print' function. The 'print' function only "
+            "outputs part of the key data information that is relied on, "
+            "and is as concise as possible.",
+            "Your code will by run in a sandbox environment(supporting python and "
+            "javascript), which means you can't access the internet or use any "
+            "libraries that are not in standard library.",
+            "It is prohibited to fabricate non-existent data to achieve goals.",
+        ],
+        desc=(
+            "Can independently write and execute python/shell code to solve various"
+            " problems"
+        ),
+    )
+
+    def __init__(self, **kwargs):
+        """Create a new CodeAssistantAgent instance."""
+        super().__init__(**kwargs)
+        self._init_actions([SandboxCodeAction])
+
+    async def correctness_check(
+        self, message: AgentMessage
+    ) -> Tuple[bool, Optional[str]]:
+        """Verify whether the current execution results meet the target expectations."""
+        task_goal = message.current_goal
+        action_report = message.action_report
+        if not action_report:
+            return False, "No execution solution results were checked"
+        check_result, model = await self.thinking(
+            messages=[
+                AgentMessage(
+                    role=ModelMessageRoleType.HUMAN,
+                    content="Please understand the following task objectives and "
+                    f"results and give your judgment:\n"
+                    f"Task goal: {task_goal}\n"
+                    f"Execution Result: {action_report.content}",
+                )
+            ],
+            prompt=CHECK_RESULT_SYSTEM_MESSAGE,
+        )
+        success = str_to_bool(check_result)
+        fail_reason = None
+        if not success:
+            fail_reason = (
+                f"Your answer was successfully executed by the agent, but "
+                f"the goal cannot be completed yet. Please regenerate based on the "
+                f"failure reason:{check_result}"
+            )
+        return success, fail_reason
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    context: AgentContext = AgentContext(conv_id="test123")
+
+    # TODO Embedding and Rerank model refactor
+    from dbgpt.rag.embedding import OpenAPIEmbeddings
+
+    silicon_embeddings = OpenAPIEmbeddings(
+        api_url=os.getenv("SILICONFLOW_API_BASE") + "/embeddings",
+        api_key=os.getenv("SILICONFLOW_API_KEY"),
+        model_name="BAAI/bge-large-zh-v1.5",
+    )
+    agent_memory = AgentMemory(
+        HybridMemory[AgentMemoryFragment].from_chroma(
+            embeddings=silicon_embeddings,
+        )
+    )
+    agent_memory.gpts_memory.init("test123")
+
+    coder = (
+        await SandboxCodeAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    user_proxy = await UserProxyAgent().bind(context).bind(agent_memory).build()
+
+    # First case: The user asks the agent to calculate 321 * 123
+    await user_proxy.initiate_chat(
+        recipient=coder,
+        reviewer=user_proxy,
+        message="计算下321 * 123等于多少",
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=coder,
+        reviewer=user_proxy,
+        message="Calculate 100 * 99, must use javascript code block",
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/single_agent_dialogue_example.py

@@ -0,0 +1,59 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export OPENAI_API_KEY=sk-xx
+        export OPENAI_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/single_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.code_assistant_agent import CodeAssistantAgent
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+
+    context: AgentContext = AgentContext(conv_id="test123", gpts_app_name="代码助手")
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test123")
+    try:
+        coder = (
+            await CodeAssistantAgent()
+            .bind(context)
+            .bind(LLMConfig(llm_client=llm_client))
+            .bind(agent_memory)
+            .build()
+        )
+
+        user_proxy = await UserProxyAgent().bind(context).bind(agent_memory).build()
+
+        await user_proxy.initiate_chat(
+            recipient=coder,
+            reviewer=user_proxy,
+            message="计算下321 * 123等于多少",  # 用python代码的方式计算下321 * 123等于多少
+            # message="download data from https://raw.githubusercontent.com/uwdata/draco/master/data/cars.csv and plot a visualization that tells us about the relationship between weight and horsepower. Save the plot to a file. Print the fields in a dataset before visualizing it.",
+        )
+    finally:
+        agent_memory.gpts_memory.clear(conv_id="test123")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/single_summary_agent_dialogue_example.py

@@ -0,0 +1,143 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export OPENAI_API_KEY=sk-xx
+        export OPENAI_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/single_summary_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.summary_assistant_agent import SummaryAssistantAgent
+
+
+async def summary_example_with_success():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    context: AgentContext = AgentContext(conv_id="summarize")
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="summarize")
+
+    summarizer = (
+        await SummaryAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    await user_proxy.initiate_chat(
+        recipient=summarizer,
+        reviewer=user_proxy,
+        message="""I want to summarize advantages of Nuclear Power according to the following content.
+
+            Nuclear power in space is the use of nuclear power in outer space, typically either small fission systems or radioactive decay for electricity or heat. Another use is for scientific observation, as in a Mössbauer spectrometer. The most common type is a radioisotope thermoelectric generator, which has been used on many space probes and on crewed lunar missions. Small fission reactors for Earth observation satellites, such as the TOPAZ nuclear reactor, have also been flown.[1] A radioisotope heater unit is powered by radioactive decay and can keep components from becoming too cold to function, potentially over a span of decades.[2]
+
+            The United States tested the SNAP-10A nuclear reactor in space for 43 days in 1965,[3] with the next test of a nuclear reactor power system intended for space use occurring on 13 September 2012 with the Demonstration Using Flattop Fission (DUFF) test of the Kilopower reactor.[4]
+
+            After a ground-based test of the experimental 1965 Romashka reactor, which used uranium and direct thermoelectric conversion to electricity,[5] the USSR sent about 40 nuclear-electric satellites into space, mostly powered by the BES-5 reactor. The more powerful TOPAZ-II reactor produced 10 kilowatts of electricity.[3]
+
+            Examples of concepts that use nuclear power for space propulsion systems include the nuclear electric rocket (nuclear powered ion thruster(s)), the radioisotope rocket, and radioisotope electric propulsion (REP).[6] One of the more explored concepts is the nuclear thermal rocket, which was ground tested in the NERVA program. Nuclear pulse propulsion was the subject of Project Orion.[7]
+
+            Regulation and hazard prevention[edit]
+            After the ban of nuclear weapons in space by the Outer Space Treaty in 1967, nuclear power has been discussed at least since 1972 as a sensitive issue by states.[8] Particularly its potential hazards to Earth's environment and thus also humans has prompted states to adopt in the U.N. General Assembly the Principles Relevant to the Use of Nuclear Power Sources in Outer Space (1992), particularly introducing safety principles for launches and to manage their traffic.[8]
+
+            Benefits
+
+            Both the Viking 1 and Viking 2 landers used RTGs for power on the surface of Mars. (Viking launch vehicle pictured)
+            While solar power is much more commonly used, nuclear power can offer advantages in some areas. Solar cells, although efficient, can only supply energy to spacecraft in orbits where the solar flux is sufficiently high, such as low Earth orbit and interplanetary destinations close enough to the Sun. Unlike solar cells, nuclear power systems function independently of sunlight, which is necessary for deep space exploration. Nuclear-based systems can have less mass than solar cells of equivalent power, allowing more compact spacecraft that are easier to orient and direct in space. In the case of crewed spaceflight, nuclear power concepts that can power both life support and propulsion systems may reduce both cost and flight time.[9]
+
+            Selected applications and/or technologies for space include:
+
+            Radioisotope thermoelectric generator
+            Radioisotope heater unit
+            Radioisotope piezoelectric generator
+            Radioisotope rocket
+            Nuclear thermal rocket
+            Nuclear pulse propulsion
+            Nuclear electric rocket
+            """,
+    )
+
+    # dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("summarize"))
+
+
+async def summary_example_with_faliure():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    context: AgentContext = AgentContext(conv_id="summarize")
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="summarize")
+
+    summarizer = (
+        await SummaryAssistantAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .build()
+    )
+
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    # Test the failure example
+    await user_proxy.initiate_chat(
+        recipient=summarizer,
+        reviewer=user_proxy,
+        message="""I want to summarize advantages of Nuclear Power according to the following content.
+
+            Taylor Swift is an American singer-songwriter and actress who is one of the most prominent and successful figures in the music industry. She was born on December 13, 1989, in Reading, Pennsylvania, USA. Taylor Swift gained widespread recognition for her narrative songwriting style, which often draws from her personal experiences and relationships.
+
+            Swift's career began in country music, and her self-titled debut album was released in 2006. She quickly became a sensation in the country music scene with hits like "Tim McGraw" and "Teardrops on My Guitar." However, it was her transition to pop music with albums like "Fearless," "Speak Now," and "Red" that catapulted her to international superstardom.
+
+            Throughout her career, Taylor Swift has won numerous awards, including multiple Grammy Awards. Her albums consistently top charts, and her songs resonate with a wide audience due to their relatable lyrics and catchy melodies. Some of her most famous songs include "Love Story," "Blank Space," "Shake It Off," "Bad Blood," and "Lover."
+
+            Beyond music, Taylor Swift has ventured into acting with roles in movies like "Valentine's Day" and "The Giver." She is also known for her philanthropic efforts and her willingness to use her platform to advocate for various causes.
+
+            Taylor Swift is not only a successful artist but also an influential cultural icon known for her evolving musical style, storytelling abilities, and her impact on the entertainment industry.
+            """,
+    )
+
+    print(await agent_memory.gpts_memory.app_link_chat_message("summarize"))
+
+
+if __name__ == "__main__":
+    print(
+        "\033[92m=======================Start The Summary Assistant with Successful Results==================\033[0m"
+    )
+    asyncio.run(summary_example_with_success())
+    print(
+        "\033[92m=======================The Summary Assistant with Successful Results Ended==================\n\n\033[91m"
+    )
+
+    print(
+        "\033[91m=======================Start The Summary Assistant with Fail Results==================\033[91m"
+    )
+    asyncio.run(summary_example_with_faliure())
+    print(
+        "\033[91m=======================The Summary Assistant with Fail Results Ended==================\033[91m"
+    )

examples/agents/skill_agent_example.py

@@ -0,0 +1,209 @@
+"""Example: Agent with Skill loading mechanism.
+
+This example demonstrates how to use the SKILL loading mechanism
+with DB-GPT agents.
+"""
+
+import asyncio
+import logging
+import os
+
+from dbgpt.agent import (
+    AgentContext,
+    AgentMemory,
+    ConversableAgent,
+    LLMConfig,
+    UserProxyAgent,
+)
+from dbgpt.agent.core.action.base import ActionOutput
+from dbgpt.agent.core.profile.base import ProfileConfig
+from dbgpt.agent.expand.actions.react_action import Terminate
+from dbgpt.agent.expand.actions.tool_action import ToolAction
+from dbgpt.agent.resource import ToolPack, tool
+from dbgpt.agent.skill import (
+    Skill,
+    SkillBuilder,
+    SkillLoader,
+    SkillManager,
+    SkillType,
+    get_skill_manager,
+    initialize_skill,
+)
+from dbgpt.component import SystemApp
+from dbgpt.model import AutoLLMClient
+
+
+@tool
+def calculate(expression: str) -> str:
+    """Calculate a mathematical expression.
+
+    Args:
+        expression: The mathematical expression to calculate (e.g., "1 + 2 * 3").
+
+    Returns:
+        The result of the calculation.
+    """
+    try:
+        result = eval(expression, {"__builtins__": {}}, {})
+        return str(result)
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+# Use ToolAction as the agent's action module to enable tool usage.
+
+
+class MathSkillAgent(ConversableAgent):
+    """Agent with math skill.
+
+    Supports binding a Skill instance using `.bind(skill_instance)` so that
+    skills can be provided either in the constructor or later via `bind`.
+    """
+
+    def __init__(self, skill: Skill | None = None, **kwargs):
+        """Initialize the agent with an optional skill."""
+        super().__init__(**kwargs)
+        self._skill = skill
+
+    @property
+    def skill(self) -> Skill:
+        """Return the skill if bound, otherwise raise a helpful error."""
+        if not getattr(self, "_skill", None):
+            raise ValueError(
+                "Skill not bound to agent. Call .bind(skill) before build()."
+            )
+        return self._skill
+
+
+async def main():
+    """Main function."""
+    system_app = SystemApp()
+
+    # Initialize skill manager
+    initialize_skill(system_app)
+    skill_manager = get_skill_manager(system_app)
+
+    # First try to load a SKILL.md from skills/claude
+    loader = SkillLoader()
+    loaded_from_file = None
+    try:
+        loaded_from_file = loader.load_skill_from_file(
+            "/Users/chenketing.ckt/Desktop/project/DB-GPT/skills/claude/math_assistant/SKILL.md"
+        )
+        if loaded_from_file:
+            # register loaded skill (demonstrate file-based loading path)
+            skill_manager.register_skill(
+                skill_instance=loaded_from_file, name=loaded_from_file.metadata.name
+            )
+            print(f"Loaded SKILL.md skill: {loaded_from_file.metadata.name}")
+    except Exception:
+        loaded_from_file = None
+
+    # If SKILL.md not available, fall back to building programmatically
+    if not loaded_from_file:
+        math_skill = (
+            SkillBuilder(
+                name="math_assistant", description="Mathematical calculation assistant"
+            )
+            .with_version("1.0.0")
+            .with_author("DB-GPT Team")
+            .with_skill_type(SkillType.Chat)
+            .with_tags(["math", "calculation"])
+            .with_prompt_template(
+                "You are a mathematical assistant. Help users with calculations and "
+                "explain mathematical concepts clearly. Use the calculate tool for "
+                "computations."
+            )
+            .with_required_tool("calculate")
+            .build()
+        )
+
+        # Register the skill
+        skill_manager.register_skill(
+            skill_instance=math_skill,
+            name="math_assistant",
+        )
+        loaded_skill = skill_manager.get_skill(name="math_assistant")
+        if loaded_skill:
+            print(f"Loaded programmatic skill: {loaded_skill.metadata.name}")
+    else:
+        loaded_skill = loaded_from_file
+
+    # Create an LLM client similar to react_agent_example so the example can
+    # interact with a real model provider. Configure via environment variables.
+    logging.basicConfig(level=logging.INFO)
+    llm_client = AutoLLMClient(
+        provider=os.getenv("LLM_PROVIDER", "proxy/siliconflow"),
+        name=os.getenv("LLM_MODEL_NAME", "Qwen/Qwen2.5-Coder-32B-Instruct"),
+    )
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="skill_test_001")
+
+    context: AgentContext = AgentContext(
+        conv_id="skill_test_001", gpts_app_name="Math Skill Agent"
+    )
+
+    # Create agent with skill (provide ProfileConfig required by agent role)
+    profile = ProfileConfig(name="MathAssistant", role="math_assistant")
+    # Instantiate agent with profile and skill
+    # If ResourceManager/SkillResource is not initialized when running example
+    # standalone, the agent will still work as we bind tools directly. However
+    # for completeness, register SkillResource with the global ResourceManager
+    # so other components (ToolAction) can resolve skills if needed.
+    try:
+        from dbgpt.agent.resource.manage import (
+            get_resource_manager,
+            initialize_resource,
+        )
+        from dbgpt.agent.resource.skill_resource import SkillResource
+
+        initialize_resource(system_app)
+        rm = get_resource_manager(system_app)
+        rm.register_resource(SkillResource, resource_type=None)
+    except Exception as e:
+        print(e)
+        # ignore registration failures in example runs
+        pass
+
+    # Create a ToolPack from the calculate tool and bind it as the agent's resource
+    tool_packs = ToolPack.from_resource([calculate, Terminate()])
+    tool_pack = tool_packs[0]
+
+    # Create agent and bind the loaded skill via .bind(skill) so skills can be
+    # injected at runtime rather than only via constructor.
+    math_agent = (
+        await MathSkillAgent(profile=profile)
+        .bind(loaded_skill)
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(agent_memory)
+        .bind(tool_pack)
+        .bind(ToolAction)
+        .build()
+    )
+
+    print("Math Skill Agent created successfully!")
+    print(f"Skill: {math_agent.skill.metadata.name}")
+    print(f"Skill type: {math_agent.skill.metadata.skill_type}")
+    print(f"Required tools: {math_agent.skill.required_tools}")
+
+    # Create a user proxy to interact with the agent (same pattern as react example)
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    # Example interactions
+    await user_proxy.initiate_chat(
+        recipient=math_agent,
+        reviewer=user_proxy,
+        message="Compute 10 * 99 using the calculate tool and return the numeric result.",
+    )
+
+    # Show dbgpt-vis link messages
+    try:
+        print(await agent_memory.gpts_memory.app_link_chat_message("skill_test_001"))
+    except Exception:
+        pass
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/sql_agent_dialogue_example.py

@@ -0,0 +1,67 @@
+"""Agents: single agents about CodeAssistantAgent?
+
+Examples:
+
+    Execute the following command in the terminal:
+    Set env params.
+    .. code-block:: shell
+
+        export OPENAI_API_KEY=sk-xx
+        export OPENAI_API_BASE=https://xx:80/v1
+
+    run example.
+    ..code-block:: shell
+        python examples/agents/single_agent_dialogue_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.agent import AgentContext, AgentMemory, LLMConfig, UserProxyAgent
+from dbgpt.agent.expand.data_scientist_agent import DataScientistAgent
+from dbgpt.agent.resource import SQLiteDBResource
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt.util.tracer import initialize_tracer
+
+test_plugin_dir = os.path.join(ROOT_PATH, "test_files")
+
+initialize_tracer("/tmp/agent_trace.jsonl", create_system_app=True)
+
+
+async def main():
+    from dbgpt.model.proxy.llms.siliconflow import SiliconFlowLLMClient
+
+    llm_client = SiliconFlowLLMClient(
+        model_alias=os.getenv(
+            "SILICONFLOW_MODEL_VERSION", "Qwen/Qwen2.5-Coder-32B-Instruct"
+        ),
+    )
+    context: AgentContext = AgentContext(conv_id="test456")
+
+    agent_memory = AgentMemory()
+    agent_memory.gpts_memory.init(conv_id="test456")
+
+    sqlite_resource = SQLiteDBResource("SQLite Database", f"{test_plugin_dir}/dbgpt.db")
+    user_proxy = await UserProxyAgent().bind(agent_memory).bind(context).build()
+
+    sql_boy = (
+        await DataScientistAgent()
+        .bind(context)
+        .bind(LLMConfig(llm_client=llm_client))
+        .bind(sqlite_resource)
+        .bind(agent_memory)
+        .build()
+    )
+
+    await user_proxy.initiate_chat(
+        recipient=sql_boy,
+        reviewer=user_proxy,
+        message="当前库有那些表",
+    )
+
+    ## dbgpt-vis message infos
+    print(await agent_memory.gpts_memory.app_link_chat_message("test456"))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/awel/data_analyst_assistant.py

@@ -0,0 +1,387 @@
+"""AWEL: Data analyst assistant.
+
+    DB-GPT will automatically load and execute the current file after startup.
+
+    Examples:
+
+        .. code-block:: shell
+
+            # Run this file in your terminal with dev mode.
+            # First terminal
+            export OPENAI_API_KEY=xxx
+            export OPENAI_API_BASE=https://api.openai.com/v1
+            python examples/awel/simple_chat_history_example.py
+
+
+        Code fix command, return no streaming response
+
+        .. code-block:: shell
+
+            # Open a new terminal
+            # Second terminal
+
+            DBGPT_SERVER="http://127.0.0.1:5555"
+            MODEL="gpt-3.5-turbo"
+            # Fist round
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/data_analyst/copilot \
+            -H "Content-Type: application/json" -d '{
+                "command": "dbgpt_awel_data_analyst_code_fix",
+                "model": "'"$MODEL"'",
+                "stream": false,
+                "context": {
+                    "conv_uid": "uuid_conv_copilot_1234",
+                    "chat_mode": "chat_with_code"
+                },
+                "messages": "SELECT * FRM orders WHERE order_amount > 500;"
+            }'
+
+"""
+
+import logging
+import os
+from functools import cache
+from typing import Any, Dict, List, Optional
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core import (
+    ChatPromptTemplate,
+    HumanPromptTemplate,
+    MessagesPlaceholder,
+    ModelMessage,
+    ModelRequest,
+    ModelRequestContext,
+    PromptManager,
+    PromptTemplate,
+    SystemPromptTemplate,
+)
+from dbgpt.core.awel import (
+    DAG,
+    BranchJoinOperator,
+    HttpTrigger,
+    JoinOperator,
+    MapOperator,
+)
+from dbgpt.core.operators import (
+    BufferedConversationMapperOperator,
+    HistoryDynamicPromptBuilderOperator,
+    LLMBranchOperator,
+)
+from dbgpt.model.operators import (
+    LLMOperator,
+    OpenAIStreamingOutputOperator,
+    StreamingLLMOperator,
+)
+from dbgpt_serve.conversation.operators import ServePreChatHistoryLoadOperator
+
+logger = logging.getLogger(__name__)
+
+PROMPT_LANG_ZH = "zh"
+PROMPT_LANG_EN = "en"
+
+CODE_DEFAULT = "dbgpt_awel_data_analyst_code_default"
+CODE_FIX = "dbgpt_awel_data_analyst_code_fix"
+CODE_PERF = "dbgpt_awel_data_analyst_code_perf"
+CODE_EXPLAIN = "dbgpt_awel_data_analyst_code_explain"
+CODE_COMMENT = "dbgpt_awel_data_analyst_code_comment"
+CODE_TRANSLATE = "dbgpt_awel_data_analyst_code_translate"
+
+CODE_DEFAULT_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师。
+你可以根据最佳实践来优化代码, 也可以对代码进行修复, 解释, 添加注释, 以及将代码翻译成其他语言。"""
+CODE_DEFAULT_TEMPLATE_EN = """As an experienced data warehouse developer and data analyst.
+You can optimize the code according to best practices, or fix, explain, add comments to the code, 
+and you can also translate the code into other languages.
+"""
+
+CODE_FIX_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师，
+这里有一段 {language} 代码。请按照最佳实践检查代码，找出并修复所有错误。请给出修复后的代码，并且提供对您所做的每一行更正的逐行解释，请使用和用户相同的语言进行回答。"""
+CODE_FIX_TEMPLATE_EN = """As an experienced data warehouse developer and data analyst, 
+here is a snippet of code of {language}. Please review the code following best practices to identify and fix all errors. 
+Provide the corrected code and include a line-by-line explanation of all the fixes you've made, please use the same language as the user."""
+
+CODE_PERF_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师，这里有一段 {language} 代码。
+请你按照最佳实践来优化这段代码。请在代码中加入注释点明所做的更改，并解释每项优化的原因，以便提高代码的维护性和性能，请使用和用户相同的语言进行回答。"""
+CODE_PERF_TEMPLATE_EN = """As an experienced data warehouse developer and data analyst, 
+you are provided with a snippet of code of {language}. Please optimize the code according to best practices. 
+Include comments to highlight the changes made and explain the reasons for each optimization for better maintenance and performance, 
+please use the same language as the user."""
+CODE_EXPLAIN_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师，
+现在给你的是一份 {language} 代码。请你逐行解释代码的含义，请使用和用户相同的语言进行回答。"""
+
+CODE_EXPLAIN_TEMPLATE_EN = """As an experienced data warehouse developer and data analyst, 
+you are provided with a snippet of code of {language}. Please explain the meaning of the code line by line, 
+please use the same language as the user."""
+
+CODE_COMMENT_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师，现在给你的是一份 {language} 代码。
+请你为每一行代码添加注释，解释每个部分的作用，请使用和用户相同的语言进行回答。"""
+
+CODE_COMMENT_TEMPLATE_EN = """As an experienced Data Warehouse Developer and Data Analyst. 
+Below is a snippet of code written in {language}. 
+Please provide line-by-line comments explaining what each section of the code does, please use the same language as the user."""
+
+CODE_TRANSLATE_TEMPLATE_ZH = """作为一名经验丰富的数据仓库开发者和数据分析师，现在手头有一份用{source_language}语言编写的代码片段。
+请你将这段代码准确无误地翻译成{target_language}语言，确保语法和功能在翻译后的代码中得到正确体现，请使用和用户相同的语言进行回答。"""
+CODE_TRANSLATE_TEMPLATE_EN = """As an experienced data warehouse developer and data analyst, 
+you're presented with a snippet of code written in {source_language}. 
+Please translate this code into {target_language} ensuring that the syntax and functionalities are accurately reflected in the translated code, 
+please use the same language as the user."""
+
+
+class ReqContext(BaseModel):
+    user_name: Optional[str] = Field(
+        None, description="The user name of the model request."
+    )
+
+    sys_code: Optional[str] = Field(
+        None, description="The system code of the model request."
+    )
+    conv_uid: Optional[str] = Field(
+        None, description="The conversation uid of the model request."
+    )
+    chat_mode: Optional[str] = Field(
+        "chat_with_code", description="The chat mode of the model request."
+    )
+
+
+class TriggerReqBody(BaseModel):
+    messages: str = Field(..., description="User input messages")
+    command: Optional[str] = Field(
+        default=None, description="Command name, None if common chat"
+    )
+    model: Optional[str] = Field(default="gpt-3.5-turbo", description="Model name")
+    stream: Optional[bool] = Field(default=False, description="Whether return stream")
+    language: Optional[str] = Field(default="hive", description="Language")
+    target_language: Optional[str] = Field(
+        default="hive", description="Target language, use in translate"
+    )
+    context: Optional[ReqContext] = Field(
+        default=None, description="The context of the model request."
+    )
+
+
+@cache
+def load_or_save_prompt_template(pm: PromptManager):
+    zh_ext_params = {
+        "chat_scene": "chat_with_code",
+        "sub_chat_scene": "data_analyst",
+        "prompt_type": "common",
+        "prompt_language": PROMPT_LANG_ZH,
+    }
+    en_ext_params = {
+        "chat_scene": "chat_with_code",
+        "sub_chat_scene": "data_analyst",
+        "prompt_type": "common",
+        "prompt_language": PROMPT_LANG_EN,
+    }
+
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_DEFAULT_TEMPLATE_ZH),
+        prompt_name=CODE_DEFAULT,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_DEFAULT_TEMPLATE_EN),
+        prompt_name=CODE_DEFAULT,
+        **en_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_FIX_TEMPLATE_ZH),
+        prompt_name=CODE_FIX,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_FIX_TEMPLATE_EN),
+        prompt_name=CODE_FIX,
+        **en_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_PERF_TEMPLATE_ZH),
+        prompt_name=CODE_PERF,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_PERF_TEMPLATE_EN),
+        prompt_name=CODE_PERF,
+        **en_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_EXPLAIN_TEMPLATE_ZH),
+        prompt_name=CODE_EXPLAIN,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_EXPLAIN_TEMPLATE_EN),
+        prompt_name=CODE_EXPLAIN,
+        **en_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_COMMENT_TEMPLATE_ZH),
+        prompt_name=CODE_COMMENT,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_COMMENT_TEMPLATE_EN),
+        prompt_name=CODE_COMMENT,
+        **en_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_TRANSLATE_TEMPLATE_ZH),
+        prompt_name=CODE_TRANSLATE,
+        **zh_ext_params,
+    )
+    pm.query_or_save(
+        PromptTemplate.from_template(CODE_TRANSLATE_TEMPLATE_EN),
+        prompt_name=CODE_TRANSLATE,
+        **en_ext_params,
+    )
+
+
+class PromptTemplateBuilderOperator(MapOperator[TriggerReqBody, ChatPromptTemplate]):
+    """Build prompt template for chat with code."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._default_prompt_manager = PromptManager()
+
+    async def map(self, input_value: TriggerReqBody) -> ChatPromptTemplate:
+        from dbgpt_serve.prompt.serve import SERVE_APP_NAME as PROMPT_SERVE_APP_NAME
+        from dbgpt_serve.prompt.serve import Serve as PromptServe
+
+        prompt_serve = self.system_app.get_component(
+            PROMPT_SERVE_APP_NAME, PromptServe, default_component=None
+        )
+        if prompt_serve:
+            pm = prompt_serve.prompt_manager
+        else:
+            pm = self._default_prompt_manager
+        load_or_save_prompt_template(pm)
+
+        user_language = self.system_app.config.get_current_lang(default="en")
+        if not input_value.command:
+            # No command, just chat, not include system prompt.
+            default_prompt_list = pm.prefer_query(
+                CODE_DEFAULT, prefer_prompt_language=user_language
+            )
+            default_prompt_template = (
+                default_prompt_list[0].to_prompt_template().template
+            )
+            prompt = ChatPromptTemplate(
+                messages=[
+                    SystemPromptTemplate.from_template(default_prompt_template),
+                    MessagesPlaceholder(variable_name="chat_history"),
+                    HumanPromptTemplate.from_template("{user_input}"),
+                ]
+            )
+            return prompt
+
+        # Query prompt template from prompt manager by command name
+        prompt_list = pm.prefer_query(
+            input_value.command, prefer_prompt_language=user_language
+        )
+        if not prompt_list:
+            error_msg = f"Prompt not found for command {input_value.command}, user_language: {user_language}"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+        prompt_template = prompt_list[0].to_prompt_template()
+
+        return ChatPromptTemplate(
+            messages=[
+                SystemPromptTemplate.from_template(prompt_template.template),
+                MessagesPlaceholder(variable_name="chat_history"),
+                HumanPromptTemplate.from_template("{user_input}"),
+            ]
+        )
+
+
+def parse_prompt_args(req: TriggerReqBody) -> Dict[str, Any]:
+    prompt_args = {"user_input": req.messages}
+    if not req.command:
+        return prompt_args
+    if req.command == CODE_TRANSLATE:
+        prompt_args["source_language"] = req.language
+        prompt_args["target_language"] = req.target_language
+    else:
+        prompt_args["language"] = req.language
+    return prompt_args
+
+
+async def build_model_request(
+    messages: List[ModelMessage], req_body: TriggerReqBody
+) -> ModelRequest:
+    return ModelRequest.build_request(
+        model=req_body.model,
+        messages=messages,
+        context=req_body.context,
+        stream=req_body.stream,
+    )
+
+
+with DAG("dbgpt_awel_data_analyst_assistant") as dag:
+    trigger = HttpTrigger(
+        "/examples/data_analyst/copilot",
+        request_body=TriggerReqBody,
+        methods="POST",
+        streaming_predict_func=lambda x: x.stream,
+    )
+
+    prompt_template_load_task = PromptTemplateBuilderOperator()
+
+    # Load and store chat history
+    chat_history_load_task = ServePreChatHistoryLoadOperator()
+    keep_start_rounds = int(os.getenv("DBGPT_AWEL_DATA_ANALYST_KEEP_START_ROUNDS", 0))
+    keep_end_rounds = int(os.getenv("DBGPT_AWEL_DATA_ANALYST_KEEP_END_ROUNDS", 5))
+    # History transform task, here we keep `keep_start_rounds` round messages of history,
+    # and keep `keep_end_rounds` round messages of history.
+    history_transform_task = BufferedConversationMapperOperator(
+        keep_start_rounds=keep_start_rounds, keep_end_rounds=keep_end_rounds
+    )
+    history_prompt_build_task = HistoryDynamicPromptBuilderOperator(
+        history_key="chat_history"
+    )
+
+    model_request_build_task = JoinOperator(build_model_request)
+
+    # Use BaseLLMOperator to generate response.
+    llm_task = LLMOperator(task_name="llm_task")
+    streaming_llm_task = StreamingLLMOperator(task_name="streaming_llm_task")
+    branch_task = LLMBranchOperator(
+        stream_task_name="streaming_llm_task", no_stream_task_name="llm_task"
+    )
+    model_parse_task = MapOperator(lambda out: out.to_dict())
+    openai_format_stream_task = OpenAIStreamingOutputOperator()
+    result_join_task = BranchJoinOperator()
+    trigger >> prompt_template_load_task >> history_prompt_build_task
+
+    (
+        trigger
+        >> MapOperator(
+            lambda req: ModelRequestContext(
+                conv_uid=req.context.conv_uid,
+                stream=req.stream,
+                user_name=req.context.user_name,
+                sys_code=req.context.sys_code,
+                chat_mode=req.context.chat_mode,
+            )
+        )
+        >> chat_history_load_task
+        >> history_transform_task
+        >> history_prompt_build_task
+    )
+
+    trigger >> MapOperator(parse_prompt_args) >> history_prompt_build_task
+
+    history_prompt_build_task >> model_request_build_task
+    trigger >> model_request_build_task
+
+    model_request_build_task >> branch_task
+    # The branch of no streaming response.
+    (branch_task >> llm_task >> model_parse_task >> result_join_task)
+    # The branch of streaming response.
+    (branch_task >> streaming_llm_task >> openai_format_stream_task >> result_join_task)
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag])
+    else:
+        pass

examples/awel/simple_chat_dag_example.py

@@ -0,0 +1,58 @@
+"""AWEL: Simple chat dag example
+
+    DB-GPT will automatically load and execute the current file after startup.
+
+    Example:
+
+    .. code-block:: shell
+
+        DBGPT_SERVER="http://127.0.0.1:5555"
+        MODEL="gpt-3.5-turbo"
+        curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_chat \
+        -H "Content-Type: application/json" -d '{
+            "model": "'"$MODEL"'",
+            "user_input": "hello"
+        }'
+"""
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core import ModelMessage, ModelRequest
+from dbgpt.core.awel import DAG, HttpTrigger, MapOperator
+from dbgpt.model.operators import LLMOperator
+
+
+class TriggerReqBody(BaseModel):
+    model: str = Field(..., description="Model name")
+    user_input: str = Field(..., description="User input")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, ModelRequest]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> ModelRequest:
+        messages = [ModelMessage.build_human_message(input_value.user_input)]
+        print(f"Receive input value: {input_value}")
+        return ModelRequest.build_request(input_value.model, messages)
+
+
+with DAG("dbgpt_awel_simple_dag_example", tags={"label": "example"}) as dag:
+    # Receive http request and trigger dag to run.
+    trigger = HttpTrigger(
+        "/examples/simple_chat", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    llm_task = LLMOperator(task_name="llm_task")
+    model_parse_task = MapOperator(lambda out: out.to_dict())
+    trigger >> request_handle_task >> llm_task >> model_parse_task
+
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        # Production mode, DB-GPT will automatically load and execute the current file after startup.
+        pass

examples/awel/simple_chat_history_example.py

@@ -0,0 +1,185 @@
+"""AWEL: Simple chat with history example
+
+    DB-GPT will automatically load and execute the current file after startup.
+
+    Examples:
+
+        Call with non-streaming response.
+        .. code-block:: shell
+
+            DBGPT_SERVER="http://127.0.0.1:5555"
+            MODEL="gpt-3.5-turbo"
+            # Fist round
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_history/multi_round/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "context": {
+                    "conv_uid": "uuid_conv_1234"
+                },
+                "messages": "Who is elon musk?"
+            }'
+
+            # Second round
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_history/multi_round/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "context": {
+                    "conv_uid": "uuid_conv_1234"
+                },
+                "messages": "Is he rich?"
+            }'
+
+        Call with streaming response.
+        .. code-block:: shell
+
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_history/multi_round/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "context": {
+                    "conv_uid": "uuid_conv_stream_1234"
+                },
+                "stream": true,
+                "messages": "Who is elon musk?"
+            }'
+
+            # Second round
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_history/multi_round/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "context": {
+                    "conv_uid": "uuid_conv_stream_1234"
+                },
+                "stream": true,
+                "messages": "Is he rich?"
+            }'
+
+
+"""
+
+import logging
+from typing import Dict, List, Optional, Union
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core import (
+    ChatPromptTemplate,
+    HumanPromptTemplate,
+    InMemoryStorage,
+    MessagesPlaceholder,
+    ModelMessage,
+    ModelRequest,
+    ModelRequestContext,
+    SystemPromptTemplate,
+)
+from dbgpt.core.awel import DAG, BranchJoinOperator, HttpTrigger, MapOperator
+from dbgpt.core.operators import (
+    ChatComposerInput,
+    ChatHistoryPromptComposerOperator,
+    LLMBranchOperator,
+)
+from dbgpt.model.operators import (
+    LLMOperator,
+    OpenAIStreamingOutputOperator,
+    StreamingLLMOperator,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ReqContext(BaseModel):
+    user_name: Optional[str] = Field(
+        None, description="The user name of the model request."
+    )
+
+    sys_code: Optional[str] = Field(
+        None, description="The system code of the model request."
+    )
+    conv_uid: Optional[str] = Field(
+        None, description="The conversation uid of the model request."
+    )
+
+
+class TriggerReqBody(BaseModel):
+    messages: Union[str, List[Dict[str, str]]] = Field(
+        ..., description="User input messages"
+    )
+    model: str = Field(..., description="Model name")
+    stream: Optional[bool] = Field(default=False, description="Whether return stream")
+    context: Optional[ReqContext] = Field(
+        default=None, description="The context of the model request."
+    )
+
+
+async def build_model_request(
+    messages: List[ModelMessage], req_body: TriggerReqBody
+) -> ModelRequest:
+    return ModelRequest.build_request(
+        model=req_body.model,
+        messages=messages,
+        context=req_body.context,
+        stream=req_body.stream,
+    )
+
+
+with DAG("dbgpt_awel_simple_chat_history") as multi_round_dag:
+    # Receive http request and trigger dag to run.
+    trigger = HttpTrigger(
+        "/examples/simple_history/multi_round/chat/completions",
+        methods="POST",
+        request_body=TriggerReqBody,
+        streaming_predict_func=lambda req: req.stream,
+    )
+    prompt = ChatPromptTemplate(
+        messages=[
+            SystemPromptTemplate.from_template("You are a helpful chatbot."),
+            MessagesPlaceholder(variable_name="chat_history"),
+            HumanPromptTemplate.from_template("{user_input}"),
+        ]
+    )
+
+    composer_operator = ChatHistoryPromptComposerOperator(
+        prompt_template=prompt,
+        keep_end_rounds=5,
+        storage=InMemoryStorage(),
+        message_storage=InMemoryStorage(),
+    )
+
+    # Use BaseLLMOperator to generate response.
+    llm_task = LLMOperator(task_name="llm_task")
+    streaming_llm_task = StreamingLLMOperator(task_name="streaming_llm_task")
+    branch_task = LLMBranchOperator(
+        stream_task_name="streaming_llm_task", no_stream_task_name="llm_task"
+    )
+    model_parse_task = MapOperator(lambda out: out.to_dict())
+    openai_format_stream_task = OpenAIStreamingOutputOperator()
+    result_join_task = BranchJoinOperator()
+
+    req_handle_task = MapOperator(
+        lambda req: ChatComposerInput(
+            context=ModelRequestContext(
+                conv_uid=req.context.conv_uid, stream=req.stream
+            ),
+            prompt_dict={"user_input": req.messages},
+            model_dict={
+                "model": req.model,
+                "context": req.context,
+                "stream": req.stream,
+            },
+        )
+    )
+
+    trigger >> req_handle_task >> composer_operator >> branch_task
+
+    # The branch of no streaming response.
+    branch_task >> llm_task >> model_parse_task >> result_join_task
+    # The branch of streaming response.
+    branch_task >> streaming_llm_task >> openai_format_stream_task >> result_join_task
+
+if __name__ == "__main__":
+    if multi_round_dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([multi_round_dag], port=5555)
+    else:
+        # Production mode, DB-GPT will automatically load and execute the current file after startup.
+        pass

examples/awel/simple_dag_example.py

@@ -0,0 +1,43 @@
+"""AWEL: Simple dag example
+
+DB-GPT will automatically load and execute the current file after startup.
+
+Example:
+
+.. code-block:: shell
+
+    DBGPT_SERVER="http://127.0.0.1:5555"
+    curl -X GET $DBGPT_SERVER/api/v1/awel/trigger/examples/hello\?name\=zhangsan
+
+"""
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core.awel import DAG, HttpTrigger, MapOperator
+
+
+class TriggerReqBody(BaseModel):
+    name: str = Field(..., description="User name")
+    age: int = Field(18, description="User age")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, str]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> str:
+        print(f"Receive input value: {input_value}")
+        return f"Hello, {input_value.name}, your age is {input_value.age}"
+
+
+with DAG("simple_dag_example") as dag:
+    trigger = HttpTrigger("/examples/hello", request_body=TriggerReqBody)
+    map_node = RequestHandleOperator()
+    trigger >> map_node
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag])
+    else:
+        pass

examples/awel/simple_llm_client_example.py

@@ -0,0 +1,125 @@
+"""AWEL: Simple llm client example
+
+    DB-GPT will automatically load and execute the current file after startup.
+
+    Examples:
+
+        Call with non-streaming response.
+        .. code-block:: shell
+
+            DBGPT_SERVER="http://127.0.0.1:5555"
+            MODEL="gpt-3.5-turbo"
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_client/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "messages": "hello"
+            }'
+
+        Call with streaming response.
+        .. code-block:: shell
+
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_client/chat/completions \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "messages": "hello",
+                "stream": true
+            }'
+
+        Call model and count token.
+         .. code-block:: shell
+
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/simple_client/count_token \
+            -H "Content-Type: application/json" -d '{
+                "model": "'"$MODEL"'",
+                "messages": "hello"
+            }'
+
+"""
+
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core import LLMClient
+from dbgpt.core.awel import DAG, BranchJoinOperator, HttpTrigger, MapOperator
+from dbgpt.core.operators import LLMBranchOperator, RequestBuilderOperator
+from dbgpt.model.operators import (
+    LLMOperator,
+    MixinLLMOperator,
+    OpenAIStreamingOutputOperator,
+    StreamingLLMOperator,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TriggerReqBody(BaseModel):
+    messages: Union[str, List[Dict[str, str]]] = Field(
+        ..., description="User input messages"
+    )
+    model: str = Field(..., description="Model name")
+    stream: Optional[bool] = Field(default=False, description="Whether return stream")
+
+
+class MyModelToolOperator(
+    MixinLLMOperator, MapOperator[TriggerReqBody, Dict[str, Any]]
+):
+    def __init__(self, llm_client: Optional[LLMClient] = None, **kwargs):
+        super().__init__(llm_client)
+        MapOperator.__init__(self, llm_client, **kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict[str, Any]:
+        prompt_tokens = await self.llm_client.count_token(
+            input_value.model, input_value.messages
+        )
+        available_models = await self.llm_client.models()
+        return {
+            "prompt_tokens": prompt_tokens,
+            "available_models": available_models,
+        }
+
+
+with DAG("dbgpt_awel_simple_llm_client_generate") as client_generate_dag:
+    # Receive http request and trigger dag to run.
+    trigger = HttpTrigger(
+        "/examples/simple_client/chat/completions",
+        methods="POST",
+        request_body=TriggerReqBody,
+        streaming_predict_func=lambda req: req.stream,
+    )
+    request_handle_task = RequestBuilderOperator()
+    llm_task = LLMOperator(task_name="llm_task")
+    streaming_llm_task = StreamingLLMOperator(task_name="streaming_llm_task")
+    branch_task = LLMBranchOperator(
+        stream_task_name="streaming_llm_task", no_stream_task_name="llm_task"
+    )
+    model_parse_task = MapOperator(lambda out: out.to_dict())
+    openai_format_stream_task = OpenAIStreamingOutputOperator()
+    result_join_task = BranchJoinOperator()
+
+    trigger >> request_handle_task >> branch_task
+    branch_task >> llm_task >> model_parse_task >> result_join_task
+    branch_task >> streaming_llm_task >> openai_format_stream_task >> result_join_task
+
+
+with DAG("dbgpt_awel_simple_llm_client_count_token") as client_count_token_dag:
+    # Receive http request and trigger dag to run.
+    trigger = HttpTrigger(
+        "/examples/simple_client/count_token",
+        methods="POST",
+        request_body=TriggerReqBody,
+    )
+    model_task = MyModelToolOperator()
+    trigger >> model_task
+
+
+if __name__ == "__main__":
+    if client_generate_dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        dags = [client_generate_dag, client_count_token_dag]
+        setup_dev_environment(dags, port=5555)
+    else:
+        # Production mode, DB-GPT will automatically load and execute the current file after startup.
+        pass

examples/awel/simple_nl_schema_sql_chart_example.py

@@ -0,0 +1,261 @@
+import os
+from typing import Any, Dict, Optional
+
+from pandas import DataFrame
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH
+from dbgpt.core import LLMClient, ModelMessage, ModelMessageRoleType, ModelRequest
+from dbgpt.core.awel import DAG, HttpTrigger, JoinOperator, MapOperator
+from dbgpt.datasource.rdbms.base import RDBMSConnector
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.util.chat_util import run_async_tasks
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteTempConnector
+from dbgpt_ext.rag.operators.schema_linking import SchemaLinkingOperator
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+"""AWEL: Simple nl-schemalinking-sql-chart operator example
+
+    pre-requirements:
+        1. install openai python sdk
+        ```
+            pip install "db-gpt[openai]"
+        ```
+        2. set openai key and base
+        ```
+            export OPENAI_API_KEY={your_openai_key}
+            export OPENAI_API_BASE={your_openai_base}
+        ```
+        or
+        ```
+            import os
+            os.environ["OPENAI_API_KEY"] = {your_openai_key}
+            os.environ["OPENAI_API_BASE"] = {your_openai_base}
+        ```
+        python examples/awel/simple_nl_schema_sql_chart_example.py
+    Examples:
+        ..code-block:: shell
+        curl --location 'http://127.0.0.1:5555/api/v1/awel/trigger/examples/rag/schema_linking' \
+--header 'Content-Type: application/json' \
+--data '{"query": "Statistics of user age in the user table are based on three categories: age is less than 10, age is greater than or equal to 10 and less than or equal to 20, and age is greater than 20. The first column of the statistical results is different ages, and the second column is count."}' 
+"""
+
+INSTRUCTION = (
+    "I want you to act as a SQL terminal in front of an example database, you need only to return the sql "
+    "command to me.Below is an instruction that describes a task, Write a response that appropriately "
+    "completes the request.\n###Instruction:\n{}"
+)
+INPUT_PROMPT = "\n###Input:\n{}\n###Response:"
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(persist_path=PILOT_PATH)
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+def _create_temporary_connection():
+    """Create a temporary database connection for testing."""
+    connect = SQLiteTempConnector.create_temporary_db()
+    connect.create_temp_tables(
+        {
+            "user": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                },
+                "data": [
+                    (1, "Tom", 8),
+                    (2, "Jerry", 16),
+                    (3, "Jack", 18),
+                    (4, "Alice", 20),
+                    (5, "Bob", 22),
+                ],
+            }
+        }
+    )
+    connect.create_temp_tables(
+        {
+            "job": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                },
+                "data": [
+                    (1, "student", 8),
+                    (2, "student", 16),
+                    (3, "student", 18),
+                    (4, "teacher", 20),
+                    (5, "teacher", 22),
+                ],
+            }
+        }
+    )
+    connect.create_temp_tables(
+        {
+            "student": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                    "info": "TEXT",
+                },
+                "data": [
+                    (1, "Andy", 8, "good"),
+                    (2, "Jerry", 16, "bad"),
+                    (3, "Wendy", 18, "good"),
+                    (4, "Spider", 20, "bad"),
+                    (5, "David", 22, "bad"),
+                ],
+            }
+        }
+    )
+    return connect
+
+
+def _prompt_join_fn(query: str, chunks: str) -> str:
+    prompt = INSTRUCTION.format(chunks + INPUT_PROMPT.format(query))
+    return prompt
+
+
+class TriggerReqBody(BaseModel):
+    query: str = Field(..., description="User query")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "query": input_value.query,
+        }
+        print(f"Receive input value: {input_value.query}")
+        return params
+
+
+class SqlGenOperator(MapOperator[Any, Any]):
+    """The Sql Generation Operator."""
+
+    def __init__(self, llm: Optional[LLMClient], model_name: str, **kwargs):
+        """Init the sql generation operator
+        Args:
+           llm (Optional[LLMClient]): base llm
+        """
+        super().__init__(**kwargs)
+        self._llm = llm
+        self._model_name = model_name
+
+    async def map(self, prompt_with_query_and_schema: str) -> str:
+        """generate sql by llm.
+        Args:
+            prompt_with_query_and_schema (str): prompt
+        Return:
+            str: sql
+        """
+
+        messages = [
+            ModelMessage(
+                role=ModelMessageRoleType.SYSTEM, content=prompt_with_query_and_schema
+            )
+        ]
+        request = ModelRequest(model=self._model_name, messages=messages)
+        tasks = [self._llm.generate(request)]
+        output = await run_async_tasks(tasks=tasks, concurrency_limit=1)
+        sql = output[0].text
+        return sql
+
+
+class SqlExecOperator(MapOperator[Any, Any]):
+    """The Sql Execution Operator."""
+
+    def __init__(self, connector: Optional[RDBMSConnector] = None, **kwargs):
+        """
+        Args:
+            connection (Optional[RDBMSConnector]): RDBMSConnector connection
+        """
+        super().__init__(**kwargs)
+        self._connector = connector
+
+    def map(self, sql: str) -> DataFrame:
+        """retrieve table schemas.
+        Args:
+            sql (str): query.
+        Return:
+            str: sql execution
+        """
+        dataframe = self._connector.run_to_df(command=sql, fetch="all")
+        print(f"sql data is \n{dataframe}")
+        return dataframe
+
+
+class ChartDrawOperator(MapOperator[Any, Any]):
+    """The Chart Draw Operator."""
+
+    def __init__(self, **kwargs):
+        """
+        Args:
+        connection (RDBMSConnector): The connection.
+        """
+        super().__init__(**kwargs)
+
+    def map(self, df: DataFrame) -> str:
+        """get sql result in db and draw.
+        Args:
+            sql (str): str.
+        """
+        import matplotlib.pyplot as plt
+
+        category_column = df.columns[0]
+        count_column = df.columns[1]
+        plt.figure(figsize=(8, 4))
+        plt.bar(df[category_column], df[count_column])
+        plt.xlabel(category_column)
+        plt.ylabel(count_column)
+        plt.show()
+        return str(df)
+
+
+with DAG("simple_nl_schema_sql_chart_example") as dag:
+    trigger = HttpTrigger(
+        "/examples/rag/schema_linking", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    query_operator = MapOperator(lambda request: request["query"])
+    llm = (OpenAILLMClient(api_key=os.getenv("OPENAI_API_KEY", "your api key")),)
+    model_name = "gpt-3.5-turbo"
+    retriever_task = SchemaLinkingOperator(
+        connector=_create_temporary_connection(), llm=llm, model_name=model_name
+    )
+    prompt_join_operator = JoinOperator(combine_function=_prompt_join_fn)
+    sql_gen_operator = SqlGenOperator(llm=llm, model_name=model_name)
+    sql_exec_operator = SqlExecOperator(connector=_create_temporary_connection())
+    draw_chart_operator = ChartDrawOperator(connector=_create_temporary_connection())
+    trigger >> request_handle_task >> query_operator >> prompt_join_operator
+    (
+        trigger
+        >> request_handle_task
+        >> query_operator
+        >> retriever_task
+        >> prompt_join_operator
+    )
+    prompt_join_operator >> sql_gen_operator >> sql_exec_operator >> draw_chart_operator
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/awel/simple_rag_rewrite_example.py

@@ -0,0 +1,79 @@
+"""AWEL: Simple rag rewrite example
+
+    pre-requirements:
+        1. install openai python sdk
+        ```
+            pip install openai
+        ```
+        2. set openai key and base
+        ```
+            export OPENAI_API_KEY={your_openai_key}
+            export OPENAI_API_BASE={your_openai_base}
+        ```
+        or
+        ```
+            import os
+            os.environ["OPENAI_API_KEY"] = {your_openai_key}
+            os.environ["OPENAI_API_BASE"] = {your_openai_base}
+        ```
+        python examples/awel/simple_rag_rewrite_example.py
+    Example:
+
+    .. code-block:: shell
+
+        DBGPT_SERVER="http://127.0.0.1:5555"
+        curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/rag/rewrite \
+        -H "Content-Type: application/json" -d '{
+            "query": "compare curry and james",
+            "context":"steve curry and lebron james are nba all-stars"
+        }'
+"""
+
+import os
+from typing import Dict
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core.awel import DAG, HttpTrigger, MapOperator
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.operators import QueryRewriteOperator
+
+
+class TriggerReqBody(BaseModel):
+    query: str = Field(..., description="User query")
+    context: str = Field(..., description="context")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "query": input_value.query,
+            "context": input_value.context,
+        }
+        print(f"Receive input value: {input_value}")
+        return params
+
+
+with DAG("dbgpt_awel_simple_rag_rewrite_example") as dag:
+    trigger = HttpTrigger(
+        "/examples/rag/rewrite", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    # build query rewrite operator
+    rewrite_task = QueryRewriteOperator(
+        llm_client=OpenAILLMClient(api_key=os.getenv("OPENAI_API_KEY", "your api key")),
+        nums=2,
+    )
+    trigger >> request_handle_task >> rewrite_task
+
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/awel/simple_rag_summary_example.py

@@ -0,0 +1,90 @@
+"""AWEL:
+This example shows how to use AWEL to build a simple rag summary example.
+    pre-requirements:
+        1. install openai python sdk
+        ```
+            pip install openai
+        ```
+        2. set openai key and base
+        ```
+            export OPENAI_API_KEY={your_openai_key}
+            export OPENAI_API_BASE={your_openai_base}
+            export MODEL_NAME={LLM_MODEL_NAME}
+        ```
+        or
+        ```
+            import os
+            os.environ["OPENAI_API_KEY"] = {your_openai_key}
+            os.environ["OPENAI_API_BASE"] = {your_openai_base}
+        ```
+        python examples/awel/simple_rag_summary_example.py
+    Example:
+
+    .. code-block:: shell
+
+        curl -X POST http://127.0.0.1:5555/api/v1/awel/trigger/examples/rag/summary \
+        -H "Content-Type: application/json" -d '{
+            "url": "http://docs.dbgpt.cn/docs/awel/"
+        }'
+"""
+
+import os
+from typing import Dict
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.core.awel import DAG, HttpTrigger, MapOperator
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.knowledge.base import KnowledgeType
+from dbgpt_ext.rag.operators import KnowledgeOperator, SummaryAssemblerOperator
+
+
+class TriggerReqBody(BaseModel):
+    url: str = Field(..., description="url")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "url": input_value.url,
+        }
+        print(f"Receive input value: {input_value}")
+        return params
+
+
+with DAG("dbgpt_awel_simple_rag_summary_example") as dag:
+    trigger = HttpTrigger(
+        "/examples/rag/summary", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    path_operator = MapOperator(lambda request: {"source": request["url"]})
+    # build knowledge operator
+    knowledge_operator = KnowledgeOperator(knowledge_type=KnowledgeType.URL.name)
+    # build summary assembler operator
+    summary_operator = SummaryAssemblerOperator(
+        llm_client=OpenAILLMClient(
+            api_key=os.getenv("OPENAI_API_KEY", "your api key"),
+            api_base=os.getenv("OPENAI_API_BASE", "your api base"),
+        ),
+        language="zh",
+        model_name=os.getenv("MODEL_NAME", "your model name"),
+    )
+    (
+        trigger
+        >> request_handle_task
+        >> path_operator
+        >> knowledge_operator
+        >> summary_operator
+    )
+
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/client/app_crud_example.py

@@ -0,0 +1,34 @@
+"""Client: Simple App CRUD example.
+
+This example demonstrates how to use the dbgpt client to get, list apps.
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        client = Client(api_key=DBGPT_API_KEY)
+        # 1. List all apps
+        res = await list_app(client)
+        # 2. Get an app
+        res = await get_app(client, app_id="bf1c7561-13fc-4fe0-bf5d-c22e724766a8")
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+from dbgpt_client.app import list_app
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    client = Client(api_key=DBGPT_API_KEY)
+    try:
+        res = await list_app(client)
+        print(res)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/client/client_chat_example.py

@@ -0,0 +1,76 @@
+"""Client: Simple Chat example.
+
+This example demonstrates how to use the dbgpt client to chat with the chatgpt model.
+
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        # chat with stream
+        client = Client(api_key=DBGPT_API_KEY)
+
+        # 1. chat normal
+        async for data in client.chat_stream(
+            model="chatgpt_proxyllm",
+            messages="hello",
+        ):
+            print(data.dict())
+
+        # chat with no stream
+        res = await client.chat(model="chatgpt_proxyllm", messages="Hello?")
+        print(res.json())
+
+        # 2. chat with app
+        async for data in client.chat_stream(
+            model="chatgpt_proxyllm",
+            chat_mode="chat_app",
+            chat_param="${app_code}",
+            messages="hello",
+        ):
+            print(data.dict())
+
+        # 3. chat with knowledge
+        async for data in client.chat_stream(
+            model="chatgpt_proxyllm",
+            chat_mode="chat_knowledge",
+            chat_param="${space_name}",
+            messages="hello",
+        ):
+            print(data.dict())
+
+        # 4. chat with flow
+        async for data in client.chat_stream(
+            model="chatgpt_proxyllm",
+            chat_mode="chat_flow",
+            chat_param="${flow_id}",
+            messages="hello",
+        ):
+            print(data.dict())
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    client = Client(api_key=DBGPT_API_KEY)
+    try:
+        data = await client.chat(model="Qwen2.5-72B-Instruct", messages="hello")
+        print(data)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+    # async for data in client.chat_stream(
+    #     model="chatgpt_proxyllm",
+    #     messages="hello",
+    # ):
+
+    # res = await client.chat(model="chatgpt_proxyllm" ,messages="hello")
+    # print(res)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/client/client_evaluation.py

@@ -0,0 +1,95 @@
+"""Client: run evaluation example.
+
+This example demonstrates how to use the dbgpt client to evaluate with the rag recall
+and app answer.
+
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        client = Client(api_key=DBGPT_API_KEY)
+
+        # 1. evaluate with rag recall
+        request = EvaluateServeRequest(
+            # The scene type of the evaluation, e.g. support app, recall
+            scene_key="recall",
+            # e.g. app id(when scene_key is app), space id(when scene_key is recall)
+            scene_value="147",
+            context={"top_k": 5},
+            evaluate_metrics=[
+                "RetrieverHitRateMetric",
+                "RetrieverMRRMetric",
+                "RetrieverSimilarityMetric",
+            ],
+            datasets=[
+                {
+                    "query": "what awel talked about",
+                    "doc_name": "awel.md",
+                }
+            ],
+        )
+        # 2. evaluate with app answer
+        request = EvaluateServeRequest(
+            # The scene type of the evaluation, e.g. support app, recall
+            scene_key="app",
+            # e.g. app id(when scene_key is app), space id(when scene_key is recall)
+            scene_value="2c76eea2-83b6-11ef-b482-acde48001122",
+            "context"={
+                "top_k": 5,
+                "prompt": "942acd7e33b54ce28565f89f9b278044",
+                "model": "zhipu_proxyllm",
+            },
+            evaluate_metrics=[
+                "AnswerRelevancyMetric",
+            ],
+            datasets=[
+                {
+                    "query": "what awel talked about",
+                    "doc_name": "awel.md",
+                }
+            ],
+        )
+        data = await run_evaluation(client, request=request)
+        print(data)
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+from dbgpt_client.evaluation import run_evaluation
+from dbgpt_serve.evaluate.api.schemas import EvaluateServeRequest
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    SPACE_ID = "147"
+    client = Client(api_key=DBGPT_API_KEY)
+    request = EvaluateServeRequest(
+        # The scene type of the evaluation, e.g. support app, recall
+        scene_key="recall",
+        # e.g. app id(when scene_key is app), space id(when scene_key is recall)
+        scene_value=SPACE_ID,
+        context={"top_k": 5},
+        evaluate_metrics=[
+            "RetrieverHitRateMetric",
+            "RetrieverMRRMetric",
+            "RetrieverSimilarityMetric",
+        ],
+        datasets=[
+            {
+                "query": "what awel talked about",
+                "doc_name": "awel.md",
+            }
+        ],
+    )
+    try:
+        data = await run_evaluation(client, request=request)
+        print(data)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/client/client_openai_chat.py

@@ -0,0 +1,165 @@
+# /// script
+# dependencies = [
+#   "openai",
+# ]
+# [tool.uv]
+# exclude-newer = "2025-03-07T00:00:00Z"
+# ///
+"""Chat With Your DB-GPT's API by OpenAI Client
+
+Sample Usage:
+```bash
+uv run examples/client/client_openai_chat.py -m Qwen/QwQ-32B --input "Hello"
+```
+
+More examples:
+
+1. Chat Normal Mode:
+```bash
+uv run examples/client/client_openai_chat.py -m Qwen/QwQ-32B \
+    --input "Which is bigger, 9.8 or 9.11?"
+```
+
+2. Chat Database Mode(chat_with_db_qa):
+
+```bash
+uv run examples/client/client_openai_chat.py -m Qwen/QwQ-32B \
+    --chat-mode chat_with_db_qa \
+    --param "sqlite_dbgpt" \
+    --input "Which table stores database connection information?"
+```
+
+3. Chat With Your Data(chat_data):
+```bash
+uv run examples/client/client_openai_chat.py -m Qwen/QwQ-32B \
+    --chat-mode chat_data \
+    --param "sqlite_dbgpt" \
+    --input "Which database can I currently connect to? What is its name and type?"
+```
+
+4. Chat With Knowledge(chat_knowledge):
+```bash
+uv run examples/client/client_openai_chat.py -m Qwen/QwQ-32B \
+    --chat-mode chat_knowledge \
+    --param "awel" \
+    --input "What is AWEL?"
+```   
+
+
+5. Chat With Third-party API(chat_third_party):
+```bash
+uv run examples/client/client_openai_chat.py -m deepseek-chat \
+    --input "Which is bigger, 9.8 or 9.11?" \
+    --chat-mode none \
+    --api-key $DEEPSEEK_API_KEY \
+    --api-base https://api.deepseek.com/v1
+```
+
+"""  # noqa
+
+import argparse
+
+from openai import OpenAI
+
+DBGPT_API_KEY = "dbgpt"
+
+
+def handle_output(response):
+    has_thinking = False
+    print("=" * 80)
+    reasoning_content = ""
+    for chunk in response:
+        delta_content = chunk.choices[0].delta.content
+        if hasattr(chunk.choices[0].delta, "reasoning_content"):
+            reasoning_content = chunk.choices[0].delta.reasoning_content
+        if reasoning_content:
+            if not has_thinking:
+                print("<thinking>", flush=True)
+            print(reasoning_content, end="", flush=True)
+            has_thinking = True
+        if delta_content:
+            if has_thinking:
+                print("</thinking>", flush=True)
+            print(delta_content, end="", flush=True)
+            has_thinking = False
+
+
+def main():
+    parser = argparse.ArgumentParser(description="OpenAI Chat Client")
+    parser.add_argument(
+        "-m",
+        "--model",
+        type=str,
+        default="deepseek-chat",
+        help="Model name",
+    )
+    parser.add_argument(
+        "-c",
+        "--chat-mode",
+        type=str,
+        default="chat_normal",
+        help="Chat mode. Default is chat_normal",
+    )
+    parser.add_argument(
+        "-p",
+        "--param",
+        type=str,
+        default=None,
+        help="Chat param",
+    )
+    parser.add_argument(
+        "--input",
+        type=str,
+        default="Hello, how are you?",
+        help="User input",
+    )
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        default=DBGPT_API_KEY,
+        help="API key",
+    )
+    parser.add_argument(
+        "--api-base",
+        type=str,
+        default="http://localhost:5670/api/v2",
+        help="Base URL",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=4096,
+        help="Max tokens",
+    )
+    args = parser.parse_args()
+
+    client = OpenAI(
+        api_key=args.api_key,
+        base_url=args.api_base,
+    )
+
+    messages = [
+        {
+            "role": "user",
+            "content": args.input,
+        },
+    ]
+
+    extra_body = {}
+    if args.chat_mode != "none":
+        extra_body["chat_mode"] = args.chat_mode
+    if args.param:
+        extra_body["chat_param"] = args.param
+
+    response = client.chat.completions.create(
+        model=args.model,
+        messages=messages,
+        extra_body=extra_body,
+        stream=True,
+        max_tokens=args.max_tokens,
+    )
+    handle_output(response)
+
+
+if __name__ == "__main__":
+    main()

examples/client/datasource_crud_example.py

@@ -0,0 +1,67 @@
+"""Client: Simple Flow CRUD example
+
+This example demonstrates how to use the dbgpt client to create, get, update, and
+delete  datasource.
+
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        client = Client(api_key=DBGPT_API_KEY)
+        # 1. Create a flow
+        res = await create_datasource(
+            client,
+            DatasourceModel(
+                db_name="dbgpt",
+                desc="for client datasource",
+                db_type="mysql",
+                db_type="mysql",
+                db_host="127.0.0.1",
+                db_user="root",
+                db_pwd="xxxx",
+                db_port=3306,
+            ),
+        )
+        # 2. Update a flow
+        res = await update_datasource(
+            client,
+            DatasourceModel(
+                db_name="dbgpt",
+                desc="for client datasource",
+                db_type="mysql",
+                db_type="mysql",
+                db_host="127.0.0.1",
+                db_user="root",
+                db_pwd="xxxx",
+                db_port=3306,
+            ),
+        )
+        # 3. Delete a flow
+        res = await delete_datasource(client, datasource_id="10")
+        # 4. Get a flow
+        res = await get_datasource(client, datasource_id="10")
+        # 5. List all datasource
+        res = await list_datasource(client)
+
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+from dbgpt_client.datasource import list_datasource
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    client = Client(api_key=DBGPT_API_KEY)
+    try:
+        res = await list_datasource(client)
+        print(res)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/client/flow_crud_example.py

@@ -0,0 +1,49 @@
+"""Client: Simple Flow CRUD example
+
+This example demonstrates how to use the dbgpt client to create, get, update, and
+delete flows.
+
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        client = Client(api_key=DBGPT_API_KEY)
+        # 1. Create a flow
+        res = await create_flow(
+            client,
+            FlowPanel(name="test_flow", desc="for client flow", owner="dbgpt"),
+        )
+        # 2. Update a flow
+        res = await update_flow(
+            client,
+            FlowPanel(name="test_flow", desc="for client flow333", owner="dbgpt"),
+        )
+        # 3. Delete a flow
+        res = await delete_flow(client, flow_id="bf1c7561-13fc-4fe0-bf5d-c22e724766a8")
+        # 4. Get a flow
+        res = await get_flow(client, flow_id="bf1c7561-13fc-4fe0-bf5d-c22e724766a8")
+        # 5. List all flows
+        res = await list_flow(client)
+
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+from dbgpt_client.flow import list_flow
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    client = Client(api_key=DBGPT_API_KEY)
+    try:
+        res = await list_flow(client)
+        print(res)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/client/knowledge_crud_example.py

@@ -0,0 +1,128 @@
+"""Client: Simple Knowledge CRUD example.
+
+This example demonstrates how to use the dbgpt client to create, get, update, and
+delete knowledge spaces and documents.
+
+Example:
+    .. code-block:: python
+
+        DBGPT_API_KEY = "dbgpt"
+        client = Client(api_key=DBGPT_API_KEY)
+        # 1. Create a space
+        res = await create_space(
+            client,
+            SpaceModel(
+                name="test_space",
+                vector_type="Chroma",
+                desc="for client space",
+                owner="dbgpt",
+            ),
+        )
+        # 2. Update a space
+        res = await update_space(
+            client,
+            SpaceModel(
+                name="test_space",
+                vector_type="Chroma",
+                desc="for client space333",
+                owner="dbgpt",
+            ),
+        )
+        # 3. Delete a space
+        res = await delete_space(client, space_id="37")
+        # 4. Get a space
+        res = await get_space(client, space_id="5")
+        # 5. List all spaces
+        res = await list_space(client)
+        # 6. Create a document
+        res = await create_document(
+            client,
+            DocumentModel(
+                space_id="5",
+                doc_name="test_doc",
+                doc_type="TEXT",
+                doc_content="test content",
+                doc_source="",
+            ),
+        )
+        # 7. Sync a document
+        res = await sync_document(
+            client,
+            sync_model=SyncModel(
+                doc_id="153",
+                space_id="40",
+                model_name="text2vec",
+                chunk_parameters=ChunkParameters(chunk_strategy="Automatic"),
+            ),
+        )
+        # 8. Get a document
+        res = await get_document(client, "52")
+        # 9. List all documents
+        res = await list_document(client)
+        # 10. Delete a document
+        res = await delete_document(client, "150")
+"""
+
+import asyncio
+
+from dbgpt_client import Client
+from dbgpt_client.knowledge import create_space
+from dbgpt_client.schema import SpaceModel
+
+
+async def main():
+    # initialize client
+    DBGPT_API_KEY = "dbgpt"
+    client = Client(api_key=DBGPT_API_KEY)
+
+    try:
+        res = await create_space(
+            client,
+            SpaceModel(
+                name="test_space_1",
+                vector_type="Chroma",
+                desc="for client space desc",
+                owner="dbgpt",
+            ),
+        )
+        print(res)
+    finally:
+        # explicitly close client to avoid event loop closed error
+        await client.aclose()
+
+    # list all spaces
+    # res = await list_space(client)
+    # print(res)
+
+    # get space
+    # res = await get_space(client, space_id='5')
+
+    # create space
+    # res = await create_space(client, SpaceModel(name="test_space", vector_type="Chroma", desc="for client space", owner="dbgpt"))
+
+    # update space
+    # res = await update_space(client, SpaceModel(name="test_space", vector_type="Chroma", desc="for client space333", owner="dbgpt"))
+
+    # delete space
+    # res = await delete_space(client, space_id='31')
+    # print(res)
+
+    # list all documents
+    # res = await list_document(client)
+
+    # get document
+    # res = await get_document(client, "52")
+
+    # delete document
+    # res = await delete_document(client, "150")
+
+    # create document
+    # res = await create_document(client, DocumentModel(space_id="5", doc_name="test_doc", doc_type="test", doc_content="test content"
+    #                                                   , doc_file=('your_file_name', open('{your_file_path}', 'rb'))))
+
+    # sync document
+    # res = await sync_document(client, sync_model=SyncModel(doc_id="157", space_id="49", model_name="text2vec", chunk_parameters=ChunkParameters(chunk_strategy="Automatic")))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/excel/excel_analysis.py

@@ -0,0 +1,54 @@
+import os
+
+import pandas as pd
+from flask import Flask, jsonify, request
+
+app = Flask(__name__)
+UPLOAD_FOLDER = "/tmp/uploads"
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
+
+
+@app.route("/upload_excel", methods=["POST"])
+def upload_excel():
+    if "file" not in request.files:
+        return jsonify({"error": "No file uploaded"}), 400
+
+    file = request.files["file"]
+    if file.filename == "":
+        return jsonify({"error": "No selected file"}), 400
+
+    file_path = os.path.join(app.config["UPLOAD_FOLDER"], file.filename)
+    file.save(file_path)
+
+    # Optional: Parse Excel to validate it works
+    try:
+        df = pd.read_excel(file_path)
+        columns = df.columns.tolist()
+        return jsonify({"message": "File uploaded successfully", "columns": columns})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/query_excel", methods=["POST"])
+def query_excel():
+    data = request.json
+    file_name = data.get("file_name")
+    query = data.get("query")
+
+    file_path = os.path.join(app.config["UPLOAD_FOLDER"], file_name)
+    if not os.path.exists(file_path):
+        return jsonify({"error": "File not found"}), 404
+
+    try:
+        df = pd.read_excel(file_path)
+        # Placeholder logic for querying the Excel file:
+        # This should be replaced with DB-GPT integration for natural language queries.
+        response = f"Query on {len(df)} rows completed."
+        return jsonify({"query_result": response})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+if __name__ == "__main__":
+    app.run(debug=True)

examples/rag/bm25_retriever_example.py

@@ -0,0 +1,49 @@
+import asyncio
+import os
+
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler.bm25 import BM25Assembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.vector_store.elastic_store import ElasticsearchStoreConfig
+
+"""Embedding rag example.
+    pre-requirements:
+    set your elasticsearch config in your example code.
+
+    Examples:
+        ..code-block:: shell
+            python examples/rag/bm25_retriever_example.py
+"""
+
+
+def _create_es_config():
+    """Create vector connector."""
+    return ElasticsearchStoreConfig(
+        uri="localhost",
+        port="9200",
+        user="elastic",
+        password="dbgpt",
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    es_config = _create_es_config()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_SIZE")
+    # create bm25 assembler
+    assembler = BM25Assembler.load_from_knowledge(
+        knowledge=knowledge,
+        es_config=es_config,
+        chunk_parameters=chunk_parameters,
+    )
+    assembler.persist()
+    # get bm25 retriever
+    retriever = assembler.as_retriever(3)
+    chunks = retriever.retrieve_with_scores("what is awel talk about", 0.3)
+    print(f"bm25 rag example results:{chunks}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/cross_encoder_rerank_example.py

@@ -0,0 +1,70 @@
+"""This example demonstrates how to use the cross-encoder reranker
+to rerank the retrieved chunks.
+The cross-encoder reranker is a neural network model that takes a query
+and a chunk as input and outputs a score that represents the relevance of the chunk
+to the query.
+
+Download pretrained cross-encoder models can be found at https://huggingface.co/models.
+Example:
+    python examples/rag/cross_encoder_rerank_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH, ROOT_PATH
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.rag.retriever.rerank import CrossEncoderRanker
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    vector_connector = _create_vector_connector()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_MARKDOWN_HEADER")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=vector_connector,
+    )
+    assembler.persist()
+    # get embeddings retriever
+    retriever = assembler.as_retriever(3)
+    # create metadata filter
+    query = "what is awel talk about"
+    chunks = await retriever.aretrieve_with_scores(query, 0.3)
+
+    print("before rerank results:\n")
+    for i, chunk in enumerate(chunks):
+        print(f"----{i + 1}.chunk content:{chunk.content}\n score:{chunk.score}")
+    # cross-encoder rerankpython
+    cross_encoder_model = os.path.join(MODEL_PATH, "bge-reranker-base")
+    rerank = CrossEncoderRanker(topk=3, model=cross_encoder_model)
+    new_chunks = rerank.rank(chunks, query=query)
+    print("after cross-encoder rerank results:\n")
+    for i, chunk in enumerate(new_chunks):
+        print(f"----{i + 1}.chunk content:{chunk.content}\n score:{chunk.score}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/db_schema_rag_example.py

@@ -0,0 +1,70 @@
+import os
+
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteTempConnector
+from dbgpt_ext.rag.assembler import DBSchemaAssembler
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+"""DB struct rag example.
+    pre-requirements:
+    set your embedding model path in your example code.
+    ```
+    embedding_model_path = "{your_embedding_model_path}"
+    ```
+
+    Examples:
+        ..code-block:: shell
+            python examples/rag/db_schema_rag_example.py
+"""
+
+
+def _create_temporary_connection():
+    """Create a temporary database connection for testing."""
+    connect = SQLiteTempConnector.create_temporary_db()
+    connect.create_temp_tables(
+        {
+            "user": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                },
+                "data": [
+                    (1, "Tom", 10),
+                    (2, "Jerry", 16),
+                    (3, "Jack", 18),
+                    (4, "Alice", 20),
+                    (5, "Bob", 22),
+                ],
+            }
+        }
+    )
+    return connect
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+if __name__ == "__main__":
+    connection = _create_temporary_connection()
+    vector_connector = _create_vector_connector()
+    assembler = DBSchemaAssembler.load_from_connection(
+        connector=connection, table_vector_store_connector=vector_connector
+    )
+    assembler.persist()
+    # get db schema retriever
+    retriever = assembler.as_retriever(top_k=1)
+    chunks = retriever.retrieve("show columns from user")
+    print(f"db schema rag example results:{[chunk.content for chunk in chunks]}")

examples/rag/doc_tree_retriever_example.py

@@ -0,0 +1,36 @@
+import asyncio
+
+from dbgpt.model.proxy import DeepseekLLMClient
+from dbgpt.rag.knowledge.base import ChunkStrategy
+from dbgpt.rag.transformer.keyword_extractor import KeywordExtractor
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.rag.retriever.doc_tree import DocTreeRetriever
+
+
+async def main():
+    knowledge = KnowledgeFactory.from_file_path("../../docs/docs/awel/awel.md")
+    chunk_parameters = ChunkParameters(
+        chunk_strategy=ChunkStrategy.CHUNK_BY_MARKDOWN_HEADER.name
+    )
+    docs = knowledge.load()
+    docs = knowledge.extract(docs, chunk_parameters)
+    llm_client = DeepseekLLMClient(api_key="your_api_key")
+    keyword_extractor = KeywordExtractor(
+        llm_client=llm_client, model_name="deepseek-chat"
+    )
+    # doc tree retriever retriever
+    retriever = DocTreeRetriever(
+        docs=docs,
+        top_k=10,
+        keywords_extractor=keyword_extractor,
+        with_content=False,
+    )
+    tree_index = retriever._tree_indexes[0]
+    nodes = await retriever.aretrieve("Introduce awel Operators")
+    for node in nodes:
+        tree_index.display_tree(node)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/embedding_rag_example.py

@@ -0,0 +1,58 @@
+import asyncio
+import os
+
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH, ROOT_PATH
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+"""Embedding rag example.
+    pre-requirements:
+    set your embedding model path in your example code.
+    ```
+    embedding_model_path = "{your_embedding_model_path}"
+    ```
+
+    Examples:
+        ..code-block:: shell
+            python examples/rag/embedding_rag_example.py
+"""
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    vector_store = _create_vector_connector()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_SIZE")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=vector_store,
+    )
+    assembler.persist()
+    # get embeddings retriever
+    retriever = assembler.as_retriever(3)
+    chunks = await retriever.aretrieve_with_scores("what is awel talk about", 0.3)
+    print(f"embedding rag example results:{chunks}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/graph_rag_example.py

@@ -0,0 +1,123 @@
+import os
+
+import pytest
+
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt.core import Chunk, HumanPromptTemplate, ModelMessage, ModelRequest
+from dbgpt.model.proxy.llms.chatgpt import OpenAILLMClient
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.rag.retriever import RetrieverStrategy
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.graph_store.tugraph_store import TuGraphStoreConfig
+from dbgpt_ext.storage.knowledge_graph.community_summary import (
+    CommunitySummaryKnowledgeGraph,
+)
+from dbgpt_ext.storage.knowledge_graph.knowledge_graph import (
+    BuiltinKnowledgeGraph,
+)
+
+"""GraphRAG example.
+    ```
+    # Set LLM config (url/sk) in `.env`.
+    # Install pytest utils: `pip install pytest pytest-asyncio`
+    GRAPH_STORE_TYPE=TuGraph
+    TUGRAPH_HOST=127.0.0.1
+    TUGRAPH_PORT=7687
+    TUGRAPH_USERNAME=admin
+    TUGRAPH_PASSWORD=73@TuGraph
+    ```
+    Examples:
+        ..code-block:: shell
+            pytest -s examples/rag/graph_rag_example.py
+"""
+
+llm_client = OpenAILLMClient()
+model_name = "gpt-4o-mini"
+
+
+@pytest.mark.asyncio
+async def test_naive_graph_rag():
+    await __run_graph_rag(
+        knowledge_file="examples/test_files/graphrag-mini.md",
+        chunk_strategy="CHUNK_BY_SIZE",
+        knowledge_graph=__create_naive_kg_connector(),
+        question="What's the relationship between TuGraph and DB-GPT ?",
+    )
+
+
+@pytest.mark.asyncio
+async def test_community_graph_rag():
+    await __run_graph_rag(
+        knowledge_file="examples/test_files/graphrag-mini.md",
+        chunk_strategy="CHUNK_BY_MARKDOWN_HEADER",
+        knowledge_graph=__create_community_kg_connector(),
+        question="What's the relationship between TuGraph and DB-GPT ?",
+    )
+
+
+def __create_naive_kg_connector():
+    """Create knowledge graph connector."""
+    return BuiltinKnowledgeGraph(
+        config=TuGraphStoreConfig(),
+        name="naive_graph_rag_test",
+        embedding_fn=None,
+        llm_client=llm_client,
+        llm_model=model_name,
+    )
+
+
+def __create_community_kg_connector():
+    """Create community knowledge graph connector."""
+    return CommunitySummaryKnowledgeGraph(
+        config=TuGraphStoreConfig(),
+        name="community_graph_rag_test",
+        embedding_fn=DefaultEmbeddingFactory.openai(),
+        llm_client=llm_client,
+        llm_model=model_name,
+    )
+
+
+async def ask_chunk(chunk: Chunk, question) -> str:
+    rag_template = (
+        "Based on the following [Context] {context}, answer [Question] {question}."
+    )
+    template = HumanPromptTemplate.from_template(rag_template)
+    messages = template.format_messages(context=chunk.content, question=question)
+    model_messages = ModelMessage.from_base_messages(messages)
+    request = ModelRequest(model=model_name, messages=model_messages)
+    response = await llm_client.generate(request=request)
+
+    if not response.success:
+        code = str(response.error_code)
+        reason = response.text
+        raise Exception(f"request llm failed ({code}) {reason}")
+
+    return response.text
+
+
+async def __run_graph_rag(knowledge_file, chunk_strategy, knowledge_graph, question):
+    file_path = os.path.join(ROOT_PATH, knowledge_file).format()
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    try:
+        chunk_parameters = ChunkParameters(chunk_strategy=chunk_strategy)
+
+        # get embedding assembler
+        assembler = await EmbeddingAssembler.aload_from_knowledge(
+            knowledge=knowledge,
+            chunk_parameters=chunk_parameters,
+            index_store=knowledge_graph,
+            retrieve_strategy=RetrieverStrategy.GRAPH,
+        )
+        await assembler.apersist()
+
+        # get embeddings retriever
+        retriever = assembler.as_retriever(1)
+        chunks = await retriever.aretrieve_with_scores(question, score_threshold=0.3)
+
+        # chat
+        print(f"{await ask_chunk(chunks[0], question)}")
+
+    finally:
+        knowledge_graph.delete_vector_name(knowledge_graph.get_config().name)

examples/rag/keyword_rag_example.py

@@ -0,0 +1,54 @@
+import asyncio
+import os
+
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.full_text.elasticsearch import (
+    ElasticDocumentStore,
+    ElasticsearchStoreConfig,
+)
+
+"""Keyword rag example.
+    pre-requirements:
+    set your Elasticsearch environment.
+
+    Examples:
+        ..code-block:: shell
+            python examples/rag/keyword_rag_example.py
+"""
+
+
+def _create_es_connector():
+    """Create es connector."""
+    config = ElasticsearchStoreConfig(
+        uri="localhost",
+        port="9200",
+        user="elastic",
+        password="dbgpt",
+    )
+
+    return ElasticDocumentStore(config, name="keyword_rag_test")
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    keyword_store = _create_es_connector()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_SIZE")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=keyword_store,
+    )
+    assembler.persist()
+    # get embeddings retriever
+    retriever = assembler.as_retriever(3)
+    chunks = await retriever.aretrieve_with_scores("what is awel talk about", 0.3)
+    print(f"keyword rag example results:{chunks}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/metadata_filter_example.py

@@ -0,0 +1,62 @@
+"""Rag Metadata Properties filter example.
+pre-requirements:
+make sure you have set your embedding model path in your example code.
+
+Examples:
+    ..code-block:: shell
+        python examples/rag/metadata_filter_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH, ROOT_PATH
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.storage.vector_store.filters import MetadataFilter, MetadataFilters
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    vector_store = _create_vector_connector()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_MARKDOWN_HEADER")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=vector_store,
+    )
+    assembler.persist()
+    # get embeddings retriever
+    retriever = assembler.as_retriever(3)
+    # create metadata filter
+    metadata_filter = MetadataFilter(key="Header2", value="AWEL Design")
+    filters = MetadataFilters(filters=[metadata_filter])
+    chunks = await retriever.aretrieve_with_scores(
+        "what is awel talk about", 0.0, filters
+    )
+    print(f"embedding rag example results:{chunks}")
+    vector_store.delete_vector_name("metadata_rag_test")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/rag_embedding_api_example.py

@@ -0,0 +1,88 @@
+"""A RAG example using the OpenAPIEmbeddings.
+
+Example:
+
+    Test with `OpenAI embeddings
+    <https://platform.openai.com/docs/api-reference/embeddings/create>`_.
+
+    .. code-block:: shell
+
+        export API_SERVER_BASE_URL=${OPENAI_API_BASE:-"https://api.openai.com/v1"}
+        export API_SERVER_API_KEY="${OPENAI_API_KEY}"
+        export API_SERVER_EMBEDDINGS_MODEL="text-embedding-ada-002"
+        python examples/rag/rag_embedding_api_example.py
+
+    Test with DB-GPT `API Server
+    <https://docs.dbgpt.site/docs/installation/advanced_usage/OpenAI_SDK_call#start-apiserver>`_.
+
+    .. code-block:: shell
+        export API_SERVER_BASE_URL="http://localhost:8100/api/v1"
+        export API_SERVER_API_KEY="your_api_key"
+        export API_SERVER_EMBEDDINGS_MODEL="text2vec"
+        python examples/rag/rag_embedding_api_example.py
+
+"""
+
+import asyncio
+import os
+from typing import Optional
+
+from dbgpt.configs.model_config import PILOT_PATH, ROOT_PATH
+from dbgpt.rag.embedding import OpenAPIEmbeddings
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_embeddings(
+    api_url: str = None, api_key: Optional[str] = None, model_name: Optional[str] = None
+) -> OpenAPIEmbeddings:
+    if not api_url:
+        api_server_base_url = os.getenv(
+            "API_SERVER_BASE_URL", "http://localhost:8100/api/v1/"
+        )
+        api_url = f"{api_server_base_url}/embeddings"
+    if not api_key:
+        api_key = os.getenv("API_SERVER_API_KEY")
+
+    if not model_name:
+        model_name = os.getenv("API_SERVER_EMBEDDINGS_MODEL", "text2vec")
+
+    return OpenAPIEmbeddings(api_url=api_url, api_key=api_key, model_name=model_name)
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=_create_embeddings(),
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    vector_store = _create_vector_connector()
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_SIZE")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=vector_store,
+    )
+    assembler.persist()
+    # get embeddings retriever
+    retriever = assembler.as_retriever(3)
+    chunks = await retriever.aretrieve_with_scores("what is awel talk about", 0.3)
+    print(f"embedding rag example results:{chunks}")
+    vector_store.delete_vector_name("embedding_api_rag_test")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/retriever_evaluation_example.py

@@ -0,0 +1,95 @@
+import asyncio
+import os
+from typing import Optional
+
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH, ROOT_PATH
+from dbgpt.core import Embeddings
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.rag.evaluation import RetrieverEvaluator
+from dbgpt.rag.evaluation.retriever import (
+    RetrieverHitRateMetric,
+    RetrieverMRRMetric,
+    RetrieverSimilarityMetric,
+)
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import EmbeddingAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+from dbgpt_ext.rag.operators import EmbeddingRetrieverOperator
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_embeddings(
+    model_name: Optional[str] = "text2vec-large-chinese",
+) -> Embeddings:
+    """Create embeddings."""
+    return DefaultEmbeddingFactory(
+        default_model_name=os.path.join(MODEL_PATH, model_name),
+    ).create()
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=_create_embeddings(),
+    )
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    embeddings = _create_embeddings()
+    vector_connector = _create_vector_connector(embeddings)
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_MARKDOWN_HEADER")
+    # get embedding assembler
+    assembler = EmbeddingAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        index_store=vector_connector,
+    )
+    assembler.persist()
+
+    dataset = [
+        {
+            "query": "what is awel talk about",
+            "contexts": [
+                "# What is AWEL? \n\nAgentic Workflow Expression Language(AWEL) is a "
+                "set of intelligent agent workflow expression language specially "
+                "designed for large model application\ndevelopment. It provides great "
+                "functionality and flexibility. Through the AWEL API, you can focus on "
+                "the development of business logic for LLMs applications\nwithout "
+                "paying attention to cumbersome model and environment details.\n\nAWEL "
+                "adopts a layered API design. AWEL's layered API design architecture is "
+                "shown in the figure below."
+            ],
+        },
+    ]
+    evaluator = RetrieverEvaluator(
+        operator_cls=EmbeddingRetrieverOperator,
+        embeddings=embeddings,
+        operator_kwargs={
+            "top_k": 5,
+            "index_store": vector_connector,
+        },
+    )
+    metrics = [
+        RetrieverHitRateMetric(),
+        RetrieverMRRMetric(),
+        RetrieverSimilarityMetric(embeddings=embeddings),
+    ]
+    results = await evaluator.evaluate(dataset, metrics)
+    for result in results:
+        for metric in result:
+            print("Metric:", metric.metric_name)
+            print("Question:", metric.query)
+            print("Score:", metric.score)
+    print(f"Results:\n{results}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/rag/rewrite_rag_example.py

@@ -0,0 +1,42 @@
+"""Query rewrite example.
+
+pre-requirements:
+    1. install openai python sdk
+    ```
+        pip install openai
+    ```
+    2. set openai key and base
+    ```
+        export OPENAI_API_KEY={your_openai_key}
+        export OPENAI_API_BASE={your_openai_base}
+    ```
+    or
+    ```
+        import os
+        os.environ["OPENAI_API_KEY"] = {your_openai_key}
+        os.environ["OPENAI_API_BASE"] = {your_openai_base}
+    ```
+Examples:
+    ..code-block:: shell
+        python examples/rag/rewrite_rag_example.py
+"""
+
+import asyncio
+
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.retriever import QueryRewrite
+
+
+async def main():
+    query = "compare steve curry and lebron james"
+    llm_client = OpenAILLMClient()
+    reinforce = QueryRewrite(
+        llm_client=llm_client,
+        model_name="gpt-3.5-turbo",
+    )
+    return await reinforce.rewrite(origin_query=query, nums=1)
+
+
+if __name__ == "__main__":
+    output = asyncio.run(main())
+    print(f"output: \n\n{output}")

examples/rag/simple_dbschema_retriever_example.py

@@ -0,0 +1,130 @@
+"""AWEL: Simple rag db schema embedding operator example
+
+    if you not set vector_store_connector, it will return all tables schema in database.
+    ```
+    retriever_task = DBSchemaRetrieverOperator(
+        connector=_create_temporary_connection()
+    )
+    ```
+    if you set vector_store_connector, it will recall topk similarity tables schema in database.
+    ```
+    retriever_task = DBSchemaRetrieverOperator(
+        connector=_create_temporary_connection()
+        top_k=1,
+        index_store=vector_store_connector
+    )
+    ```
+
+    Examples:
+        ..code-block:: shell
+            curl --location 'http://127.0.0.1:5555/api/v1/awel/trigger/examples/rag/dbschema' \
+            --header 'Content-Type: application/json' \
+            --data '{"query": "what is user name?"}'
+"""
+
+import os
+from typing import Dict, List
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH
+from dbgpt.core import Chunk
+from dbgpt.core.awel import DAG, HttpTrigger, JoinOperator, MapOperator
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteTempConnector
+from dbgpt_ext.rag.operators import DBSchemaAssemblerOperator
+from dbgpt_ext.rag.operators.db_schema import DBSchemaRetrieverOperator
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+def _create_temporary_connection():
+    """Create a temporary database connection for testing."""
+    connect = SQLiteTempConnector.create_temporary_db()
+    connect.create_temp_tables(
+        {
+            "user": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                },
+                "data": [
+                    (1, "Tom", 10),
+                    (2, "Jerry", 16),
+                    (3, "Jack", 18),
+                    (4, "Alice", 20),
+                    (5, "Bob", 22),
+                ],
+            }
+        }
+    )
+    return connect
+
+
+def _join_fn(chunks: List[Chunk], query: str) -> str:
+    print(f"db schema info is {[chunk.content for chunk in chunks]}")
+    return query
+
+
+class TriggerReqBody(BaseModel):
+    query: str = Field(..., description="User query")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "query": input_value.query,
+        }
+        print(f"Receive input value: {input_value}")
+        return params
+
+
+with DAG("simple_rag_db_schema_example") as dag:
+    trigger = HttpTrigger(
+        "/examples/rag/dbschema", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    query_operator = MapOperator(lambda request: request["query"])
+    index_store = _create_vector_connector()
+    connector = _create_temporary_connection()
+    assembler_task = DBSchemaAssemblerOperator(
+        connector=connector,
+        index_store=index_store,
+    )
+    join_operator = JoinOperator(combine_function=_join_fn)
+    retriever_task = DBSchemaRetrieverOperator(
+        connector=_create_temporary_connection(),
+        top_k=1,
+        index_store=index_store,
+    )
+    result_parse_task = MapOperator(lambda chunks: [chunk.content for chunk in chunks])
+    trigger >> assembler_task >> join_operator
+    trigger >> request_handle_task >> query_operator >> join_operator
+    join_operator >> retriever_task >> result_parse_task
+
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/rag/simple_rag_embedding_example.py

@@ -0,0 +1,97 @@
+"""AWEL: Simple rag embedding operator example.
+
+    Examples:
+    pre-requirements:
+        python examples/awel/simple_rag_embedding_example.py
+        ..code-block:: shell
+            curl --location --request POST 'http://127.0.0.1:5555/api/v1/awel/trigger/examples/rag/embedding' \
+            --header 'Content-Type: application/json' \
+            --data-raw '{
+              "url": "https://docs.dbgpt.site/docs/latest/awel/"
+            }'
+"""
+
+import os
+from typing import Dict, List
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH
+from dbgpt.core.awel import DAG, HttpTrigger, MapOperator
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt.rag.knowledge import KnowledgeType
+from dbgpt_ext.rag.operators import EmbeddingAssemblerOperator, KnowledgeOperator
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+class TriggerReqBody(BaseModel):
+    url: str = Field(..., description="url")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "url": input_value.url,
+        }
+        print(f"Receive input value: {input_value}")
+        return params
+
+
+class ResultOperator(MapOperator):
+    """The Result Operator."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, chunks: List) -> str:
+        result = f"embedding success, there are {len(chunks)} chunks."
+        print(result)
+        return result
+
+
+with DAG("simple_sdk_rag_embedding_example") as dag:
+    trigger = HttpTrigger(
+        "/examples/rag/embedding", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    knowledge_operator = KnowledgeOperator(knowledge_type=KnowledgeType.URL.name)
+    vector_store = _create_vector_connector()
+    url_parser_operator = MapOperator(map_function=lambda x: x["url"])
+    embedding_operator = EmbeddingAssemblerOperator(
+        index_store=vector_store,
+    )
+    output_task = ResultOperator()
+    (
+        trigger
+        >> request_handle_task
+        >> url_parser_operator
+        >> knowledge_operator
+        >> embedding_operator
+        >> output_task
+    )
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/rag/simple_rag_retriever_example.py

@@ -0,0 +1,127 @@
+"""AWEL: Simple rag embedding operator example
+
+    pre-requirements:
+        1. install openai python sdk
+
+        ```
+            pip install openai
+        ```
+         2. set openai key and base
+        ```
+            export OPENAI_API_KEY={your_openai_key}
+            export OPENAI_API_BASE={your_openai_base}
+        ```
+        3. make sure you have vector store.
+        if there are no data in vector store, please run examples/awel/simple_rag_embedding_example.py
+
+
+    ensure your embedding model in DB-GPT/models/.
+
+    Examples:
+        ..code-block:: shell
+            DBGPT_SERVER="http://127.0.0.1:5555"
+            curl -X POST $DBGPT_SERVER/api/v1/awel/trigger/examples/rag/retrieve \
+            -H "Content-Type: application/json" -d '{ \
+                "query": "what is awel talk about?"
+            }'
+"""
+
+import os
+from typing import Dict, List
+
+from dbgpt._private.pydantic import BaseModel, Field
+from dbgpt.configs.model_config import MODEL_PATH, PILOT_PATH
+from dbgpt.core import Chunk
+from dbgpt.core.awel import DAG, HttpTrigger, JoinOperator, MapOperator
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt_ext.rag.operators import (
+    EmbeddingRetrieverOperator,
+    QueryRewriteOperator,
+    RerankOperator,
+)
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+
+class TriggerReqBody(BaseModel):
+    query: str = Field(..., description="User query")
+
+
+class RequestHandleOperator(MapOperator[TriggerReqBody, Dict]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, input_value: TriggerReqBody) -> Dict:
+        params = {
+            "query": input_value.query,
+        }
+        print(f"Receive input value: {input_value}")
+        return params
+
+
+def _context_join_fn(context_dict: Dict, chunks: List[Chunk]) -> Dict:
+    """context Join function for JoinOperator.
+
+    Args:
+        context_dict (Dict): context dict
+        chunks (List[Chunk]): chunks
+    Returns:
+        Dict: context dict
+    """
+    context_dict["context"] = "\n".join([chunk.content for chunk in chunks])
+    return context_dict
+
+
+def _create_vector_connector():
+    """Create vector connector."""
+    config = ChromaVectorConfig(
+        persist_path=PILOT_PATH,
+    )
+
+    return ChromaStore(
+        config,
+        name="embedding_rag_test",
+        embedding_fn=DefaultEmbeddingFactory(
+            default_model_name=os.path.join(MODEL_PATH, "text2vec-large-chinese"),
+        ).create(),
+    )
+
+
+with DAG("simple_sdk_rag_retriever_example") as dag:
+    vector_store = _create_vector_connector()
+    trigger = HttpTrigger(
+        "/examples/rag/retrieve", methods="POST", request_body=TriggerReqBody
+    )
+    request_handle_task = RequestHandleOperator()
+    query_parser = MapOperator(map_function=lambda x: x["query"])
+    context_join_operator = JoinOperator(combine_function=_context_join_fn)
+    rewrite_operator = QueryRewriteOperator(llm_client=OpenAILLMClient())
+    retriever_context_operator = EmbeddingRetrieverOperator(
+        top_k=3,
+        index_store=vector_store,
+    )
+    retriever_operator = EmbeddingRetrieverOperator(
+        top_k=3,
+        index_store=vector_store,
+    )
+    rerank_operator = RerankOperator()
+    model_parse_task = MapOperator(lambda out: out.to_dict())
+
+    trigger >> request_handle_task >> context_join_operator
+    (
+        trigger
+        >> request_handle_task
+        >> query_parser
+        >> retriever_context_operator
+        >> context_join_operator
+    )
+    context_join_operator >> rewrite_operator >> retriever_operator >> rerank_operator
+
+if __name__ == "__main__":
+    if dag.leaf_nodes[0].dev_mode:
+        # Development mode, you can run the dag locally for debugging.
+        from dbgpt.core.awel import setup_dev_environment
+
+        setup_dev_environment([dag], port=5555)
+    else:
+        pass

examples/rag/summary_extractor_example.py

@@ -0,0 +1,49 @@
+"""Summary extractor example.
+pre-requirements:
+    1. install openai python sdk
+    ```
+        pip install openai
+    ```
+    2. set openai key and base
+    ```
+        export OPENAI_API_KEY={your_openai_key}
+        export OPENAI_API_BASE={your_openai_base}
+    ```
+    or
+    ```
+        import os
+        os.environ["OPENAI_API_KEY"] = {your_openai_key}
+        os.environ["OPENAI_API_BASE"] = {your_openai_base}
+    ```
+Examples:
+    ..code-block:: shell
+        python examples/rag/summary_extractor_example.py
+"""
+
+import asyncio
+import os
+
+from dbgpt.configs.model_config import ROOT_PATH
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.assembler import SummaryAssembler
+from dbgpt_ext.rag.knowledge import KnowledgeFactory
+
+
+async def main():
+    file_path = os.path.join(ROOT_PATH, "docs/docs/awel/awel.md")
+    llm_client = OpenAILLMClient()
+    knowledge = KnowledgeFactory.from_file_path(file_path)
+    chunk_parameters = ChunkParameters(chunk_strategy="CHUNK_BY_SIZE")
+    assembler = SummaryAssembler.load_from_knowledge(
+        knowledge=knowledge,
+        chunk_parameters=chunk_parameters,
+        llm_client=llm_client,
+        model_name="gpt-3.5-turbo",
+    )
+    return await assembler.generate_summary()
+
+
+if __name__ == "__main__":
+    output = asyncio.run(main())
+    print(f"output: \n\n{output}")

examples/sdk/chat_data_with_awel.py

@@ -0,0 +1,263 @@
+import asyncio
+import json
+import shutil
+
+import pandas as pd
+
+from dbgpt.configs.model_config import PILOT_PATH
+from dbgpt.core import (
+    ChatPromptTemplate,
+    HumanPromptTemplate,
+    SQLOutputParser,
+    SystemPromptTemplate,
+)
+from dbgpt.core.awel import (
+    DAG,
+    BranchOperator,
+    InputOperator,
+    InputSource,
+    JoinOperator,
+    MapOperator,
+    is_empty_data,
+)
+from dbgpt.core.operators import PromptBuilderOperator, RequestBuilderOperator
+from dbgpt.datasource.operators import DatasourceOperator
+from dbgpt.model.operators import LLMOperator
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.embedding import DefaultEmbeddingFactory
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteTempConnector
+from dbgpt_ext.rag import ChunkParameters
+from dbgpt_ext.rag.operators import DBSchemaAssemblerOperator
+from dbgpt_ext.rag.operators.db_schema import DBSchemaRetrieverOperator
+from dbgpt_ext.storage.vector_store.chroma_store import ChromaStore, ChromaVectorConfig
+
+# Delete old vector store directory(/tmp/awel_with_data_vector_store)
+shutil.rmtree("/tmp/awel_with_data_vector_store", ignore_errors=True)
+
+embeddings = DefaultEmbeddingFactory.openai()
+
+# Here we use the openai LLM model, if you want to use other models, you can replace
+# it according to the previous example.
+llm_client = OpenAILLMClient()
+
+db_conn = SQLiteTempConnector.create_temporary_db()
+db_conn.create_temp_tables(
+    {
+        "user": {
+            "columns": {
+                "id": "INTEGER PRIMARY KEY",
+                "name": "TEXT",
+                "age": "INTEGER",
+            },
+            "data": [
+                (1, "Tom", 10),
+                (2, "Jerry", 16),
+                (3, "Jack", 18),
+                (4, "Alice", 20),
+                (5, "Bob", 22),
+            ],
+        }
+    }
+)
+
+config = ChromaVectorConfig(persist_path=PILOT_PATH)
+vector_store = ChromaStore(
+    config,
+    name="db_schema_vector_store",
+    embedding_fn=embeddings,
+)
+
+antv_charts = [
+    {"response_line_chart": "used to display comparative trend analysis data"},
+    {
+        "response_pie_chart": "suitable for scenarios such as proportion and distribution statistics"
+    },
+    {
+        "response_table": "suitable for display with many display columns or non-numeric columns"
+    },
+    # {"response_data_text":" the default display method, suitable for single-line or simple content display"},
+    {
+        "response_scatter_plot": "Suitable for exploring relationships between variables, detecting outliers, etc."
+    },
+    {
+        "response_bubble_chart": "Suitable for relationships between multiple variables, highlighting outliers or special situations, etc."
+    },
+    {
+        "response_donut_chart": "Suitable for hierarchical structure representation, category proportion display and highlighting key categories, etc."
+    },
+    {
+        "response_area_chart": "Suitable for visualization of time series data, comparison of multiple groups of data, analysis of data change trends, etc."
+    },
+    {
+        "response_heatmap": "Suitable for visual analysis of time series data, large-scale data sets, distribution of classified data, etc."
+    },
+]
+display_type = "\n".join(
+    f"{key}:{value}" for dict_item in antv_charts for key, value in dict_item.items()
+)
+
+system_prompt = """You are a database expert. Please answer the user's question based on the database selected by the user and some of the available table structure definitions of the database.
+Database name:
+    {db_name}
+Table structure definition:
+    {table_info}
+
+Constraint:
+1.Please understand the user's intention based on the user's question, and use the given table structure definition to create a grammatically correct {dialect} sql. If sql is not required, answer the user's question directly.. 
+2.Always limit the query to a maximum of {top_k} results unless the user specifies in the question the specific number of rows of data he wishes to obtain.
+3.You can only use the tables provided in the table structure information to generate sql. If you cannot generate sql based on the provided table structure, please say: "The table structure information provided is not enough to generate sql queries." It is prohibited to fabricate information at will.
+4.Please be careful not to mistake the relationship between tables and columns when generating SQL.
+5.Please check the correctness of the SQL and ensure that the query performance is optimized under correct conditions.
+6.Please choose the best one from the display methods given below for data rendering, and put the type name into the name parameter value that returns the required format. If you cannot find the most suitable one, use 'Table' as the display method.
+the available data display methods are as follows: {display_type}
+
+User Question:
+    {user_input}
+Please think step by step and respond according to the following JSON format:
+    {response}
+Ensure the response is correct json and can be parsed by Python json.loads.
+"""
+
+RESPONSE_FORMAT_SIMPLE = {
+    "thoughts": "thoughts summary to say to user",
+    "sql": "SQL Query to run",
+    "display_type": "Data display method",
+}
+
+prompt = ChatPromptTemplate(
+    messages=[
+        SystemPromptTemplate.from_template(
+            system_prompt,
+            response_format=json.dumps(
+                RESPONSE_FORMAT_SIMPLE, ensure_ascii=False, indent=4
+            ),
+        ),
+        HumanPromptTemplate.from_template("{user_input}"),
+    ]
+)
+
+
+class TwoSumOperator(MapOperator[pd.DataFrame, int]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, df: pd.DataFrame) -> int:
+        return await self.blocking_func_to_async(self._two_sum, df)
+
+    def _two_sum(self, df: pd.DataFrame) -> int:
+        return df["age"].sum()
+
+
+def branch_even(x: int) -> bool:
+    return x % 2 == 0
+
+
+def branch_odd(x: int) -> bool:
+    return not branch_even(x)
+
+
+class DataDecisionOperator(BranchOperator[int, int]):
+    def __init__(self, odd_task_name: str, even_task_name: str, **kwargs):
+        super().__init__(**kwargs)
+        self.odd_task_name = odd_task_name
+        self.even_task_name = even_task_name
+
+    async def branches(self):
+        return {branch_even: self.even_task_name, branch_odd: self.odd_task_name}
+
+
+class OddOperator(MapOperator[int, str]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, x: int) -> str:
+        print(f"{x} is odd")
+        return f"{x} is odd"
+
+
+class EvenOperator(MapOperator[int, str]):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    async def map(self, x: int) -> str:
+        print(f"{x} is even")
+        return f"{x} is even"
+
+
+class MergeOperator(JoinOperator[str]):
+    def __init__(self, **kwargs):
+        super().__init__(combine_function=self.merge_func, **kwargs)
+
+    async def merge_func(self, odd: str, even: str) -> str:
+        return odd if not is_empty_data(odd) else even
+
+
+with DAG("load_schema_dag") as load_schema_dag:
+    input_task = InputOperator.dummy_input()
+    # Load database schema to vector store
+    assembler_task = DBSchemaAssemblerOperator(
+        connector=db_conn,
+        index_store=vector_store,
+        chunk_parameters=ChunkParameters(chunk_strategy="CHUNK_BY_SIZE"),
+    )
+    input_task >> assembler_task
+
+chunks = asyncio.run(assembler_task.call())
+print(chunks)
+
+
+with DAG("chat_data_dag") as chat_data_dag:
+    input_task = InputOperator(input_source=InputSource.from_callable())
+    retriever_task = DBSchemaRetrieverOperator(
+        top_k=1,
+        index_store=vector_store,
+    )
+    content_task = MapOperator(lambda cks: [c.content for c in cks])
+    merge_task = JoinOperator(
+        lambda table_info, ext_dict: {"table_info": table_info, **ext_dict}
+    )
+    prompt_task = PromptBuilderOperator(prompt)
+    req_build_task = RequestBuilderOperator(model="gpt-3.5-turbo")
+    llm_task = LLMOperator(llm_client=llm_client)
+    sql_parse_task = SQLOutputParser()
+    db_query_task = DatasourceOperator(connector=db_conn)
+
+    (
+        input_task
+        >> MapOperator(lambda x: x["user_input"])
+        >> retriever_task
+        >> content_task
+        >> merge_task
+    )
+    input_task >> merge_task
+    merge_task >> prompt_task >> req_build_task >> llm_task >> sql_parse_task
+    sql_parse_task >> MapOperator(lambda x: x["sql"]) >> db_query_task
+
+    two_sum_task = TwoSumOperator()
+    decision_task = DataDecisionOperator(
+        odd_task_name="odd_task", even_task_name="even_task"
+    )
+    odd_task = OddOperator(task_name="odd_task")
+    even_task = EvenOperator(task_name="even_task")
+    merge_task = MergeOperator()
+
+    db_query_task >> two_sum_task >> decision_task
+    decision_task >> odd_task >> merge_task
+    decision_task >> even_task >> merge_task
+
+final_result = asyncio.run(
+    merge_task.call(
+        {
+            "user_input": "Query the name and age of users younger than 18 years old",
+            "db_name": "user_management",
+            "dialect": "SQLite",
+            "top_k": 1,
+            "display_type": display_type,
+            "response": json.dumps(
+                RESPONSE_FORMAT_SIMPLE, ensure_ascii=False, indent=4
+            ),
+        }
+    )
+)
+print("The final result is:")
+print(final_result)

examples/sdk/simple_sdk_llm_example.py

@@ -0,0 +1,25 @@
+import asyncio
+
+from dbgpt.core import BaseOutputParser
+from dbgpt.core.awel import DAG
+from dbgpt.core.operators import (
+    BaseLLMOperator,
+    PromptBuilderOperator,
+    RequestBuilderOperator,
+)
+from dbgpt.model.proxy import OpenAILLMClient
+
+with DAG("simple_sdk_llm_example_dag") as dag:
+    prompt_task = PromptBuilderOperator(
+        "Write a SQL of {dialect} to query all data of {table_name}."
+    )
+    model_pre_handle_task = RequestBuilderOperator(model="gpt-3.5-turbo")
+    llm_task = BaseLLMOperator(OpenAILLMClient())
+    out_parse_task = BaseOutputParser()
+    prompt_task >> model_pre_handle_task >> llm_task >> out_parse_task
+
+if __name__ == "__main__":
+    output = asyncio.run(
+        out_parse_task.call(call_data={"dialect": "mysql", "table_name": "user"})
+    )
+    print(f"output: \n\n{output}")

examples/sdk/simple_sdk_llm_sql_example.py

@@ -0,0 +1,155 @@
+import asyncio
+import json
+from typing import Dict, List
+
+from dbgpt.core import SQLOutputParser
+from dbgpt.core.awel import (
+    DAG,
+    InputOperator,
+    JoinOperator,
+    MapOperator,
+    SimpleCallDataInputSource,
+)
+from dbgpt.core.operators import (
+    BaseLLMOperator,
+    PromptBuilderOperator,
+    RequestBuilderOperator,
+)
+from dbgpt.datasource.operators.datasource_operator import DatasourceOperator
+from dbgpt.model.proxy import OpenAILLMClient
+from dbgpt.rag.operators.datasource import DatasourceRetrieverOperator
+from dbgpt_ext.datasource.rdbms.conn_sqlite import SQLiteTempConnector
+
+
+def _create_temporary_connection():
+    """Create a temporary database connection for testing."""
+    conn = SQLiteTempConnector.create_temporary_db()
+    conn.create_temp_tables(
+        {
+            "user": {
+                "columns": {
+                    "id": "INTEGER PRIMARY KEY",
+                    "name": "TEXT",
+                    "age": "INTEGER",
+                },
+                "data": [
+                    (1, "Tom", 10),
+                    (2, "Jerry", 16),
+                    (3, "Jack", 18),
+                    (4, "Alice", 20),
+                    (5, "Bob", 22),
+                ],
+            }
+        }
+    )
+    return conn
+
+
+def _sql_prompt() -> str:
+    """This is a prompt template for SQL generation.
+
+    Format of arguments:
+        {db_name}: database name
+        {table_info}: table structure information
+        {dialect}: database dialect
+        {top_k}: maximum number of results
+        {user_input}: user question
+        {response}: response format
+
+    Returns:
+        str: prompt template
+    """
+    return """Please answer the user's question based on the database selected by the user and some of the available table structure definitions of the database. 
+Database name: 
+    {db_name} 
+    
+Table structure definition: 
+    {table_info}
+
+Constraint:
+    1.Please understand the user's intention based on the user's question, and use the given table structure definition to create a grammatically correct {dialect} sql. If sql is not required, answer the user's question directly.. 
+    2.Always limit the query to a maximum of {top_k} results unless the user specifies in the question the specific number of rows of data he wishes to obtain.
+    3.You can only use the tables provided in the table structure information to generate sql. If you cannot generate sql based on the provided table structure, please say: "The table structure information provided is not enough to generate sql queries." It is prohibited to fabricate information at will.
+    4.Please be careful not to mistake the relationship between tables and columns when generating SQL.
+    5.Please check the correctness of the SQL and ensure that the query performance is optimized under correct conditions.
+
+User Question:
+    {user_input}
+Please think step by step and respond according to the following JSON format:
+    {response}
+Ensure the response is correct json and can be parsed by Python json.loads.
+"""
+
+
+def _join_func(query_dict: Dict, db_summary: List[str]):
+    """Join function for JoinOperator.
+
+    Build the format arguments for the prompt template.
+
+    Args:
+        query_dict (Dict): The query dict from DAG input.
+        db_summary (List[str]): The table structure information from DatasourceRetrieverOperator.
+
+    Returns:
+        Dict: The query dict with the format arguments.
+    """
+    default_response = {
+        "thoughts": "thoughts summary to say to user",
+        "sql": "SQL Query to run",
+    }
+    response = json.dumps(default_response, ensure_ascii=False, indent=4)
+    query_dict["table_info"] = db_summary
+    query_dict["response"] = response
+    return query_dict
+
+
+class SQLResultOperator(JoinOperator[Dict]):
+    """Merge the SQL result and the model result."""
+
+    def __init__(self, **kwargs):
+        super().__init__(combine_function=self._combine_result, **kwargs)
+
+    def _combine_result(self, sql_result_df, model_result: Dict) -> Dict:
+        model_result["data_df"] = sql_result_df
+        return model_result
+
+
+with DAG("simple_sdk_llm_sql_example") as dag:
+    db_connection = _create_temporary_connection()
+    input_task = InputOperator(input_source=SimpleCallDataInputSource())
+    retriever_task = DatasourceRetrieverOperator(connector=db_connection)
+    # Merge the input data and the table structure information.
+    prompt_input_task = JoinOperator(combine_function=_join_func)
+    prompt_task = PromptBuilderOperator(_sql_prompt())
+    model_pre_handle_task = RequestBuilderOperator(model="gpt-3.5-turbo")
+    llm_task = BaseLLMOperator(OpenAILLMClient())
+    out_parse_task = SQLOutputParser()
+    sql_parse_task = MapOperator(map_function=lambda x: x["sql"])
+    db_query_task = DatasourceOperator(connector=db_connection)
+    sql_result_task = SQLResultOperator()
+    input_task >> prompt_input_task
+    input_task >> retriever_task >> prompt_input_task
+    (
+        prompt_input_task
+        >> prompt_task
+        >> model_pre_handle_task
+        >> llm_task
+        >> out_parse_task
+        >> sql_parse_task
+        >> db_query_task
+        >> sql_result_task
+    )
+    out_parse_task >> sql_result_task
+
+
+if __name__ == "__main__":
+    input_data = {
+        "db_name": "test_db",
+        "dialect": "sqlite",
+        "top_k": 5,
+        "user_input": "What is the name and age of the user with age less than 18",
+    }
+    output = asyncio.run(sql_result_task.call(call_data=input_data))
+    print(f"\nthoughts: {output.get('thoughts')}\n")
+    print(f"sql: {output.get('sql')}\n")
+    print(f"result data:\n{output.get('data_df')}")

examples/test_files/graphrag-mini.md

@@ -0,0 +1,97 @@
+
+# TuGraph DB项目生态图谱
+Entities:
+(TuGraph-family/tugraph-db#github_repo)
+(vesoft-inc/nebula#github_repo)
+(PaddlePaddle/Paddle#github_repo)
+(apache/brpc#github_repo)
+(TuGraph-family/tugraph-web#github_repo)
+(TuGraph-family/tugraph-db-client-java#github_repo)
+(alibaba/GraphScope#github_repo)
+(ClickHouse/ClickHouse#github_repo)
+(TuGraph-family/fma-common#github_repo)
+(vesoft-inc/nebula-docs-cn#github_repo)
+(eosphoros-ai/DB-GPT#github_repo)
+(eosphoros-ai#github_organization)
+(yandex#github_organization)
+(alibaba#github_organization)
+(TuGraph-family#github_organization)
+(baidu#github_organization)
+(apache#github_organization)
+(vesoft-inc#github_organization)
+
+Relationships:
+(TuGraph-family/tugraph-db#common_developer#vesoft-inc/nebula#common_developer count 10)
+(TuGraph-family/tugraph-db#common_developer#PaddlePaddle/Paddle#common_developer count 9)
+(TuGraph-family/tugraph-db#common_developer#apache/brpc#common_developer count 7)
+(TuGraph-family/tugraph-db#common_developer#TuGraph-family/tugraph-web#common_developer count 7)
+(TuGraph-family/tugraph-db#common_developer#TuGraph-family/tugraph-db-client-java#common_developer count 7)
+(TuGraph-family/tugraph-db#common_developer#alibaba/GraphScope#common_developer count 6)
+(TuGraph-family/tugraph-db#common_developer#ClickHouse/ClickHouse#common_developer count 6)
+(TuGraph-family/tugraph-db#common_developer#TuGraph-family/fma-common#common_developer count 6)
+(TuGraph-family/tugraph-db#common_developer#vesoft-inc/nebula-docs-cn#common_developer count 6)
+(TuGraph-family/tugraph-db#common_developer#eosphoros-ai/DB-GPT#common_developer count 6)
+(eosphoros-ai/DB-GPT#belong_to#eosphoros-ai#belong_to)
+(ClickHouse/ClickHouse#belong_to#yandex#belong_to)
+(alibaba/GraphScope#belong_to#alibaba#belong_to)
+(TuGraph-family/tugraph-db#belong_to#TuGraph-family#belong_to)
+(TuGraph-family/tugraph-web#belong_to#TuGraph-family#belong_to)
+(TuGraph-family/fma-common#belong_to#TuGraph-family#belong_to)
+(TuGraph-family/tugraph-db-client-java#belong_to#TuGraph-family#belong_to)
+(PaddlePaddle/Paddle#belong_to#baidu#belong_to)
+(apache/brpc#belong_to#apache#belong_to)
+(vesoft-inc/nebula#belong_to#vesoft-inc#belong_to)
+(vesoft-inc/nebula-docs-cn#belong_to#vesoft-inc#belong_to)
+
+
+# DB-GPT项目生态图谱
+Entities:
+(eosphoros-ai/DB-GPT#github_repo)
+(chatchat-space/Langchain-Chatchat#github_repo)
+(hiyouga/LLaMA-Factory#github_repo)
+(lm-sys/FastChat#github_repo)
+(langchain-ai/langchain#github_repo)
+(eosphoros-ai/DB-GPT-Hub#github_repo)
+(THUDM/ChatGLM-6B#github_repo)
+(langgenius/dify#github_repo)
+(vllm-project/vllm#github_repo)
+(QwenLM/Qwen#github_repo)
+(PaddlePaddle/PaddleOCR#github_repo)
+(vllm-project#github_organization)
+(eosphoros-ai#github_organization)
+(PaddlePaddle#github_organization)
+(QwenLM#github_organization)
+(THUDM#github_organization)
+(lm-sys#github_organization)
+(chatchat-space#github_organization)
+(langchain-ai#github_organization)
+(langgenius#github_organization)
+
+Relationships:
+(eosphoros-ai/DB-GPT#common_developer#chatchat-space/Langchain-Chatchat#common_developer count 82)
+(eosphoros-ai/DB-GPT#common_developer#hiyouga/LLaMA-Factory#common_developer count 45)
+(eosphoros-ai/DB-GPT#common_developer#lm-sys/FastChat#common_developer count 39)
+(eosphoros-ai/DB-GPT#common_developer#langchain-ai/langchain#common_developer count 37)
+(eosphoros-ai/DB-GPT#common_developer#eosphoros-ai/DB-GPT-Hub#common_developer count 37)
+(eosphoros-ai/DB-GPT#common_developer#THUDM/ChatGLM-6B#common_developer count 31)
+(eosphoros-ai/DB-GPT#common_developer#langgenius/dify#common_developer count 30)
+(eosphoros-ai/DB-GPT#common_developer#vllm-project/vllm#common_developer count 27)
+(eosphoros-ai/DB-GPT#common_developer#QwenLM/Qwen#common_developer count 26)
+(eosphoros-ai/DB-GPT#common_developer#PaddlePaddle/PaddleOCR#common_developer count 24)
+(vllm-project/vllm#belong_to#vllm-project#belong_to)
+(eosphoros-ai/DB-GPT#belong_to#eosphoros-ai#belong_to)
+(eosphoros-ai/DB-GPT-Hub#belong_to#eosphoros-ai#belong_to)
+(PaddlePaddle/PaddleOCR#belong_to#PaddlePaddle#belong_to)
+(QwenLM/Qwen#belong_to#QwenLM#belong_to)
+(THUDM/ChatGLM-6B#belong_to#THUDM#belong_to)
+(lm-sys/FastChat#belong_to#lm-sys#belong_to)
+(chatchat-space/Langchain-Chatchat#belong_to#chatchat-space#belong_to)
+(langchain-ai/langchain#belong_to#langchain-ai#belong_to)
+(langgenius/dify#belong_to#langgenius#belong_to)
+
+
+# TuGraph简介
+TuGraph图数据库由蚂蚁集团与清华大学联合研发，构建了一套包含图存储、图计算、图学习、图研发平台的完善的图技术体系，支持海量多源的关联数据的实时处理，显著提升数据分析效率，支撑了蚂蚁支付、安全、社交、公益、数据治理等300多个场景应用。拥有业界领先规模的图集群，解决了图数据分析面临的大数据量、高吞吐率和低延迟等重大挑战，是蚂蚁集团金融风控能力的重要基础设施，显著提升了欺诈洗钱等金融风险的实时识别能力和审理分析效率，并面向金融、工业、政务服务等行业客户。TuGraph产品家族中，开源产品包括：TuGraph DB、TuGraph Analytics、OSGraph、ChatTuGraph等。内源产品包括：GeaBase、GeaFlow、GeaLearn、GeaMaker等。
+
+# DB-GPT简介
+DB-GPT是一个开源的AI原生数据应用开发框架(AI Native Data App Development framework with AWEL(Agentic Workflow Expression Language) and Agents)。目的是构建大模型领域的基础设施，通过开发多模型管理(SMMF)、Text2SQL效果优化、RAG框架以及优化、Multi-Agents框架协作、AWEL(智能体工作流编排)等多种技术能力，让围绕数据库构建大模型应用更简单，更方便。 

examples/test_files/tranformers_story.md

@@ -0,0 +1,110 @@
+Billions of years ago
+A mysterious mechanoid race, calling themselves the "Creators," continually tried to create life throughout the universe.
+The "Creators" used a type of molecular weapon to transform the surfaces of planets that harbored organic life, converting those organisms into "Transformium": a special metal that could be used to create and repair silicon-based life forms.
+Using the collected "Transformium," the Creators made the planet Cybertron and soon developed the "Cube" to power it, creating a race of sentient mechanical beings there. These beings called themselves Cybertronians and regarded the "Cube" as a sacred object of their creation, naming it the "Allspark."
+Among all Cybertronians created by the "Allspark," the first born seven "Primes" joined forces to establish the "Dynasty of Primes," unifying the entire Cybertron.
+Unlike their descendants, the seven "Primes" did not have the ability to transform, but because they were directly created by the "Cube," their abilities far exceeded other Cybertronians.
+
+4 billion years ago
+Before constructing Cybertron, the "Creators" also experimentally built another giant mechanical metal planet, which, due to a failed experiment, became self-aware - later known as "Unicron."
+In the early days of the "Dynasty of Primes," there were continuous battles against "Unicron," and in the end, with the power of the "Allspark," the seven "Primes" defeated "Unicron" and isolated its consciousness, pushing its body to the edge of the galaxy. "Unicron" later fell into the gravitational pull of a star, orbiting it stably, with interstellar matter gradually wrapping around its body to form a new planet - Earth.
+
+65 million years ago
+The "Creators" visited the solar system, using molecular weapons on Earth's surface to create a large amount of "Transformium." During this process, one of the "Creators," the "Quintessa," also used the energy of the sun to create another batch of silicon-based life forms, which could mimic dinosaurs on Earth and transform at will. Some of them became the later "Dinobots," while others were selected as Quintessa's guardians, the "Knights of Iacon."
+
+60 million years ago
+The "Allspark" had expended much of its energy after creating the seven "Primes," so the later Cybertronians were not created using the "Allspark." The "Dynasty of Primes," through reverse engineering the "Allspark," developed a substance called "Energon" that could achieve a similar effect to creating mechanical life but required energy on a stellar level to produce.
+The rule of the "Dynasty of Primes" on Cybertron stabilized, and due to the enormous stability of silicon-based life, the planet's population began to increase sharply. The seven "Ancestors" started considering expanding their dynasty's territory to other star systems.
+Creating "Energon" required large-scale collection of stellar energy, a process that could cause the core of the star to wither, so the seven "Primes" unanimously agreed not to extract stellar energy from any star systems where life had already appeared.
+
+25,000 years ago
+One of the seven "Primes," "Megatronus Prime," believed he was the chosen one and became dissatisfied with his six brothers' conservative expansion policy. He felt there was no reason to waste time on those primitive and backward carbon-based life forms; all stars Cybertron could utilize should be "harvested."
+Thus, "Megatronus Prime" began to quietly recruit those within the dynasty who shared his ideology, calling them "Decepticons," with his most trusted being "Jetfire" and "Megatron."
+
+"Jetfire"
+"Jetfire" found for "Megatronus Prime" a large number of stars that could be "harvested." Although many of these star systems harbored life, "Megatronus Prime" did not care and claimed within the dynasty that these star systems had no life. They then wantonly harvested the energy of the stars, leading to the extinction of a large number of lives.
+
+17,000 years ago
+In this period, "Megatronus Prime" continuously expanded his and the "Decepticons'" power, reaching a level where they could contend with the "Dynasty of Primes." They began to openly build the "Star Harvester" on Earth, attempting to absorb the energy of the sun to continue building their arsenal. "Megatronus Prime" hoped to force the other six "Primes" to appear on Earth to stop his plan, falling into the trap set by the "Decepticons" and eliminating the leaders of the "Dynasty of Primes."
+
+"Star Harvester"
+However, the repeated extinction of lives caused "Jetfire" to fall into remorse, ultimately leading him to betray "Megatronus Prime" and the "Decepticons." This plan was leaked by "Jetfire," and although it allowed the six "Primes" to avoid immediate danger, they were still no match for the greatly strengthened "Megatronus Prime." As they were about to be eliminated by "Megatronus Prime" one by one, the "Primes" chose to sacrifice their lives, sealing the key to activating the "Star Harvester" - the "Matrix of Leadership" - in a secret location on Earth, and imprisoning "Megatronus Prime" in a dimensionally isolated sarcophagus.
+
+17,000-11,000 years ago
+The sudden disappearance of the seven "Primes" led to a severe civil war on Cybertron, with various factions fighting each other for the limited "Energon." Among these factions, the "Decepticons" were the most powerful and most likely to reunify Cybertron. Still, the fact that "Megatronus Prime" betrayed the "Dynasty of Primes" also led many smaller factions to gradually unite and form the "Autobots." Under the leadership of two descendants of the "Primes," Sentinel Prime and Optimus Prime, they resisted the rule of the "Decepticons."
+
+11,000 years ago
+After years of civil war, the "Autobots" were still inferior in strength to the "Decepticons." To prevent their opponents from getting the "Allspark," Optimus Prime, the former science advisor of Cybertron, had no choice but to launch the "Cube" into deep space.
+At this time, the retreating "Autobots" were unable to resist the "Decepticons'" attacks. Cybertron was ravaged by war. Sentinel Prime believed that if their race were to survive, they needed to find a new path. Thus, he decided to inform Megatron of the "Cube's" whereabouts and secretly agreed to meet on Earth to discuss the future of Cybertron.
+Unexpectedly, while en route to Earth on the "Ark," Sentinel Prime was attacked by the "Decepticons" who were unaware of the plan, its faster-than-light engine was destroyed, causing the "Ark" to drift aimlessly at low speed.
+
+10,000 years ago
+Around 10,000 BC, the "Cube" fell into Earth's atmosphere. Megatron, who followed the Cube, was hit by a meteorite causing him to go off course and crash in the Arctic. Megatron's consciousness was disrupted by the impact, and he was frozen under the polar ice for a long time.
+
+8,000 years ago
+Although the "Decepticons" ultimately won the war, Cybertron had become uninhabitable due to the conflict. Most of them chose to flee to other star systems and live in exile.
+
+7,500 years ago
+As the creators of Cybertron, the "Creators" were not satisfied with this batch of "experimental products," considering them as barbaric and no different from organic life forms. Most of the "Creators" decided to abandon Cybertron and move on to the next creation experiment. Only "Quintessa" believed that Cybertron was still worth saving.
+Thus, Quintessa stayed behind to continue the experiment. She awakened the "Knights of Iacon" that she had created on Earth earlier to serve as her guardians, helping her implement the plan to revive Cybertron.
+
+6,000 BC
+The "Knights of Iacon" learned of Quintessa's plan to absorb the energy of "Unicron" and transfer it to Cybertron to reactivate the planet's core, which would also completely destroy Earth.
+Born on Earth, the "Knights of Iacon" decided to betray their creator, stealing Quintessa's staff and hiding it secretly on Earth.
+
+500 BC
+A young human named Merlin accidentally entered the temple of the "Knights of Iacon." He reached an agreement with these giant intelligent machines to keep their secret in exchange for their assistance when needed to perform "miracles."
+Years later, Merlin used the power of the "Knights of Iacon" to help King Arthur repel the Saxon invasion, earning the knights' trust and becoming the guardian of "Quintessa's Staff."
+Because "Quintessa's Staff" was encoded with Merlin's DNA, only he and his descendants could use it, leading Merlin to create the "Order of Witwiccans" to protect the secrets of the Cybertronians on Earth.
+
+1897
+Archibald Witwicky, a member of the "Order of Witwiccans" and a polar explorer, discovered the crash site of Megatron in the Arctic Circle and "accidentally" activated his navigation system, imprinting the coordinates of the "Cube" on his glasses.
+
+1913
+The "Cube" was discovered upstream of the Colorado River in North America. In response to this significant discovery, the U.S. government established "Sector Seven" to conduct research related to it.
+
+1934
+The crash site of Megatron was secretly excavated by Sector Seven, who then designated him as "NBE-1" (Non-Biological Extraterrestrial), and transported him in cryostasis to a research base beneath the Hoover Dam. Reverse-engineering research on "NBE-1" sparked the information revolution, greatly accelerating human technological progress.
+
+1961
+Human deep-space probes detected a crashed alien spacecraft on the far side of the moon (Sentinel Prime's Ark). President Kennedy's team believed this spacecraft was likely related to "NBE-1" discovered by Sector Seven, hence legislation was signed to initiate the Apollo program, aiming to send humans to the moon to investigate as soon as possible, triggering a space arms race between the USA and the USSR.
+Armstrong discovered several dead "NBEs" inside the Ark, unaware that one was still alive, in a deep sleep within the lower bays of the ship.
+
+1986
+Following in the footsteps of the American moon landing, the Soviet space agency also found the Ark on the moon. They successfully brought back parts of the ship's engine and an energy cell for technical research, but this cell exploded at a facility in Ukraine, causing the Chernobyl nuclear disaster.
+
+2007
+The fallen "The Fallen" (Megatronus Prime) escaped from his sarcophagus prison, believing Optimus Prime would likely send the "Allspark" to Earth, thus summoning the Decepticons to infiltrate Earth and investigate Megatron's whereabouts. Utilizing data collected by Frenzy through the communication network of Air Force One, the Decepticons discovered critical information regarding Archibald Witwicky and "NBE-1."
+Meanwhile, Sam Witwicky, the great-grandson of Archibald Witwicky, unwittingly purchased an old Chevrolet Camaro, which was actually Bumblebee in disguise. With Bumblebee's help, Sam evaded capture by the Decepticons and decided to help the Autobots locate the "Allspark."
+Sector Seven's facility holding "NBE-1" was compromised, Megatron awoke from his frozen state, led Decepticons in preparation to seize the "Allspark," planning to use its energy to transform Earth's mechanical surrogates into life forms and rule this new world. Although the Autobots rushed to assist under the command of Optimus Prime, they couldn't withstand the Decepticons' assault. In a critical moment, Sam shoved the "Allspark" into Megatron's chest, using its energy to destroy his core but also shattered the "Cube" into many pieces.
+
+2009
+Megatron's failure on Earth made The Fallen realize not to underestimate the strength of humans and Autobots. While gathering strength, he summoned Decepticons scattered across other star systems.
+Meanwhile, Soundwave, through a spy satellite, located Megatron's body and remnants of the "Allspark." The remaining Decepticons stole the last piece of the "Allspark" and used it to revive Megatron.
+Under The Fallen's guidance, Megatron and his followers decided to reactivate the Star Harvester hidden within the pyramids to absorb the sun's energy and mass-produce Energon, but this plan was thwarted by the alliance of humans and Autobots. Ultimately, even with The Fallen's personal involvement, he was defeated by Optimus Prime, reborn using the "Matrix of Leadership."
+Seeing the tide was against him, Megatron was persuaded by Starscream to abandon the attack and temporarily retreated to seclusion in the deserts of Africa.
+
+2012
+Autobots and N.E.S.T. discovered components left by the Ark during investigations in Chernobyl, revealing that humans had long been aware of a crashed alien ship on the moon.
+Optimus Prime was extremely dissatisfied with humanity's concealment of this critical information, personally went to the moon and brought back Sentinel Prime, who was in deep stasis. However, the awakened Sentinel Prime quickly turned traitor, killing Ironhide and stealing the Energon cell under Autobot control.
+The so-called "Energon cells" were the activation devices for the Space Bridge developed by Sentinel Prime, allowing objects to be remotely transported to the location of the Energon cell. Sentinel Prime had conspired with Megatron thousands of years ago, planning to transport Cybertron to Earth's location, using the sun's energy to revitalize the dying Cybertron.
+The human enterprise Accuretta Systems, already infiltrated by the Decepticons, deployed all Energon cells within Chicago's city limits. However, when the transport was halfway, Optimus interrupted it, causing Cybertron's celestial body to be severely torn apart, becoming a dead planet.
+Sentinel Prime gained the upper hand in the subsequent one-on-one confrontation with Optimus but was betrayed by Megatron. The escaping Optimus finally destroyed both Megatron and mournfully killed his gravely wounded former mentor Sentinel Prime.
+
+2012-2017
+Following the "Chicago Incident," the U.S. government stopped trusting any Cybertronian, and the newly established anti-alien task force, "Cemetery Wind," began hunting down Decepticons as well as secretly targeting the Autobots. Survivors like Optimus Prime and Bumblebee were forced into hiding.
+Seeing Cybertron, which she created, had become a dead planet, Quintessa decided to initiate her planet revival plan. She slowly moved Cybertron towards the solar system while hiring Lockdown, who remained neutral during the civil war, to capture Optimus Prime and bring the only surviving Prime back to Cybertron to face their Creators.
+Lockdown allied with the technology company KSI, exploiting the company's greed to produce "man-made Transformers" on the condition that they help capture Optimus Prime. KSI used the deceased Megatron to create a more powerful "Galvatron," unaware that it was Megatron's consciousness using KSI to create a new body for himself.
+The reborn Galvatron planned to use the "Seed," a molecular weapon left by Lockdown, to convert Earth's organic beings into Transformium. This time, however, Optimus received the aid of the ancient Dinobots and once again foiled the plans of Megatron/Galvatron.
+Lockdown, intending to personally bring Optimus back to Cybertron, was killed by Bumblebee and Optimus working together.
+After this battle, Optimus decided to take the "Seed" back to Cybertron to question the Creators' true intentions.
+
+2022
+After consecutive Cybertronian invasions, "Unicron," the core of the Earth, showed signs of awakening, with colossal horns appearing around the world and growing in size.
+Optimus Prime, who traveled to Cybertron to confront the Creators, was immediately subdued and brainwashed by Quintessa, transforming him into the fallen warrior "Nemesis Prime."
+Nemesis Prime was sent back to Earth to find “Quintessa's Staff,” which Quintessa intended to connect to an anchor point beneath Stonehenge, draining Unicron's/Earth's energy.
+Viviane Wembley learned she was the last living descendant of Merlin. With the help of the Autobots, she located Merlin's tomb. However, Nemesis Prime emerged, attempting to seize Quintessa's Staff. Bumblebee, using his original voice, caused Optimus's consciousness to resurface.
+Galvatron seized the opportunity of Autobot infighting and took the staff to activate Unicron's anchor point, allowing Cybertron, now orbiting Earth, to absorb the released energy.
+At the same time, Cybertron, pushed by Quintessa into Earth's orbit, collided with Earth, causing millions of deaths. However, with the protection of the human military and Autobots, Viviane entered Cybertron's ignition chamber, removed the staff from the anchor point, and stopped Quintessa's plan.
+Although Earth's core was no longer at risk of withering, Unicron still posed the potential to reawaken. Optimus Prime and the Autobots decided to go to Cybertron, now in a synchronous orbit with Earth, to attempt repairing the planetary core and prepare for Unicron's possible return.
+On Earth, the surviving Quintessa, disguised as a human, continued executing her plan.

examples/test_skills_middleware.py

@@ -0,0 +1,153 @@
+"""Test script for SkillsMiddlewareV2.
+
+This script demonstrates how to use the new middleware system
+to load and use skills in DB-GPT agents.
+"""
+
+import asyncio
+import os
+import sys
+
+sys.path.insert(
+    0, os.path.join(os.path.dirname(__file__), "../../packages/dbgpt-core/src")
+)
+
+from dbgpt.agent.core.agent import AgentContext
+from dbgpt.agent.core.profile.base import ProfileConfig
+from dbgpt.agent.middleware.agent import AgentConfig, MiddlewareAgent
+from dbgpt.agent.skill.middleware_v2 import SkillsMiddlewareV2
+
+
+async def test_skills_middleware():
+    """Test SkillsMiddlewareV2 functionality."""
+
+    skills_path = os.path.join(os.path.dirname(__file__), "skills/user")
+
+    if not os.path.exists(skills_path):
+        print(f"Skills directory not found: {skills_path}")
+        print("Creating test skills...")
+        return
+
+    config = AgentConfig(
+        enable_middleware=True,
+        enable_skills=True,
+        skill_sources=[skills_path],
+        skill_auto_load=True,
+        skill_auto_match=True,
+        skill_inject_to_prompt=True,
+    )
+
+    profile = ProfileConfig(
+        name="assistant",
+        role="AI Assistant",
+        goal="Help users with their tasks using available skills.",
+    )
+
+    agent = MiddlewareAgent(
+        profile=profile,
+        agent_config=config,
+    )
+
+    agent_context = AgentContext(
+        conv_id="test_conv_001",
+        language="zh-CN",
+    )
+
+    await agent.bind(agent_context).build()
+
+    print("\n=== Skills Summary ===")
+    skills = agent.middleware_manager._middlewares[0]  # Get SkillsMiddlewareV2
+    print(skills.get_skills_summary())
+
+    print("\n=== Testing Skill Matching ===")
+    test_inputs = [
+        "research quantum computing",
+        "review my python code",
+        "analyze this data",
+    ]
+
+    for test_input in test_inputs:
+        print(f"\nInput: {test_input}")
+        matched = skills.match_skills(test_input)
+        if matched:
+            print(f"Matched skills: {[s.metadata.name for s in matched]}")
+        else:
+            print("No skills matched")
+
+    print("\n=== Test Complete ===")
+
+
+async def test_custom_middleware():
+    """Test custom middleware."""
+
+    from dbgpt.agent.middleware.base import AgentMiddleware
+
+    class LoggingMiddleware(AgentMiddleware):
+        """Custom middleware for logging."""
+
+        async def before_generate_reply(self, agent, context, **kwargs):
+            print(f"[LoggingMiddleware] Before generate reply")
+            if context and hasattr(context, "message"):
+                print(f"  Message: {context.message.content[:50]}...")
+
+        async def after_generate_reply(self, agent, context, reply_message, **kwargs):
+            print(f"[LoggingMiddleware] After generate reply")
+            if reply_message:
+                print(f"  Reply: {reply_message.content[:50]}...")
+
+        async def modify_system_prompt(self, agent, original_prompt, context=None):
+            modified = (
+                f"\n[LoggingMiddleware] Custom prompt section\n\n{original_prompt}"
+            )
+            return modified
+
+    profile = ProfileConfig(
+        name="assistant",
+        role="AI Assistant",
+    )
+
+    config = AgentConfig(
+        enable_middleware=True,
+        enable_skills=False,  # Disable skills for this test
+    )
+
+    agent = MiddlewareAgent(
+        profile=profile,
+        agent_config=config,
+    )
+
+    logging_middleware = LoggingMiddleware()
+    agent.register_middleware(logging_middleware)
+
+    agent_context = AgentContext(
+        conv_id="test_logging_conv",
+        language="en",
+    )
+
+    await agent.bind(agent_context).build()
+
+    print("\n=== Custom Middleware Test ===")
+    print("LoggingMiddleware has been registered")
+    print(f"Total middleware: {len(agent.middleware_manager._middlewares)}")
+
+    print("\n=== Test Complete ===")
+
+
+async def main():
+    """Run all tests."""
+
+    print("=" * 80)
+    print("DB-GPT Skills Middleware Test")
+    print("=" * 80)
+
+    print("\n\n### Test 1: SkillsMiddlewareV2 ###")
+    await test_skills_middleware()
+
+    print("\n\n### Test 2: Custom Middleware ###")
+    await test_custom_middleware()
+
+    print("\n\n### All Tests Complete ###")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())