chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+337
@@ -0,0 +1,337 @@
|
||||
kind: pipeline # 定义对象类型，还有secret和signature两种类型
|
||||
type: docker # 定义流水线类型，还有kubernetes、exec、ssh等类型
|
||||
name: cicd # 定义流水线名称
|
||||
|
||||
clone:
|
||||
disable: true
|
||||
|
||||
steps: # 定义流水线执行步骤，这些步骤将顺序执行
|
||||
- name: clone
|
||||
image: alpine/git
|
||||
pull: if-not-exists
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
commands:
|
||||
- git config --global core.compression 0
|
||||
- git clone https://github.com/dataelement/bisheng.git .
|
||||
- git checkout $DRONE_COMMIT
|
||||
|
||||
- name: set poetry
|
||||
pull: if-not-exists
|
||||
image: golang
|
||||
environment:
|
||||
RELEASE_VERSION: 99.99.99
|
||||
NEXUS_PUBLIC:
|
||||
from_secret: NEXUS_PUBLIC
|
||||
NEXUS_PUBLIC_PASSWORD:
|
||||
from_secret: NEXUS_PUBLIC_PASSWORD
|
||||
REPO:
|
||||
from_secret: PY_NEXUS
|
||||
PROXY:
|
||||
from_secret: APT-GET
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: bisheng-cache
|
||||
path: /app/build/
|
||||
commands:
|
||||
- cd ./src/backend
|
||||
- echo $REPO
|
||||
- REPO2=$(echo $REPO | sed 's/http:\\/\\///g')
|
||||
- sed '/apt-get/ s|$| '"$PROXY"'|' Dockerfile
|
||||
- sed -i.bak 's/uv cache clean.*$/ /' Dockerfile
|
||||
- sed -i '6i\RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- sed -i '7i\ENV UV_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- sed -i '8i\ENV UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- sed -i '9i\ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- cat Dockerfile
|
||||
|
||||
- name: build_docker
|
||||
pull: if-not-exists
|
||||
image: docker:24.0.6
|
||||
privileged: true
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: apt-cache
|
||||
path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
|
||||
- name: socket
|
||||
path: /var/run/docker.sock
|
||||
- name: pro-cache
|
||||
path: /root/.local/share/pypoetry
|
||||
- name: uv-cache
|
||||
path: /root/.cache/uv
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
no_proxy: 192.168.106.8
|
||||
version: release
|
||||
docker_registry: http://192.168.106.8:6082
|
||||
docker_repo: 192.168.106.8:6082/dataelement/bisheng-backend
|
||||
docker_user:
|
||||
from_secret: NEXUS_USER
|
||||
docker_password:
|
||||
from_secret: NEXUS_PASSWORD
|
||||
commands:
|
||||
- cd ./src/backend/
|
||||
- docker login -u $docker_user -p $docker_password $docker_registry
|
||||
- docker build -t $docker_repo:$version .
|
||||
- docker push $docker_repo:$version
|
||||
|
||||
- name: build_docker_frontend
|
||||
pull: if-not-exists
|
||||
image: docker:24.0.6
|
||||
privileged: true
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: apt-cache
|
||||
path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
|
||||
- name: socket
|
||||
path: /var/run/docker.sock
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
no_proxy: 192.168.106.8
|
||||
version: release
|
||||
docker_registry: http://192.168.106.8:6082
|
||||
docker_repo: 192.168.106.8:6082/dataelement/bisheng-frontend
|
||||
docker_user:
|
||||
from_secret: NEXUS_USER
|
||||
docker_password:
|
||||
from_secret: NEXUS_PASSWORD
|
||||
commands:
|
||||
- cd ./src/frontend/
|
||||
- docker login -u $docker_user -p $docker_password $docker_registry
|
||||
- docker build -t $docker_repo:$version .
|
||||
- docker push $docker_repo:$version
|
||||
|
||||
- name: ssh deploy
|
||||
image: appleboy/drone-ssh
|
||||
pull: if-not-exists
|
||||
settings:
|
||||
host: 192.168.106.116
|
||||
username: root
|
||||
password:
|
||||
from_secret: sshpwd
|
||||
script:
|
||||
- echo =======找到目录=======
|
||||
- cd /opt/server/bisheng-test
|
||||
- echo =======直接启动=======
|
||||
- docker compose pull
|
||||
- docker compose up -d
|
||||
|
||||
- name: notify-start # notify
|
||||
pull: if-not-exists
|
||||
image: plugins/webhook
|
||||
settings:
|
||||
debug: true
|
||||
urls:
|
||||
from_secret: FEISHU_URL
|
||||
content_type: application/json
|
||||
template: |
|
||||
{
|
||||
"msg_type": "interactive",
|
||||
"card": {
|
||||
"type": "template",
|
||||
"data": {
|
||||
"template_id": "AAqkI9bnY5FUs",
|
||||
"template_variable": {
|
||||
"repo_name": "{{ repo.name }}",
|
||||
"build_branch": "{{build.branch}}",
|
||||
"build_author": "{{ DRONE_COMMIT_AUTHOR }}",
|
||||
"link": "{{build.link}}",
|
||||
"commit_msg": "{{ trim build.message }}",
|
||||
"build_tag":"{{build.tag}}",
|
||||
"build_start":"{{build.started}}",
|
||||
"status": "{{ build.status }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
when: # 成功
|
||||
status:
|
||||
- success
|
||||
trigger:
|
||||
branch:
|
||||
- release
|
||||
event:
|
||||
- push
|
||||
|
||||
volumes:
|
||||
- name: bisheng-cache
|
||||
host:
|
||||
path: /opt/drone/data/bisheng/
|
||||
- name: pro-cache
|
||||
host:
|
||||
path: /opt/drone/data/pro/
|
||||
- name: apt-cache
|
||||
host:
|
||||
path: /opt/drone/data/bisheng/apt/
|
||||
- name: socket
|
||||
host:
|
||||
path: /var/run/docker.sock
|
||||
|
||||
|
||||
---
|
||||
|
||||
kind: pipeline # 定义对象类型，还有secret和signature两种类型
|
||||
type: docker # 定义流水线类型，还有kubernetes、exec、ssh等类型
|
||||
name: feat_cicd # 定义流水线名称
|
||||
|
||||
clone:
|
||||
disable: true
|
||||
|
||||
steps: # 定义流水线执行步骤，这些步骤将顺序执行
|
||||
- name: clone
|
||||
image: alpine/git
|
||||
pull: if-not-exists
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
commands:
|
||||
- git config --global core.compression 0
|
||||
- git clone https://github.com/dataelement/bisheng.git .
|
||||
- git checkout $DRONE_COMMIT
|
||||
|
||||
- name: set poetry
|
||||
pull: if-not-exists
|
||||
image: golang
|
||||
environment:
|
||||
NEXUS_PUBLIC:
|
||||
from_secret: NEXUS_PUBLIC
|
||||
NEXUS_PUBLIC_PASSWORD:
|
||||
from_secret: NEXUS_PUBLIC_PASSWORD
|
||||
REPO:
|
||||
from_secret: PY_NEXUS
|
||||
PROXY:
|
||||
from_secret: APT-GET
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: bisheng-cache
|
||||
path: /app/build/
|
||||
commands:
|
||||
- cd ./src/backend
|
||||
- echo $REPO
|
||||
- REPO2=$(echo $REPO | sed 's/http:\\/\\///g')
|
||||
- sed '/apt-get/ s|$| '"$PROXY"'|' Dockerfile
|
||||
- sed -i '6i\RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- sed -i '7i\RUN export UV_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
|
||||
- cat Dockerfile
|
||||
|
||||
- name: build_docker
|
||||
pull: if-not-exists
|
||||
image: docker:24.0.6
|
||||
privileged: true
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: apt-cache
|
||||
path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
|
||||
- name: socket
|
||||
path: /var/run/docker.sock
|
||||
- name: pro-cache
|
||||
path: /root/.local/share/pypoetry
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
no_proxy: 192.168.106.8
|
||||
version: ${DRONE_BRANCH}
|
||||
docker_registry: http://192.168.106.8:6082
|
||||
docker_repo: 192.168.106.8:6082/dataelement/bisheng-backend
|
||||
docker_user:
|
||||
from_secret: NEXUS_USER
|
||||
docker_password:
|
||||
from_secret: NEXUS_PASSWORD
|
||||
commands:
|
||||
- echo "old tag is $version"
|
||||
- version=$(echo $version | sed 's/\\//_/g')
|
||||
- echo "build image tag is $version"
|
||||
- cd ./src/backend/
|
||||
- docker login -u $docker_user -p $docker_password $docker_registry
|
||||
- docker build -t $docker_repo:$version .
|
||||
- docker push $docker_repo:$version
|
||||
|
||||
- name: build_docker_frontend
|
||||
pull: if-not-exists
|
||||
image: docker:24.0.6
|
||||
privileged: true
|
||||
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
|
||||
- name: apt-cache
|
||||
path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
|
||||
- name: socket
|
||||
path: /var/run/docker.sock
|
||||
environment:
|
||||
http_proxy:
|
||||
from_secret: PROXY
|
||||
https_proxy:
|
||||
from_secret: PROXY
|
||||
no_proxy: 192.168.106.8
|
||||
version: ${DRONE_BRANCH}
|
||||
docker_registry: http://192.168.106.8:6082
|
||||
docker_repo: 192.168.106.8:6082/dataelement/bisheng-frontend
|
||||
docker_user:
|
||||
from_secret: NEXUS_USER
|
||||
docker_password:
|
||||
from_secret: NEXUS_PASSWORD
|
||||
commands:
|
||||
- echo "old tag is $version"
|
||||
- version=$(echo $version | sed 's/\\//_/g')
|
||||
- echo "build image tag is $version"
|
||||
- cd ./src/frontend/
|
||||
- docker login -u $docker_user -p $docker_password $docker_registry
|
||||
- docker build -t $docker_repo:$version .
|
||||
- docker push $docker_repo:$version
|
||||
|
||||
- name: notify-start # notify
|
||||
pull: if-not-exists
|
||||
image: plugins/webhook
|
||||
settings:
|
||||
debug: true
|
||||
urls:
|
||||
from_secret: FEISHU_URL
|
||||
content_type: application/json
|
||||
template: |
|
||||
{
|
||||
"msg_type": "interactive",
|
||||
"card": {
|
||||
"type": "template",
|
||||
"data": {
|
||||
"template_id": "AAqkI9bnY5FUs",
|
||||
"template_variable": {
|
||||
"repo_name": "{{ repo.name }}",
|
||||
"build_branch": "{{build.branch}}",
|
||||
"build_author": "{{ DRONE_COMMIT_AUTHOR }}",
|
||||
"link": "{{build.link}}",
|
||||
"commit_msg": "{{ trim build.message }}",
|
||||
"build_tag":"{{build.tag}}",
|
||||
"build_start":"{{build.started}}",
|
||||
"status": "{{ build.status }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
when: # 成功
|
||||
status:
|
||||
- success
|
||||
trigger:
|
||||
branch:
|
||||
- add_some_branch_you_need
|
||||
event:
|
||||
- push
|
||||
|
||||
volumes:
|
||||
- name: bisheng-cache
|
||||
host:
|
||||
path: /opt/drone/data/bisheng/
|
||||
- name: pro-cache
|
||||
host:
|
||||
path: /opt/drone/data/pro/
|
||||
- name: apt-cache
|
||||
host:
|
||||
path: /opt/drone/data/bisheng/apt/
|
||||
- name: socket
|
||||
host:
|
||||
path: /var/run/docker.sock
|
||||
@@ -0,0 +1,34 @@
|
||||
# 默认：自动识别文本，统一用 LF 存库
|
||||
#* text=auto eol=lf
|
||||
|
||||
# 明确常见文本文件用 LF
|
||||
*.py text eol=lf
|
||||
*.sh text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.md text eol=lf
|
||||
*.txt text eol=lf
|
||||
*.json text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.ini text eol=lf
|
||||
|
||||
# Windows 脚本保留 CRLF
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
# 二进制：禁止任何换行转换和 diff
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.ico binary
|
||||
*.pdf binary
|
||||
*.zip binary
|
||||
*.tar binary
|
||||
*.gz binary
|
||||
*.7z binary
|
||||
*.mp4 binary
|
||||
*.docx binary
|
||||
*.xlsx binary
|
||||
*.pptx binary
|
||||
@@ -0,0 +1,129 @@
|
||||
name: BASE_CI
|
||||
|
||||
on:
|
||||
push:
|
||||
# Sequence of patterns matched against refs/tags
|
||||
tags:
|
||||
- "base.v*"
|
||||
|
||||
env:
|
||||
DOCKERHUB_REPO: dataelement/
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build_bisheng_arm:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
# if: startsWith(github.event.ref, 'refs/tags')
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# because ibm-db driver not support linux arm64
|
||||
- name: fix ibm-db lib error
|
||||
run: |
|
||||
# remove ibm-db lib
|
||||
sed -i '/ibm-db*/d' ./src/backend/pyproject.toml
|
||||
|
||||
- name: Build backend arm64 and push
|
||||
id: docker_build_backend
|
||||
run: |
|
||||
docker buildx build --build-arg PANDOC_ARCH=arm64 --file ./src/backend/base.Dockerfile --platform linux/arm64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 --push ./src/backend/
|
||||
|
||||
build_bisheng_amd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build backend amd64 and push
|
||||
id: docker_build_backend
|
||||
run: |
|
||||
docker buildx build --build-arg PANDOC_ARCH=amd64 --file ./src/backend/base.Dockerfile --platform linux/amd64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64 --push ./src/backend/
|
||||
|
||||
|
||||
combine_two_images:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build_bisheng_amd
|
||||
- build_bisheng_arm
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Combine Two images
|
||||
run: |
|
||||
docker manifest create ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }} ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
docker manifest push ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
# 获取提交信息
|
||||
- name: Process git message
|
||||
id: process_message
|
||||
run: |
|
||||
value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
|
||||
value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
|
||||
echo "message=${value}" >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
# 飞书通知
|
||||
- name: notify feishu
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.FEISHU_WEBHOOK }}
|
||||
method: 'POST'
|
||||
data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}发布成功", "content": [[{"tag":"text","text":"基础镜像"},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
|
||||
@@ -0,0 +1,110 @@
|
||||
name: Build Linux Binaries (x86_64 & ARM)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- "build.*"
|
||||
|
||||
jobs:
|
||||
build_pyc:
|
||||
name: Build for pyc files only
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: dataelement/bisheng-telemetry-search
|
||||
token: ${{ secrets.CROSS_REPO_TOKEN }}
|
||||
path: bisheng-telemetry-search
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Build and Clean Artifacts
|
||||
working-directory: ./bisheng-telemetry-search/telemetry_search
|
||||
run: |
|
||||
# 执行构建
|
||||
python -m compileall -b .
|
||||
|
||||
# 在上传前彻底清理源代码和中间文件
|
||||
# 即使 build_all.py 做了清理，这里再次强制清理，防止 .c 文件泄露
|
||||
find . -name "*.py" -delete
|
||||
find . -name "__pycache__" -exec rm -rf {} +
|
||||
|
||||
# 准备输出
|
||||
mkdir ../../build_output
|
||||
cp -r ./* ../../build_output/
|
||||
|
||||
echo "Listing artifacts to be uploaded:"
|
||||
ls -R ../../build_output/
|
||||
|
||||
- name: Upload PYC Artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: telemetry_search
|
||||
path: ./build_output/*
|
||||
|
||||
deploy_to_sso_project:
|
||||
name: Deploy Artifacts to SSO Project
|
||||
needs: [ build_pyc ]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Determine Target Branch
|
||||
id: extract_branch
|
||||
run: |
|
||||
if [[ "${{ github.ref_type }}" == "tag" && "${{ github.ref_name }}" == build.* ]]; then
|
||||
VERSION="${{ github.ref_name }}"
|
||||
VERSION="${VERSION#build.}"
|
||||
TARGET_BRANCH="feat/${VERSION}"
|
||||
else
|
||||
TARGET_BRANCH="feat/2.4.0"
|
||||
fi
|
||||
echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
|
||||
echo "Determined target branch: $TARGET_BRANCH"
|
||||
|
||||
- name: Checkout SSO Project
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: dataelement/bisheng
|
||||
token: ${{ secrets.CROSS_REPO_TOKEN }}
|
||||
# 使用 ref 指定分支，而不是 base
|
||||
ref: ${{ steps.extract_branch.outputs.target_branch }}
|
||||
path: bisheng
|
||||
# 确保拉取完整的历史以便正确提交
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download PYC Artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: telemetry_search
|
||||
path: ./telemetry_search
|
||||
|
||||
- name: Merge and Place Files
|
||||
run: |
|
||||
|
||||
find ./telemetry_search -name "*.py" -delete
|
||||
|
||||
# 移动到目标仓库目录
|
||||
cp -r ./telemetry_search ./bisheng/src/backend/bisheng/
|
||||
|
||||
echo "Final content of target directory:"
|
||||
ls -lh ./bisheng/src/backend/bisheng/telemetry_search
|
||||
|
||||
- name: Commit and Push changes
|
||||
working-directory: ./bisheng
|
||||
run: |
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
git add .
|
||||
|
||||
# 检查是否有变更，有才提交
|
||||
if git diff --staged --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "chore: update telemetry_search build artifacts [skip ci]"
|
||||
git push origin ${{ steps.extract_branch.outputs.target_branch }}
|
||||
fi
|
||||
@@ -0,0 +1,188 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
# Sequence of patterns matched against refs/tags
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
env:
|
||||
DOCKERHUB_REPO: dataelement/
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
||||
build_bisheng_backend:
|
||||
runs-on: ubuntu-latest
|
||||
# if: startsWith(github.event.ref, 'refs/tags')
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=1.3.1" >> $GITHUB_ENV
|
||||
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# 构建 backend 并推送到 Docker hub
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build backend and push
|
||||
id: docker_build_backend
|
||||
run: |
|
||||
docker buildx build --file ./src/backend/Dockerfile --platform linux/amd64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64 --push ./src/backend/
|
||||
|
||||
build_backend_arm:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=1.3.1" >> $GITHUB_ENV
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# - name: Set up QEMU
|
||||
# uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# because ibm-db driver not support linux arm64
|
||||
- name: fix ibm-db lib error
|
||||
run: |
|
||||
# remove ibm-db lib
|
||||
sed -i '/ibm-db*/d' ./src/backend/pyproject.toml
|
||||
|
||||
- name: Build backend and push
|
||||
id: docker_build_backend
|
||||
run: |
|
||||
docker buildx build --file ./src/backend/Dockerfile --platform linux/arm64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 --push ./src/backend/
|
||||
|
||||
build_bisheng_frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build frontend and push
|
||||
id: docker_build_frontend
|
||||
run: |
|
||||
docker buildx build --file ./src/frontend/Dockerfile --platform linux/amd64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64 --push ./src/frontend/
|
||||
|
||||
build_frontend_arm:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Set Environment Variable
|
||||
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
|
||||
|
||||
# - name: Set up QEMU
|
||||
# uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build frontend and push
|
||||
id: docker_build_frontend
|
||||
run: |
|
||||
docker buildx build --file ./src/frontend/Dockerfile --platform linux/arm64 --provenance false --tag ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64 --push ./src/frontend/
|
||||
|
||||
notify_feishu:
|
||||
needs:
|
||||
- build_bisheng_backend
|
||||
- build_backend_arm
|
||||
- build_bisheng_frontend
|
||||
- build_frontend_arm
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Process git message
|
||||
id: process_message
|
||||
run: |
|
||||
value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
|
||||
value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
|
||||
echo "message=${value}" >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
- name: notify feishu
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.FEISHU_WEBHOOK }}
|
||||
method: 'POST'
|
||||
data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}-amd64镜像预发布成功", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
name: PublishRelease
|
||||
|
||||
# 在github上新建release发行版时触发此CICD，主要是把预发布镜像的tag改为正式镜像的tag，并同步到私有镜像仓库
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
env:
|
||||
DOCKERHUB_REPO: dataelement/
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
combine_publish_images:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
- name: Echo version
|
||||
id: echo_version
|
||||
run: |
|
||||
echo "this release is link version: ${{ steps.get_version.outputs.VERSION }}"
|
||||
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Combine two images
|
||||
id: combine_two_images
|
||||
run: |
|
||||
docker manifest create ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }} ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
docker manifest push ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
docker manifest create ${{ env.DOCKERHUB_REPO }}bisheng-backend:latest ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
docker manifest push ${{ env.DOCKERHUB_REPO }}bisheng-backend:latest
|
||||
|
||||
docker manifest create ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }} ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64 ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
docker manifest push ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
docker manifest create ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64 ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
docker manifest push ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
|
||||
|
||||
sync_dataelem_repos:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
- name: Echo version
|
||||
id: echo_version
|
||||
run: |
|
||||
echo "this release is link version: ${{ steps.get_version.outputs.VERSION }}"
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
registry: https://cr.dataelem.com/
|
||||
username: ${{ secrets.CR_DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.CR_DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Sync images
|
||||
id: sync_images
|
||||
run: |
|
||||
echo "sync backend images"
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-amd64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:latest
|
||||
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:latest
|
||||
|
||||
echo "sync frontend images"
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64
|
||||
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-amd64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
|
||||
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
|
||||
|
||||
echo "sync arm image"
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
docker tag ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64 cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
docker push cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}-arm64
|
||||
echo "--- sync over ---"
|
||||
|
||||
test_pull_images:
|
||||
needs:
|
||||
- combine_publish_images
|
||||
- sync_dataelem_repos
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
- name: Echo version
|
||||
id: echo_version
|
||||
run: |
|
||||
echo "this release is link version: ${{ steps.get_version.outputs.VERSION }}"
|
||||
# 登录 cr docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
registry: https://cr.dataelem.com/
|
||||
username: ${{ secrets.CR_DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.CR_DOCKERHUB_TOKEN }}
|
||||
# 登录 docker hub
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
|
||||
# DOCKERHUB_USERNAME 是 docker hub 账号名.
|
||||
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Test pull images
|
||||
run: |
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker pull cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
docker pull ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
docker pull cr.dataelem.com/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
|
||||
notify_feishu:
|
||||
needs:
|
||||
- test_pull_images
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
|
||||
|
||||
- name: Process git message
|
||||
id: process_message
|
||||
run: |
|
||||
value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
|
||||
value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
|
||||
echo "message=${value}" >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
- name: notify feishu
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.FEISHU_WEBHOOK }}
|
||||
method: 'POST'
|
||||
data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}镜像发布成功", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
|
||||
@@ -0,0 +1,100 @@
|
||||
name: test_build
|
||||
|
||||
on:
|
||||
push:
|
||||
# Sequence of patterns matched against refs/tags
|
||||
branches:
|
||||
- "develop/*"
|
||||
|
||||
env:
|
||||
DOCKERHUB_REPO: project/
|
||||
PY_NEXUS: 110.16.193.170:50083
|
||||
DOCKER_NEXUS: 110.16.193.170:50080
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
#if: startsWith(github.event.ref, 'refs/tags')
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Get version
|
||||
id: get_version
|
||||
run: |
|
||||
echo ::set-output name=VERSION::${GITHUB_REF/refs\/heads\/develop\//}
|
||||
echo $GITHUB_REF
|
||||
echo $VERSION
|
||||
|
||||
# 构建 bisheng-langchain
|
||||
- name: Set python version 3.8
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
# 发布到 私有仓库
|
||||
- name: set insecure registry
|
||||
run: |
|
||||
echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
|
||||
sudo service docker restart
|
||||
|
||||
# - name: Set up QEMU
|
||||
# uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: Login Nexus Container Registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: http://${{ env.DOCKER_NEXUS }}/
|
||||
username: ${{ secrets.NEXUS_USER }}
|
||||
password: ${{ secrets.NEXUS_PASSWORD }}
|
||||
|
||||
# 替换poetry编译为私有服务
|
||||
- name: replace self-host repo
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
installer-parallel: true
|
||||
|
||||
- name: build lock
|
||||
run: |
|
||||
cd ./src/backend
|
||||
poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
|
||||
poetry lock
|
||||
cd ../../
|
||||
|
||||
# 构建 backend 并推送到 Docker hub
|
||||
- name: Build backend and push
|
||||
id: docker_build_backend
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
# backend 的context目录
|
||||
context: "./src/backend/"
|
||||
# 是否 docker push
|
||||
push: true
|
||||
# docker build arg, 注入 APP_NAME/APP_VERSION
|
||||
build-args: |
|
||||
APP_NAME="bisheng-backend"
|
||||
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
|
||||
# 生成两个 docker tag: ${APP_VERSION} 和 latest
|
||||
tags: |
|
||||
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
|
||||
# 构建 Docker frontend 并推送到 Docker hub
|
||||
- name: Build frontend and push
|
||||
id: docker_build_frontend
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
# frontend 的context目录
|
||||
context: "./src/frontend/"
|
||||
# 是否 docker push
|
||||
push: true
|
||||
# docker build arg, 注入 APP_NAME/APP_VERSION
|
||||
build-args: |
|
||||
APP_NAME="bisheng-frontend"
|
||||
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
|
||||
# 生成两个 docker tag: ${APP_VERSION} 和 latest
|
||||
tags: |
|
||||
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
|
||||
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
# This is to avoid Opencommit hook from getting pushed
|
||||
prepare-commit-msg
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
qdrant_storage
|
||||
autogen_coding/
|
||||
|
||||
# Mac
|
||||
.DS_Store
|
||||
|
||||
# VSCode
|
||||
.vscode
|
||||
.vscode/settings.json
|
||||
.chroma
|
||||
.ruff_cache
|
||||
.isort.cfg
|
||||
.idea/
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
.isort.cfg
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# TypeScript v1 declaration files
|
||||
typings/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Microbundle cache
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
# *.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variables file
|
||||
.env
|
||||
.env.test
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and *not* Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
notebooks
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
./third_party
|
||||
develop-eggs/
|
||||
config.dev.yaml
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Poetry
|
||||
.testenv/*
|
||||
poetry.lock
|
||||
|
||||
.githooks/prepare-commit-msg
|
||||
.langchain.db
|
||||
|
||||
# docusaurus
|
||||
.docusaurus/
|
||||
|
||||
sftp-config.json
|
||||
|
||||
/tmp/*
|
||||
sftp-config.json
|
||||
|
||||
# Docker local files
|
||||
docker/data/*
|
||||
docker/mysql/data/*
|
||||
docker/office/bisheng/*.gz
|
||||
|
||||
CLAUDE.md
|
||||
!src/backend/bisheng/telemetry_search/**/*.pyc
|
||||
|
||||
docs
|
||||
@@ -0,0 +1,57 @@
|
||||
exclude: ^scripts|docs|docker|requirements|README.md|test|experimental
|
||||
repos:
|
||||
- repo: https://github.com/PyCQA/flake8.git
|
||||
rev: 3.8.3
|
||||
hooks:
|
||||
- id: flake8
|
||||
args: ["--max-line-length=120"]
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.2.0
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
hooks:
|
||||
- id: isort
|
||||
files: \.(py|pyd)$
|
||||
args: ["-l 100"]
|
||||
- repo: https://github.com/pre-commit/mirrors-yapf
|
||||
rev: v0.32.0
|
||||
hooks:
|
||||
- id: yapf
|
||||
files: \.(py|pyd)$
|
||||
args: ["--style={column_limit: 120}"]
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
files: \.(py|pyd)$
|
||||
- id: check-yaml
|
||||
- id: end-of-file-fixer
|
||||
files: \.(py|pyd)$
|
||||
- id: requirements-txt-fixer
|
||||
- id: double-quote-string-fixer
|
||||
- id: check-merge-conflict
|
||||
- id: fix-encoding-pragma
|
||||
args: ["--remove"]
|
||||
- id: mixed-line-ending
|
||||
args: ["--fix=lf"]
|
||||
files: \.(py|pyd)$
|
||||
# - repo: https://github.com/jumanjihouse/pre-commit-hooks
|
||||
# rev: 2.1.4
|
||||
# hooks:
|
||||
# - id: markdownlint
|
||||
# args: ["-r", "~MD002,~MD013,~MD029,~MD033,~MD034,~MD005"]
|
||||
# - repo: https://github.com/myint/docformatter
|
||||
# rev: v1.3.1
|
||||
# hooks:
|
||||
# - id: docformatter
|
||||
# args: ["--in-place", "--wrap-descriptions", "79"]
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: clang-format
|
||||
name: clang-format
|
||||
description: Format files with ClangFormat
|
||||
entry: clang-format -i
|
||||
language: system
|
||||
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$
|
||||
@@ -0,0 +1,125 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, caste, color, religion, or sexual
|
||||
identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the overall
|
||||
community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or advances of
|
||||
any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email address,
|
||||
without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
codeofconduct@globalsecuritydatabase.org.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series of
|
||||
actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or permanent
|
||||
ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within the
|
||||
community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version [v2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html).
|
||||
|
||||
Community Impact Guidelines were inspired by
|
||||
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
[FAQ](https://www.contributor-covenant.org/faq). Translations are available at
|
||||
[translations](https://www.contributor-covenant.org/translations)
|
||||
@@ -0,0 +1,190 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
the copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by the Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributors that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, the Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assuming any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your behalf and Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright © 2024 Dataelement Technologies, Inc
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,109 @@
|
||||
**Proudly made by Chinese，May we, like the creators of Deepseek and Black Myth: Wukong, bring more wonder and greatness to the world.**
|
||||
|
||||
> 源自中国匠心，希望我们能像 [Deepseek]、[黑神话：悟空] 团队一样，给世界带来更多美好。
|
||||
|
||||
<img src="https://dataelem.com/bs/face.png" alt="Bisheng banner">
|
||||
|
||||
<p align="center">
|
||||
<a href="https://dataelem.feishu.cn/wiki/ZxW6wZyAJicX4WkG0NqcWsbynde"><img src="https://img.shields.io/badge/docs-Wiki-brightgreen"></a>
|
||||
<img src="https://img.shields.io/github/license/dataelement/bisheng" alt="license"/>
|
||||
<a href=""><img src="https://img.shields.io/github/last-commit/dataelement/bisheng"></a>
|
||||
<a href="https://star-history.com/#dataelement/bisheng&Timeline"><img src="https://img.shields.io/github/stars/dataelement/bisheng?color=yellow"></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="./README_CN.md">简体中文</a> |
|
||||
<a href="./README.md">English</a> |
|
||||
<a href="./README_JPN.md">日本語</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/717" target="_blank"><img src="https://trendshift.io/api/badge/repositories/717" alt="dataelement%2Fbisheng | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<div class="column" align="middle">
|
||||
<!-- <a href="https://bisheng.slack.com/join/shared_invite/"> -->
|
||||
<!-- <img src="https://img.shields.io/badge/Join-Slack-orange" alt="join-slack"/> -->
|
||||
</a>
|
||||
<!-- <img src="https://img.shields.io/github/license/bisheng-io/bisheng" alt="license"/> -->
|
||||
<!-- <img src="https://img.shields.io/docker/pulls/bisheng-io/bisheng" alt="docker-pull-count" /> -->
|
||||
</div>
|
||||
|
||||
|
||||
BISHENG is an open LLM application devops platform, focusing on enterprise scenarios. It has been used by a large number of industry leading organizations and Fortune 500 companies.
|
||||
|
||||
"Bi Sheng" was the inventor of movable type printing, which played a vital role in promoting the transmission of human knowledge. We hope that BISHENG can also provide strong support for the widespread implementation of intelligent applications. Everyone is welcome to participate.
|
||||
|
||||
|
||||
## Features
|
||||
1. **Lingsight, a general-purpose agent with expert-level taste**: Through the [AGL](https://github.com/dataelement/AgentGuidanceLanguage)(Agent Guidance Language) framework, we embed domain experts’ preferences, experience, and business logic into the AI, enabling the agent to exhibit “expert-level understanding” when handling tasks.
|
||||
<p align="center"><img src="https://dataelem.com/bs/Linsight.png" alt="sence1"></p>
|
||||
|
||||
2. **Unique [BISHENG Workflow](https://dataelem.feishu.cn/wiki/R7HZwH5ZGiJUDrkHZXicA9pInif)**
|
||||
- 🧩 **Independent and comprehensive application orchestration framework**: Enables the execution of various tasks within a single framework (while similar products rely on bot invocation or separate chatflow and workflow modules for different tasks).
|
||||
- 🔄 **Human in the loop**: Allows users to intervene and provide feedback during the execution of workflows (including multi-turn conversations), whereas similar products can only execute workflows from start to finish without intervention.
|
||||
- 💥 **Powerful**: Supports loops, parallelism, batch processing, conditional logic, and free combination of all logic components. It also handles complex scenarios such as multi-type input/output, report generation, content review, and more.
|
||||
- 🖐️ **User-friendly and intuitive**: Operations like loops, parallelism, and batch processing, which require specialized components in similar products, can be easily visualized in BISHENG as a "flowchart" (drawing a loop forms a loop, aligning elements creates parallelism, and selecting multiple items enables batch processing).
|
||||
<p align="center"><img src="https://dataelem.com/bs/bisheng_workflow.png" alt="sence0"></p>
|
||||
|
||||
3. <b>Designed for Enterprise Applications</b>: Document review, fixed-layout report generation, multi-agent collaboration, policy update comparison, support ticket assistance, customer service assistance, meeting minutes generation, resume screening, call record analysis, unstructured data governance, knowledge mining, data analysis, and more.
|
||||
The platform supports the construction of <b>highly complex enterprise application scenarios</b> and offers <b>deep optimization</b> with hundreds of components and thousands of parameters.
|
||||
<p align="center"><img src="https://dataelem.com/bs/chat.png" alt="sence1"></p>
|
||||
|
||||
4. <b>Enterprise-grade</b> features are the fundamental guarantee for application implementation: security review, RBAC, user group management, traffic control by group, SSO/LDAP, vulnerability scanning and patching, high availability deployment solutions, monitoring, statistics, and more.
|
||||
<p align="center"><img src="https://dataelem.com/bs/pro.png" alt="sence2"></p>
|
||||
|
||||
5. <b>High-Precision Document Parsing</b>: Our high-precision document parsing model is trained on a vast amount of high-quality data accumulated over past 5 years. It includes high-precision printed text, handwritten text, and rare character recognition models, table recognition models, layout analysis models, and seal models., table recognition models, layout analysis models, and seal models. You can deploy it privately for free.
|
||||
<p align="center"><img src="https://dataelem.com/bs/ocr.png" alt="sence3"></p>
|
||||
|
||||
6. A community for sharing best practices across various enterprise scenarios: An open repository of application cases and best practices.
|
||||
## Quick start
|
||||
|
||||
Please ensure the following conditions are met before installing BISHENG:
|
||||
- CPU >= 4 Virtual Cores
|
||||
- RAM >= 16 GB
|
||||
- Docker 19.03.9+
|
||||
- Docker Compose 1.25.1+
|
||||
> Recommended hardware condition: 18 virtual cores, 48G. In addition to installing BISHENG, we will also install the following third-party components by default: ES, Milvus, and Onlyoffice.
|
||||
|
||||
Download BISHENG
|
||||
```bash
|
||||
git clone https://github.com/dataelement/bisheng.git
|
||||
# Enter the installation directory
|
||||
cd bisheng/docker
|
||||
|
||||
# If the system does not have the git command, you can download the BISHENG code as a zip file.
|
||||
wget https://github.com/dataelement/bisheng/archive/refs/heads/main.zip
|
||||
# Unzip and enter the installation directory
|
||||
unzip main.zip && cd bisheng-main/docker
|
||||
```
|
||||
Start BISHENG
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -p bisheng up -d
|
||||
```
|
||||
After the startup is complete, access http://IP:3001 in the browser. The login page will appear, proceed with user registration.
|
||||
|
||||
By default, the first registered user will become the system admin.
|
||||
|
||||
For more installation and deployment issues, refer to:：[Self-hosting](https://dataelem.feishu.cn/wiki/BSCcwKd4Yiot3IkOEC8cxGW7nPc)
|
||||
|
||||
## Acknowledgement
|
||||
This repo benefits from [langchain](https://github.com/langchain-ai/langchain) [langflow](https://github.com/logspace-ai/langflow) [unstructured](https://github.com/Unstructured-IO/unstructured) and [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) . Thanks for their wonderful works.
|
||||
|
||||
<b>Thank you to our contributors：</b>
|
||||
|
||||
<a href="https://github.com/dataelement/bisheng/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=dataelement/bisheng" />
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
## Community & contact
|
||||
Welcome to join our discussion group
|
||||
|
||||
<img src="https://www.dataelem.com/nstatic/qrcode.png" alt="Wechat QR Code">
|
||||
|
||||
|
||||
<!--
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#dataelement/bisheng&Date)
|
||||
-->
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`dataelement/bisheng`
|
||||
- 原始仓库：https://github.com/dataelement/bisheng
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
<img src="https://dataelem.com/bs/face.png" alt="Bisheng banner">
|
||||
|
||||
<p align="center">
|
||||
<a href="https://dataelem.feishu.cn/wiki/ZxW6wZyAJicX4WkG0NqcWsbynde"><img src="https://img.shields.io/badge/docs-Wiki-brightgreen"></a>
|
||||
<img src="https://img.shields.io/github/license/dataelement/bisheng" alt="license"/>
|
||||
<img src="https://img.shields.io/docker/pulls/dataelement/bisheng-frontend" alt="docker-pull-count" />
|
||||
<a href=""><img src="https://img.shields.io/github/last-commit/dataelement/bisheng"></a>
|
||||
<a href="https://star-history.com/#dataelement/bisheng&Timeline"><img src="https://img.shields.io/github/stars/dataelement/bisheng?color=yellow"></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="./README_CN.md">简体中文</a> |
|
||||
<a href="./README.md">English</a> |
|
||||
<a href="./README_JPN.md">日本語</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/717" target="_blank"><img src="https://trendshift.io/api/badge/repositories/717" alt="dataelement%2Fbisheng | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<div class="column" align="middle">
|
||||
<!-- <a href="https://bisheng.slack.com/join/shared_invite/"> -->
|
||||
<!-- <img src="https://img.shields.io/badge/Join-Slack-orange" alt="join-slack"/> -->
|
||||
</a>
|
||||
<!-- <img src="https://img.shields.io/github/license/bisheng-io/bisheng" alt="license"/> -->
|
||||
<!-- <img src="https://img.shields.io/docker/pulls/bisheng-io/bisheng" alt="docker-pull-count" /> -->
|
||||
</div>
|
||||
|
||||
|
||||
BISHENG毕昇 是一款 <b>开源</b> LLM应用开发平台，主攻<b>企业场景</b>， 已有大量行业头部组织及世界500强企业在使用。
|
||||
|
||||
“毕昇”是活字印刷术的发明人，活字印刷术为人类知识的传递起到了巨大的推动作用。我们希望“BISHENG毕昇”同样能够为智能应用的广泛落地提供有力支撑。欢迎大家一道参与。
|
||||
|
||||
|
||||
## 特点
|
||||
1. **具备专家级品味的通用Agent灵思**：通过 [AGL](https://github.com/dataelement/AgentGuidanceLanguage)（Agent Guidance Language）框架，将领域专家的偏好、经验与业务逻辑融入 AI 之中，让 Agent 在处理任务时能具备 「专家级理解」。
|
||||
<p align="center"><img src="https://dataelem.com/bs/Linsight.png" alt="sence1"></p>
|
||||
|
||||
2. **独具特色的[BISHENG workflow](https://dataelem.feishu.cn/wiki/R7HZwH5ZGiJUDrkHZXicA9pInif)**
|
||||
|
||||
- 🧩 **独立、完备的应用编排框架**：可在一个框架下实现各类任务（同类产品需要被 bot 调用，或划分成 chatflow 与 workflow 来完成不同类型的任务）。
|
||||
- 🔄 **Human in the loop**：支持用户在Workflow执行的中间过程进行干预和反馈（包括多轮对话），而同类产品只能从头执行到尾。
|
||||
- 💥 **强大**：支持成环、并行、跑批、判断逻辑以及所有逻辑的任意自由组合；支持多类型输入输出、撰写报告、内容审核等复杂场景。
|
||||
- 🖐️ **易用、符合直觉**：如成环、并行、批量运行操作，在同类产品中用户需借助专门组件实现，在BISHENG中只需完全按照直觉连接成“流程图”即可（画圈成环、并列即并行、多选即批量）。
|
||||
<p align="center"><img src="https://dataelem.com/bs/bisheng_workflow.png" alt="sence0"></p>
|
||||
|
||||
3. **专为企业应用而生**：文档审核、固定版式报告生成、多智能体协作、规范制度更新差异比对、工单问答、客服辅助、会议纪要生成、简历筛选、通话记录分析、非结构化数据治理、知识挖掘、数据分析...平台支持高复杂度企业应用场景构建，支持数百个组件与数千个参数的深度调优。
|
||||
<p align="center"><img src="https://dataelem.com/bs/chat.png" alt="sence1"></p>
|
||||
|
||||
4. **企业级特性是应用落地的基本保障**：安全审查、基于角色的细颗粒度权限管理、用户组管理、分组流量控制、SSO/LDAP、漏洞扫描修复、高可用部署方案、监控、统计...
|
||||
<p align="center"><img src="https://dataelem.com/bs/pro.png" alt="sence2"></p>
|
||||
|
||||
5. **高精度文档解析**：5年海量数据沉淀，高精度文档解析模型支持免费私有化部署使用，包括高精度印刷体、手写体与生僻字识别模型、表格识别模型、版式分析模型、印章模型...
|
||||
<p align="center"><img src="https://dataelem.com/bs/ocr.png" alt="sence3"></p>
|
||||
|
||||
6. **大量企业场景落地最佳实践分享社区**：开放的应用案例与最佳实践库。
|
||||
<p align="center"><img src="https://dataelem.com/bs/sence.png" alt="sence4"></p>
|
||||
|
||||
|
||||
## 快速安装
|
||||
|
||||
安装BISHENG前请先确保满足以下条件：
|
||||
- CPU >= 8 Core
|
||||
- RAM >= 32 GB
|
||||
- Docker 19.03.9+
|
||||
- Docker Compose 1.25.1+
|
||||
> 除了BISHENG前后端，我们默认还会安装第三方组件ES、Milvus、Onlyoffice
|
||||
|
||||
下载BISHENG代码
|
||||
```bash
|
||||
# 如果系统中有git命令，可以直接下载毕昇代码
|
||||
git clone https://github.com/dataelement/bisheng.git
|
||||
# 进入安装目录
|
||||
cd bisheng/docker
|
||||
|
||||
# 如果系统没有没有git命令，可以下载毕昇代码zip包
|
||||
wget https://github.com/dataelement/bisheng/archive/refs/heads/main.zip
|
||||
# 解压并进入安装目录
|
||||
unzip main.zip && cd bisheng-main/docker
|
||||
```
|
||||
启动BISHENG
|
||||
```bash
|
||||
# 进入bisheng/docker或bisheng-main/docker目录，执行
|
||||
docker compose -f docker-compose.yml -p bisheng up -d
|
||||
```
|
||||
启动后，在浏览器中访问 http://IP:3001 ，出现登录页，进行用户注册。默认第一个注册的用户会成为系统admin。
|
||||
|
||||
其他安装部署问题参考：[私有化部署](https://dataelem.feishu.cn/wiki/BSCcwKd4Yiot3IkOEC8cxGW7nPc)
|
||||
|
||||
|
||||
## 资源
|
||||
- [📄应用案例/场景库](https://dataelem.feishu.cn/wiki/ZfkmwLPfeiAhQSkK2WvcX87unxc)
|
||||
- [📄经验技巧](https://dataelem.feishu.cn/wiki/OWFRwknFaiIMajke4m5cFeLrnie)
|
||||
- [📄功能使用说明](https://dataelem.feishu.cn/wiki/WxH6wubbAiBkRIkSEyecmpDMnjF)
|
||||
- [📄BISHENG Blog](https://dataelem.feishu.cn/wiki/BiNowcaYWilewdksXQ5cZl3tnzy)
|
||||
|
||||
|
||||
## 感谢
|
||||
|
||||
感谢我们的贡献者：
|
||||
|
||||
<a href="https://github.com/dataelement/bisheng/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=dataelement/bisheng" />
|
||||
</a>
|
||||
|
||||
|
||||
<br>
|
||||
Bisheng 采用了以下依赖库:
|
||||
|
||||
- 感谢开源LLM应用开发库 [langchain](https://github.com/langchain-ai/langchain)。
|
||||
- 感谢开源langchain可视化工具 [langflow](https://github.com/logspace-ai/langflow)。
|
||||
- 感谢开源非结构化数据解析引擎 [unstructured](https://github.com/Unstructured-IO/unstructured)。
|
||||
- 感谢开源LLM微调框架 [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) 。
|
||||
|
||||
|
||||
## 社区与支持
|
||||
欢迎加入我们的交流群
|
||||
|
||||
<img src="https://www.dataelem.com/nstatic/qrcode.png" alt="Wechat QR Code">
|
||||
|
||||
|
||||
<!--
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#dataelement/bisheng&Date)
|
||||
-->
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
以下は、あなたが提供したMarkdownコンテンツの日本語翻訳です。
|
||||
|
||||
---
|
||||
|
||||
<img src="https://dataelem.com/bs/face.png" alt="Bisheng banner">
|
||||
|
||||
<p align="center">
|
||||
<a href="https://dataelem.feishu.cn/wiki/ZxW6wZyAJicX4WkG0NqcWsbynde"><img src="https://img.shields.io/badge/docs-Wiki-brightgreen"></a>
|
||||
<img src="https://img.shields.io/github/license/dataelement/bisheng" alt="license"/>
|
||||
<img src="https://img.shields.io/docker/pulls/dataelement/bisheng-frontend" alt="docker-pull-count" />
|
||||
<a href=""><img src="https://img.shields.io/github/last-commit/dataelement/bisheng"></a>
|
||||
<a href="https://star-history.com/#dataelement/bisheng&Timeline"><img src="https://img.shields.io/github/stars/dataelement/bisheng?color=yellow"></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="./README_CN.md">简体中文</a> |
|
||||
<a href="./README.md">English</a> |
|
||||
<a href="./README_JPN.md">日本語</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/717" target="_blank"><img src="https://trendshift.io/api/badge/repositories/717" alt="dataelement%2Fbisheng | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<div class="column" align="middle">
|
||||
<!-- <a href="https://bisheng.slack.com/join/shared_invite/"> -->
|
||||
<!-- <img src="https://img.shields.io/badge/Join-Slack-orange" alt="join-slack"/> -->
|
||||
</a>
|
||||
<!-- <img src="https://img.shields.io/github/license/bisheng-io/bisheng" alt="license"/> -->
|
||||
<!-- <img src="https://img.shields.io/docker/pulls/bisheng-io/bisheng" alt="docker-pull-count" /> -->
|
||||
</div>
|
||||
|
||||
BISHENGは、エンタープライズシナリオに焦点を当てたオープンなLLMアプリケーションDevOpsプラットフォームです。多くの業界リーディング企業やフォーチュン500企業で使用されています。
|
||||
|
||||
「畢昇（Bi Sheng）」は、活版印刷の発明者であり、人類の知識の伝播に重要な役割を果たしました。我々は、BISHENGがインテリジェントアプリケーションの広範な実装に強力なサポートを提供できることを願っています。皆さんの参加を歓迎します。
|
||||
|
||||
## 特徴
|
||||
1. **専門家級のセンスを備えた汎用エージェント「灵思」：**：[AGL](https://github.com/dataelement/AgentGuidanceLanguage)（Agent Guidance Language）フレームワークを通じて、分野の専門家の志向・経験・業務ロジックをAIに組み込み、エージェントがタスク処理時に「専門家レベルの理解」を備えられるようにします。
|
||||
<p align="center"><img src="https://dataelem.com/bs/Linsight.png" alt="sence1"></p>
|
||||
|
||||
2. **独自の特徴を持つ[BISHENG workflow](https://dataelem.feishu.cn/wiki/R7HZwH5ZGiJUDrkHZXicA9pInif)**
|
||||
|
||||
- 🧩 **独立性と完備性を備えたアプリケーションオーケストレーションフレームワーク**：1つのフレームワーク内でさまざまなタスクを実現可能（類似製品では、botの呼び出しが必要だったり、chatflowとworkflowに分けて異なるタスクを処理する必要があります）。
|
||||
- 🔄 **Human in the loop**：Workflowの実行途中でユーザーが介入やフィードバック（多ターン対話を含む）を行えます（類似製品では最初から最後まで一貫して実行されるのみ）。
|
||||
- 💥 **強力な機能**：ループ化、並列処理、一括処理、条件分岐ロジック、さらにこれら全ての自由な組み合わせが可能です。多種類の入出力、レポート作成、コンテンツ審査といった複雑なシナリオも対応可能。
|
||||
- 🖐️ **直感的で使いやすい**：類似製品では専用のコンポーネントを使用する必要があるループ化、並列処理、一括処理操作も、BISHENGでは直感的に「フローチャート」として接続するだけで実現可能です（円を描けばループ化、並列に配置すれば並列処理、複数選択すれば一括処理）。
|
||||
|
||||
<p align="center"><img src="https://dataelem.com/bs/bisheng_workflow.png" alt="sence0"></p>
|
||||
|
||||
3. **エンタープライズアプリケーション向けに設計**: ドキュメントレビュー、固定レイアウトレポート生成、マルチエージェント協働、ポリシー更新比較、サポートチケット支援、カスタマーサービス支援、会議議事録生成、履歴書スクリーニング、通話記録分析、非構造化データガバナンス、知識採掘、データ分析など。プラットフォームは、**高度に複雑なエンタープライズアプリケーションシナリオの構築**をサポートし、**深い最適化**を行い、数百のコンポーネントと数千のパラメータを提供します。
|
||||
<p align="center"><img src="https://dataelem.com/bs/chat.png" alt="sence1"></p>
|
||||
|
||||
4. **エンタープライズグレード**の機能は、アプリケーション実装の基本的な保証です: セキュリティレビュー、RBAC、ユーザーグループ管理、グループごとのトラフィックコントロール、SSO/LDAP、脆弱性スキャンとパッチ適用、高可用性デプロイメントソリューション、モニタリング、統計など。
|
||||
<p align="center"><img src="https://dataelem.com/bs/pro.png" alt="sence2"></p>
|
||||
|
||||
5. **高精度ドキュメント解析**: 私たちの高精度ドキュメント解析モデルは、過去5年間にわたる大量の高品質データに基づいてトレーニングされています。高精度な印刷テキスト、手書きテキスト、稀少文字認識モデル、テーブル認識モデル、レイアウト解析モデル、印鑑モデルを含みます。プライベートに無料で展開することができます。
|
||||
<p align="center"><img src="https://dataelem.com/bs/ocr.png" alt="sence3"></p>
|
||||
|
||||
6. 様々なエンタープライズシナリオにおけるベストプラクティスを共有するコミュニティ: オープンなアプリケーションケースとベストプラクティスのリポジトリ。
|
||||
|
||||
|
||||
## クイックスタート
|
||||
|
||||
BISHENGをインストールする前に、以下の条件を満たしていることを確認してください:
|
||||
- CPU >= 8 コア
|
||||
- RAM >= 32 GB
|
||||
- Docker 19.03.9以上
|
||||
- Docker Compose 1.25.1以上
|
||||
|
||||
> BISHENGをインストールする際、デフォルトで以下のサードパーティコンポーネントもインストールされます: ES, Milvus, Onlyoffice。
|
||||
|
||||
BISHENGのダウンロード
|
||||
```bash
|
||||
git clone https://github.com/dataelement/bisheng.git
|
||||
# インストールディレクトリに移動
|
||||
cd bisheng/docker
|
||||
|
||||
# システムにgitコマンドがない場合は、BISHENGのコードをzipファイルとしてダウンロードできます。
|
||||
wget https://github.com/dataelement/bisheng/archive/refs/heads/main.zip
|
||||
# 解凍してインストールディレクトリに移動
|
||||
unzip main.zip && cd bisheng-main/docker
|
||||
```
|
||||
|
||||
BISHENGの起動
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -p bisheng up -d
|
||||
```
|
||||
|
||||
起動完了後、ブラウザでhttp://IP:3001にアクセスします。ログインページが表示されるので、ユーザー登録を行います。
|
||||
|
||||
デフォルトでは、最初に登録されたユーザーがシステム管理者となります。
|
||||
|
||||
詳細なインストールおよびデプロイに関する問題は、こちらを参照してください：[私有化部署](https://dataelem.feishu.cn/wiki/BSCcwKd4Yiot3IkOEC8cxGW7nPc)
|
||||
|
||||
## 謝辞
|
||||
このリポジトリは [langchain](https://github.com/langchain-ai/langchain) [langflow](https://github.com/logspace-ai/langflow) [unstructured](https://github.com/Unstructured-IO/unstructured) および [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) の恩恵を受けています。素晴らしい作品に感謝します。
|
||||
|
||||
**貢献者に感謝します：**
|
||||
|
||||
<a href="https://github.com/dataelement/bisheng/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=dataelement/bisheng" />
|
||||
</a>
|
||||
|
||||
## コミュニティと連絡先
|
||||
ディスカッショングループへの参加を歓迎します。
|
||||
|
||||
<img src="https://www.dataelem.com/nstatic/qrcode.png" alt="Wechat QR Code">
|
||||
|
||||
---
|
||||
|
||||
この翻訳を使用して、Markdownファイルを作成できます。
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting Security Issues
|
||||
|
||||
We take the security of our project seriously. If you believe you have found a security vulnerability, please report it to us privately. **Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.**
|
||||
|
||||
> **Important Note**: Any code within the `classic/` folder is considered legacy, unsupported, and out of scope for security reports. We will not address security vulnerabilities in this deprecated code.
|
||||
|
||||
Instead, please report them via:
|
||||
- [GitHub Security Advisory](https://github.com/dataelement/bisheng/security/advisories/new)
|
||||
<!--- [Huntr.dev](https://huntr.com/repos/significant-gravitas/autogpt) - where you may be eligible for a bounty-->
|
||||
|
||||
### Reporting Process
|
||||
1. **Submit Report**: Use one of the above channels to submit your report
|
||||
2. **Response Time**: Our team will acknowledge receipt of your report within 14 business days.
|
||||
3. **Collaboration**: We will collaborate with you to understand and validate the issue
|
||||
4. **Resolution**: We will work on a fix and coordinate the release process
|
||||
|
||||
|
||||
### Disclosure Policy
|
||||
- Please provide detailed reports with reproducible steps
|
||||
- Include the version/commit hash where you discovered the vulnerability
|
||||
- Allow us a 90-day security fix window before any public disclosure
|
||||
- Share any potential mitigations or workarounds if known
|
||||
|
||||
## Supported Versions
|
||||
Only the following versions are eligible for security updates:
|
||||
|
||||
| Version | Supported |
|
||||
|---------|-----------|
|
||||
| Latest release on master branch | ✅ |
|
||||
| Development commits (pre-master) | ✅ |
|
||||
| Classic folder (deprecated) | ❌ |
|
||||
| All other versions | ❌ |
|
||||
|
||||
|
||||
|
||||
---
|
||||
Last updated: November 2024
|
||||
@@ -0,0 +1,3 @@
|
||||
# Celery 的broker配置。存储ft指令执行结果。
|
||||
# 密码加密规则和backend一致, 暂不支持集群redis
|
||||
redis_url: "redis://bisheng-redis:6379/5"
|
||||
@@ -0,0 +1,57 @@
|
||||
# 可根据loguru的文档配置不同 handlers
|
||||
logger_conf:
|
||||
# 默认输出到控制台的日志级别, 大于等于此级别都会输出
|
||||
level: DEBUG
|
||||
# 默认输出格式
|
||||
format: '[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}] [{level.name} process-{process.id}-{thread.id} {name}:{line}] - trace={extra[trace_id]} {message}'
|
||||
# 参考loguru.add()中的参数可以配置多个handler
|
||||
handlers:
|
||||
# 文件路径，支持插入一些系统环境变量，若环境变量不存在则置空。例如 HOSTNAME: 主机名。后端会处理环境变量的替换
|
||||
- sink: "/app/logs/bisheng_uns.log"
|
||||
# 日志级别
|
||||
level: INFO
|
||||
# 和原生不一样，后端会将配置使用eval()执行转为函数用来过滤特定日志级别。推荐lambda
|
||||
# filter: "lambda record: record['level'].name == 'INFO'"
|
||||
# 日志格式化函数，extra内支持trace_id
|
||||
format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}|{process.id}|{thread.id}|{message}"
|
||||
# 每天的几点进行切割
|
||||
rotation: "00:00"
|
||||
retention: "3 Days"
|
||||
- sink: "/app/logs/err-v0-BISHENG-UNS-{HOSTNAME}.log"
|
||||
level: ERROR
|
||||
filter: "lambda record: record['level'].name == 'ERROR'"
|
||||
format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}||{process.id}|{thread.id}|||#EX_ERR:POS={name},line {line},ERR=500,EMSG={message}"
|
||||
rotation: "00:00"
|
||||
retention: "3 Days"
|
||||
|
||||
# pdf解析需要用到的模型配置, 配置了rt_server环境变量的话会替换为对应的地址
|
||||
pdf_model_params:
|
||||
layout_ep: "http://192.168.106.12:9001/v2.1/models/elem_layout_v1/infer"
|
||||
cell_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_cell_detect_v1/infer"
|
||||
rowcol_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_rowcol_detect_v1/infer"
|
||||
table_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_detect_v1/infer"
|
||||
ocr_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_ocr_collection_v3/infer"
|
||||
|
||||
# 是否全部走ocr识别, false的话则由代码逻辑判断是否需要走ocr识别
|
||||
is_all_ocr: false
|
||||
# ocr识别需要的配置项
|
||||
ocr_conf:
|
||||
params:
|
||||
sort_filter_boxes: true,
|
||||
enable_huarong_box_adjust: true,
|
||||
rotateupright: false,
|
||||
support_long_image_segment: true,
|
||||
split_long_sentence_blank: true
|
||||
scene_mapping:
|
||||
print:
|
||||
det: general_text_det_mrcnn_v2.0
|
||||
recog: transformer-blank-v0.2-faster
|
||||
hand:
|
||||
det: general_text_det_mrcnn_v2.0
|
||||
recog: transformer-hand-v1.16-faster
|
||||
print_recog:
|
||||
recog: transformer-blank-v0.2-faster
|
||||
hand_recog:
|
||||
recog: transformer-hand-v1.16-faster
|
||||
det:
|
||||
det: general_text_det_mrcnn_v2.0
|
||||
@@ -0,0 +1,87 @@
|
||||
# 数据库配置， 当前加密串的密码是1234，
|
||||
# 密码加密参考 https://dataelem.feishu.cn/wiki/BSCcwKd4Yiot3IkOEC8cxGW7nPc#Gxitd1xEeof1TzxdhINcGS6JnXd
|
||||
database_url:
|
||||
"mysql+pymysql://root:gAAAAABlp4b4c59FeVGF_OQRVf6NOUIGdxq8246EBD-b0hdK_jVKRs1x4PoAn0A6C5S6IiFKmWn0Nm5eBUWu-7jxcqw6TiVjQA==@mysql:3306/bisheng?charset=utf8mb4"
|
||||
|
||||
# 缓存配置 redis://[[username]:[password]]@localhost:6379/0
|
||||
# 如果设置了密码，需要参考MySQL密码的加密逻辑对密码进行加密。eg: redis://root:gAAAAABlp4b4c59FeVGF_OQRVf6NOUIGdxq8246EBD-b0hdK_jVKRs1x4PoAn0A6C5S6IiFKmWn0Nm5eBUWu-7jxcqw6TiVjQA==@redis:6379/0
|
||||
# 普通模式:
|
||||
redis_url: "redis://redis:6379/1"
|
||||
|
||||
# 集群模式或者哨兵模式（只能选其一）:
|
||||
# redis_url:
|
||||
# mode: "cluster"
|
||||
# startup_nodes:
|
||||
# - {"host": "192.168.106.115", "port": 6002}
|
||||
# password: encrypt(gAAAAABlp4b4c59FeVGF_OQRVf6NOUIGdxq8246EBD-b0hdK_jVKRs1x4PoAn0A6C5S6IiFKmWn0Nm5eBUWu-7jxcqw6TiVjQA==)
|
||||
# #sentinel
|
||||
# mode: "sentinel"
|
||||
# sentinel_hosts: [("redis", 6379)]
|
||||
# sentinel_master: "mymaster"
|
||||
# sentinel_password: encrypt(gAAAAABlp4b4c59FeVGF_OQRVf6NOUIGdxq8246EBD-b0hdK_jVKRs1x4PoAn0A6C5S6IiFKmWn0Nm5eBUWu-7jxcqw6TiVjQA==)
|
||||
# db: 1
|
||||
|
||||
# celery的broken地址
|
||||
celery_redis_url: "redis://redis:6379/2"
|
||||
celery_task:
|
||||
# 对celery熟悉的用户可以自定义配置任务的路由，启动不同类型的worker处理不同类型的异步任务。注意工作流的执行只能在一个进程内！！！
|
||||
task_routers:
|
||||
bisheng.worker.knowledge.*: # 知识库文件处理相关任务
|
||||
queue: knowledge_celery
|
||||
bisheng.worker.workflow.*: # 工作流相关任务
|
||||
queue: workflow_celery
|
||||
|
||||
# 知识库的milvus和es配置 支持使用 !env ${PATH} 填写环境变量的值, 若环境变量不存在则会报错
|
||||
vector_stores:
|
||||
milvus:
|
||||
connection_args: !env ${BS_MILVUS_CONNECTION_ARGS}
|
||||
is_partition: !env ${BS_MILVUS_IS_PARTITION}
|
||||
partition_suffix: !env ${BS_MILVUS_PARTITION_SUFFIX}
|
||||
elasticsearch:
|
||||
url: !env ${BS_ELASTICSEARCH_URL}
|
||||
ssl_verify: !env ${BS_ELASTICSEARCH_SSL_VERIFY}
|
||||
|
||||
|
||||
# 对象存储， 目前只支持minio
|
||||
object_storage:
|
||||
type: minio
|
||||
minio:
|
||||
schema: !env ${BS_MINIO_SCHEMA}
|
||||
cert_check: !env ${BS_MINIO_CERT_CHECK}
|
||||
endpoint: !env ${BS_MINIO_ENDPOINT}
|
||||
sharepoint: !env ${BS_MINIO_SHAREPOINT}
|
||||
access_key: !env ${BS_MINIO_ACCESS_KEY}
|
||||
secret_key: !env ${BS_MINIO_SECRET_KEY}
|
||||
public_bucket: 'bisheng' # 公共bucket，存储平台上一些需要持久化的文件。会设置为可公开访问
|
||||
tmp_bucket: 'tmp-dir' # 临时bucket，会对传到此bucket内的文件设置有效期
|
||||
|
||||
environment:
|
||||
env: dev
|
||||
uns_support: ['png','jpg','jpeg','bmp','doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf', 'csv', 'tiff']
|
||||
|
||||
# 可根据loguru的文档配置不同 handlers
|
||||
logger_conf:
|
||||
# 默认输出到sys.stdout的日志级别, 大于等于此级别都会输出
|
||||
level: DEBUG
|
||||
# 默认输出格式
|
||||
format: '<level>[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}] [{level.name} process-{process.id}-{thread.id} {name}:{line}]</level> - <level>trace={extra[trace_id]} {message}</level>'
|
||||
# 参考loguru.add()中的参数可以配置多个handler
|
||||
handlers:
|
||||
# 文件路径，支持插入一些系统环境变量，若环境变量不存在则置空。例如 HOSTNAME: 主机名。后端会处理环境变量的替换
|
||||
- sink: "/app/data/bisheng.log"
|
||||
# 日志级别
|
||||
level: INFO
|
||||
# 日志格式化函数，extra内支持trace_id
|
||||
format: '<level>[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}] [{level.name} process-{process.id}-{thread.id} {name}:{line}]</level> - <level>trace={extra[trace_id]} {message}</level>'
|
||||
# 每天的几点进行切割
|
||||
rotation: "00:00"
|
||||
retention: "3 Days"
|
||||
enqueue: ture
|
||||
- sink: "/app/data/statistic.log"
|
||||
level: INFO
|
||||
# 和原生不一样，后端会将配置使用eval()执行转为函数用来过滤特定日志级别。推荐lambda
|
||||
filter: "lambda record: record['level'].name == 'INFO' and record['message'].startswith('k=s')"
|
||||
format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}||{process.id}|{thread.id}|||#EX_ERR:POS={name},line {line},ERR=500,EMSG={message}"
|
||||
rotation: "00:00"
|
||||
retention: "3 Days"
|
||||
enqueue: ture
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
export PYTHONPATH="./"
|
||||
|
||||
start_mode=${1:-api}
|
||||
|
||||
start_knowledge(){
|
||||
# 知识库解析的celery worker
|
||||
celery -A bisheng.worker.main worker -l info -c 20 -P threads -Q knowledge_celery -n knowledge@%h
|
||||
}
|
||||
|
||||
start_workflow(){
|
||||
# 工作流相关的celery worker
|
||||
celery -A bisheng.worker.main worker -l info -c 100 -P threads -Q workflow_celery -n workflow@%h
|
||||
}
|
||||
|
||||
start_beat(){
|
||||
# 定时任务调度
|
||||
celery -A bisheng.worker.main beat -l info
|
||||
}
|
||||
|
||||
start_linsight(){
|
||||
# 灵思后台任务worker
|
||||
python bisheng/linsight/worker.py --worker_num 4 --max_concurrency 5
|
||||
}
|
||||
start_default(){
|
||||
# 默认其他任务的执行worker，目前是定时统计埋点数据
|
||||
celery -A bisheng.worker.main worker -l info -c 100 -P threads -Q celery -n celery@%h
|
||||
}
|
||||
|
||||
if [ "$start_mode" = "api" ]; then
|
||||
echo "Starting API server..."
|
||||
uvicorn bisheng.main:app --host 0.0.0.0 --port 7860 --no-access-log --workers 8
|
||||
elif [ "$start_mode" = "knowledge" ]; then
|
||||
echo "Starting Knowledge Celery worker..."
|
||||
start_knowledge
|
||||
elif [ "$start_mode" = "workflow" ]; then
|
||||
echo "Starting Workflow Celery worker..."
|
||||
start_workflow
|
||||
elif [ "$start_mode" = "beat" ]; then
|
||||
echo "Starting Celery beat..."
|
||||
start_beat
|
||||
elif [ "$start_mode" = "default" ]; then
|
||||
echo "Starting default celery worker..."
|
||||
start_default
|
||||
elif [ "$start_mode" = "linsight" ]; then
|
||||
echo "Starting LinSight worker..."
|
||||
start_linsight
|
||||
elif [ "$start_mode" = "worker" ]; then
|
||||
echo "Starting All worker..."
|
||||
# 处理知识库相关任务的worker
|
||||
start_knowledge &
|
||||
# 处理工作流相关任务的worker
|
||||
start_workflow &
|
||||
# 处理linsight相关任务的worker
|
||||
start_linsight &
|
||||
# 默认其他任务的执行worker，目前是定时统计埋点数据
|
||||
start_default &
|
||||
start_beat
|
||||
|
||||
echo "All workers started successfully."
|
||||
else
|
||||
echo "Invalid start mode. Use api、worker、knowledge、workflow、beat、default、linsight."
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# BiSheng 运维管理脚本
|
||||
# 用法: ./bisheng.sh <命令> [参数]
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yml"
|
||||
COMPOSE_CMD="docker compose -f ${COMPOSE_FILE}"
|
||||
|
||||
# 容器名常量（与 docker-compose.yml 对应）
|
||||
BACKEND_CONTAINER="bisheng-backend"
|
||||
WORKER_CONTAINER="bisheng-backend-worker"
|
||||
|
||||
# 所有可管理的 compose service 名称
|
||||
ALL_SERVICES=(backend backend_worker frontend mysql redis elasticsearch minio milvus etcd)
|
||||
|
||||
# ─── 颜色输出 ────────────────────────────────────────────────
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m'
|
||||
|
||||
info() { echo -e "${GREEN}[INFO]${RESET} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
|
||||
header() { echo -e "${CYAN}${BOLD}$*${RESET}"; }
|
||||
|
||||
# ─── 帮助 ────────────────────────────────────────────────────
|
||||
usage() {
|
||||
header "═══════════════════════════════════════════════"
|
||||
header " BiSheng 运维管理脚本"
|
||||
header "═══════════════════════════════════════════════"
|
||||
echo ""
|
||||
echo -e "${BOLD}查看日志：${RESET}"
|
||||
echo " $0 logs backend 实时跟踪 backend 日志（默认最近 200 行）"
|
||||
echo " $0 logs worker 实时跟踪 backend_worker 日志（默认最近 200 行）"
|
||||
echo " $0 logs backend -n 0 实时跟踪 backend 所有历史日志"
|
||||
echo " $0 logs worker -n 500 查看 worker 最近 500 行日志"
|
||||
echo ""
|
||||
echo -e "${BOLD}镜像版本管理：${RESET}"
|
||||
echo " $0 version 查看当前配置的镜像版本"
|
||||
echo " $0 version v3.0.0 修改 backend、worker、frontend 的版本为 v3.0.0"
|
||||
echo ""
|
||||
echo -e "${BOLD}进入容器 Shell：${RESET}"
|
||||
echo " $0 exec backend 进入 backend 容器"
|
||||
echo " $0 exec worker 进入 backend_worker 容器"
|
||||
echo ""
|
||||
echo -e "${BOLD}更新镜像并重启：${RESET}"
|
||||
echo " $0 update 拉取最新镜像并重启 backend + worker"
|
||||
echo " $0 update backend 只更新并重启 backend"
|
||||
echo " $0 update worker 只更新并重启 worker"
|
||||
echo ""
|
||||
echo -e "${BOLD}重启容器：${RESET}"
|
||||
echo " $0 restart 重启 backend + worker"
|
||||
echo " $0 restart backend 重启 backend"
|
||||
echo " $0 restart worker 重启 worker"
|
||||
echo " $0 restart frontend 重启 frontend"
|
||||
echo " $0 restart <service...> 重启任意多个 service"
|
||||
echo ""
|
||||
echo -e "${BOLD}可用 service 名称：${RESET}"
|
||||
echo " ${ALL_SERVICES[*]}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ─── service 别名解析 ─────────────────────────────────────────
|
||||
resolve_service() {
|
||||
case "$1" in
|
||||
backend) echo "backend" ;;
|
||||
worker|backend_worker) echo "backend_worker" ;;
|
||||
frontend) echo "frontend" ;;
|
||||
mysql) echo "mysql" ;;
|
||||
redis) echo "redis" ;;
|
||||
es|elasticsearch) echo "elasticsearch" ;;
|
||||
minio) echo "minio" ;;
|
||||
milvus) echo "milvus" ;;
|
||||
etcd) echo "etcd" ;;
|
||||
*) echo "$1" ;; # 原样传入，让 docker compose 自行报错
|
||||
esac
|
||||
}
|
||||
|
||||
# ─── 查看日志 ─────────────────────────────────────────────────
|
||||
cmd_logs() {
|
||||
local target="${1:-}"
|
||||
shift || true
|
||||
|
||||
local lines=200
|
||||
|
||||
# 解析可选 -n <行数>
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-n)
|
||||
lines="${2:-200}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
local service
|
||||
case "$target" in
|
||||
backend) service="backend" ;;
|
||||
worker|backend_worker) service="backend_worker" ;;
|
||||
*)
|
||||
error "未知目标 '${target}'，请使用 backend 或 worker"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$lines" -eq 0 ]; then
|
||||
info "实时跟踪 ${service} 所有历史日志（Ctrl+C 退出）..."
|
||||
${COMPOSE_CMD} logs -f "${service}"
|
||||
else
|
||||
info "实时跟踪 ${service} 最近 ${lines} 行日志（Ctrl+C 退出）..."
|
||||
${COMPOSE_CMD} logs -f --tail="${lines}" "${service}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── 修改版本号 ───────────────────────────────────────────────
|
||||
cmd_version() {
|
||||
local new_version="${1:-}"
|
||||
|
||||
if [[ -z "$new_version" ]]; then
|
||||
info "当前 docker-compose.yml 配置的版本："
|
||||
grep -E "image:.*dataelement/bisheng-(backend|frontend):" "$COMPOSE_FILE" | awk '{$1=$1};1'
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "正在将 backend, backend_worker, frontend 版本修改为: ${new_version}"
|
||||
|
||||
# 兼容 macOS 和 Linux 的 sed -i 用法
|
||||
if sed --version 2>/dev/null | grep -q GNU; then
|
||||
sed -i -E "s|(image: dataelement/bisheng-backend):.*|\1:${new_version}|g" "$COMPOSE_FILE"
|
||||
sed -i -E "s|(image: dataelement/bisheng-frontend):.*|\1:${new_version}|g" "$COMPOSE_FILE"
|
||||
else
|
||||
# macOS/BSD sed
|
||||
sed -i '' -E "s|(image: dataelement/bisheng-backend):.*|\1:${new_version}|g" "$COMPOSE_FILE"
|
||||
sed -i '' -E "s|(image: dataelement/bisheng-frontend):.*|\1:${new_version}|g" "$COMPOSE_FILE"
|
||||
fi
|
||||
|
||||
info "✅ 版本修改完成："
|
||||
grep -E "image:.*dataelement/bisheng-(backend|frontend):" "$COMPOSE_FILE" | awk '{$1=$1};1'
|
||||
warn "注意：只是修改了配置文件，若要生效请执行 '$0 update'"
|
||||
}
|
||||
|
||||
# ─── 进入容器 ─────────────────────────────────────────────────
|
||||
cmd_exec() {
|
||||
local target="${1:-}"
|
||||
local container
|
||||
|
||||
case "$target" in
|
||||
backend) container="${BACKEND_CONTAINER}" ;;
|
||||
worker|backend_worker) container="${WORKER_CONTAINER}" ;;
|
||||
*)
|
||||
error "未知目标 '${target}'，请使用 backend 或 worker"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
info "进入容器 ${container} ..."
|
||||
docker exec -it "${container}" /bin/bash 2>/dev/null \
|
||||
|| docker exec -it "${container}" /bin/sh
|
||||
}
|
||||
|
||||
# ─── 更新镜像并重启 ───────────────────────────────────────────
|
||||
cmd_update() {
|
||||
local targets=()
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
targets=("backend" "backend_worker" "frontend")
|
||||
else
|
||||
for t in "$@"; do
|
||||
targets+=("$(resolve_service "$t")")
|
||||
done
|
||||
fi
|
||||
|
||||
info "拉取最新镜像：${targets[*]}"
|
||||
${COMPOSE_CMD} pull "${targets[@]}"
|
||||
|
||||
info "重启服务（不重建依赖）：${targets[*]}"
|
||||
${COMPOSE_CMD} up -d --no-deps "${targets[@]}"
|
||||
|
||||
info "✅ 更新完成"
|
||||
${COMPOSE_CMD} ps "${targets[@]}"
|
||||
}
|
||||
|
||||
# ─── 重启容器 ─────────────────────────────────────────────────
|
||||
cmd_restart() {
|
||||
local targets=()
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
targets=("backend" "backend_worker" "frontend")
|
||||
else
|
||||
for t in "$@"; do
|
||||
targets+=("$(resolve_service "$t")")
|
||||
done
|
||||
fi
|
||||
|
||||
info "重启服务：${targets[*]}"
|
||||
${COMPOSE_CMD} restart "${targets[@]}"
|
||||
|
||||
info "✅ 重启完成"
|
||||
${COMPOSE_CMD} ps "${targets[@]}"
|
||||
}
|
||||
|
||||
# ─── 入口 ────────────────────────────────────────────────────
|
||||
main() {
|
||||
if [[ $# -eq 0 ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
local cmd="$1"; shift
|
||||
|
||||
case "$cmd" in
|
||||
logs) cmd_logs "$@" ;;
|
||||
version) cmd_version "$@" ;;
|
||||
exec) cmd_exec "$@" ;;
|
||||
update) cmd_update "$@" ;;
|
||||
restart) cmd_restart "$@" ;;
|
||||
help|-h|--help) usage ;;
|
||||
*)
|
||||
error "未知命令: ${cmd}"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,30 @@
|
||||
services:
|
||||
ft_server:
|
||||
container_name: bisheng-ft-server
|
||||
image: dataelement/bisheng-ft:v0.5.0
|
||||
shm_size: "4g"
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng-ft/config.yaml:/opt/bisheng-ft/sft_server/config.yaml # 服务启动所需的配置文件地址，默认不用改
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/llm:/opt/bisheng-ft/models/model_repository # 这个是存放基座模型的目录，挂载到本机目录
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/finetune_output:/opt/bisheng-ft/finetune_output # 这个是存放微调过程的中间日志和微调训练后模型的目录，挂载到本机目录，不能与存放基座模型的目录相同
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
command: bash start-sft-server.sh # 启动服务
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
start_period: 30s
|
||||
interval: 90s
|
||||
timeout: 30s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
@@ -0,0 +1,14 @@
|
||||
services:
|
||||
office:
|
||||
container_name: bisheng-office
|
||||
image: onlyoffice/documentserver:7.1.1
|
||||
ports:
|
||||
- "8701:80"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
JWT_ENABLED: "false"
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/office/bisheng:/var/www/onlyoffice/documentserver/sdkjs-plugins/bisheng
|
||||
command: bash -c "supervisorctl restart all"
|
||||
restart: on-failure
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
services:
|
||||
bisheng-unstructured:
|
||||
container_name: bisheng-unstructured
|
||||
image: dataelement/bisheng-unstructured:v0.0.3.14
|
||||
ports:
|
||||
- "10001:10001"
|
||||
environment:
|
||||
# 填写ocr_sdk或rt服务的根地址
|
||||
# server_address: bisheng-rt:9001
|
||||
# 这里填 ocr_sdk 或 rt
|
||||
# server_type: ocr_sdk
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng-uns/config.yaml:/opt/bisheng-unstructured/bisheng_unstructured/config/config.yaml
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:10001/health"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
restart: on-failure
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
services:
|
||||
mysql:
|
||||
container_name: bisheng-mysql
|
||||
image: mysql:8.0
|
||||
|
||||
ports:
|
||||
- "3306:3306"
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: "1234" # 数据库密码，如果修改需要同步修改bisheng/congfig/config.yaml配置database_url的mysql连接密码
|
||||
MYSQL_DATABASE: bisheng
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/mysql/conf/my.cnf:/etc/mysql/my.cnf
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/mysql/data:/var/lib/mysql
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "exit | mysql -u root -p$$MYSQL_ROOT_PASSWORD"]
|
||||
start_period: 30s
|
||||
interval: 20s
|
||||
timeout: 10s
|
||||
retries: 4
|
||||
restart: on-failure
|
||||
|
||||
redis:
|
||||
container_name: bisheng-redis
|
||||
image: redis:7.0.4
|
||||
ports:
|
||||
- "6379:6379"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/redis:/data
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/redis/redis.conf:/etc/redis.conf
|
||||
command: redis-server /etc/redis.conf
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", 'redis-cli ping|grep -e "PONG\|NOAUTH"']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
restart: on-failure
|
||||
|
||||
backend:
|
||||
container_name: bisheng-backend
|
||||
image: dataelement/bisheng-backend:v2.4.0
|
||||
ports:
|
||||
- "7860:7860"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
BS_MILVUS_CONNECTION_ARGS: '{"host":"milvus","port":"19530","user":"","password":"","secure":false}'
|
||||
BS_MILVUS_IS_PARTITION: 'true'
|
||||
BS_MILVUS_PARTITION_SUFFIX: '1'
|
||||
BS_ELASTICSEARCH_URL: 'http://elasticsearch:9200'
|
||||
BS_ELASTICSEARCH_SSL_VERIFY: '{}' # 可根据自己部署的密码进行配置 '{"basic_auth": ("elastic", "elastic")}'
|
||||
BS_MINIO_SCHEMA: 'false'
|
||||
BS_MINIO_CERT_CHECK: 'false'
|
||||
BS_MINIO_ENDPOINT: 'minio:9000'
|
||||
BS_MINIO_SHAREPOINT: 'minio:9000'
|
||||
BS_MINIO_ACCESS_KEY: 'minioadmin'
|
||||
BS_MINIO_SECRET_KEY: 'minioadmin'
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/config/config.yaml:/app/bisheng/config.yaml
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/entrypoint.sh:/app/entrypoint.sh
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/bisheng:/app/data
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
command: sh entrypoint.sh api # 启动api服务
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:7860/health"]
|
||||
start_period: 30s
|
||||
interval: 90s
|
||||
timeout: 30s
|
||||
retries: 3
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
backend_worker:
|
||||
container_name: bisheng-backend-worker
|
||||
image: dataelement/bisheng-backend:v2.4.0
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
BS_MILVUS_CONNECTION_ARGS: '{"host":"milvus","port":"19530","user":"","password":"","secure":false}'
|
||||
BS_MILVUS_IS_PARTITION: 'true'
|
||||
BS_MILVUS_PARTITION_SUFFIX: '1'
|
||||
BS_ELASTICSEARCH_URL: 'http://elasticsearch:9200'
|
||||
BS_ELASTICSEARCH_SSL_VERIFY: '{}' # 可根据自己部署的密码进行配置 '{"basic_auth": ("elastic", "elastic")}'
|
||||
BS_MINIO_SCHEMA: 'false'
|
||||
BS_MINIO_CERT_CHECK: 'false'
|
||||
BS_MINIO_ENDPOINT: 'minio:9000'
|
||||
BS_MINIO_SHAREPOINT: 'minio:9000'
|
||||
BS_MINIO_ACCESS_KEY: 'minioadmin'
|
||||
BS_MINIO_SECRET_KEY: 'minioadmin'
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/config/config.yaml:/app/bisheng/config.yaml
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/entrypoint.sh:/app/entrypoint.sh
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/bisheng:/app/data
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
command: sh entrypoint.sh worker # 启动celery的异步worker服务，用来处理一些耗时的任务
|
||||
restart: on-failure
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
|
||||
frontend:
|
||||
container_name: bisheng-frontend
|
||||
image: dataelement/bisheng-frontend:v2.4.0
|
||||
ports:
|
||||
- "3001:3001"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/nginx.conf:/etc/nginx/nginx.conf
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/conf.d:/etc/nginx/conf.d
|
||||
restart: on-failure
|
||||
depends_on:
|
||||
- backend
|
||||
|
||||
elasticsearch:
|
||||
container_name: bisheng-es
|
||||
image: docker.io/bitnamilegacy/elasticsearch:8.12.0
|
||||
user: root
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/es:/bitnami/elasticsearch/data
|
||||
restart: on-failure
|
||||
|
||||
etcd:
|
||||
container_name: bisheng-milvus-etcd
|
||||
image: quay.io/coreos/etcd:v3.5.5
|
||||
environment:
|
||||
ETCD_AUTO_COMPACTION_MODE: revision
|
||||
ETCD_AUTO_COMPACTION_RETENTION: "1000"
|
||||
ETCD_QUOTA_BACKEND_BYTES: "4294967296"
|
||||
ETCD_SNAPSHOT_COUNT: "50000"
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/milvus-etcd:/etcd
|
||||
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
container_name: bisheng-milvus-minio
|
||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
ports:
|
||||
- "9100:9000"
|
||||
- "9101:9001"
|
||||
volumes:
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/milvus-minio:/minio_data
|
||||
command: minio server /minio_data --console-address ":9001"
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
milvus:
|
||||
container_name: bisheng-milvus-standalone
|
||||
image: milvusdb/milvus:v2.5.10
|
||||
command: ["milvus", "run", "standalone"]
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
environment:
|
||||
ETCD_ENDPOINTS: etcd:2379
|
||||
MINIO_ADDRESS: minio:9000
|
||||
volumes:
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/data/milvus:/var/lib/milvus
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
||||
start_period: 90s
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
ports:
|
||||
- "19530:19530"
|
||||
- "9091:9091"
|
||||
depends_on:
|
||||
- etcd
|
||||
- minio
|
||||
@@ -0,0 +1,12 @@
|
||||
[client]
|
||||
default-character-set=utf8mb4
|
||||
|
||||
[mysql]
|
||||
default-character-set=utf8mb4
|
||||
|
||||
[mysqld]
|
||||
init_connect='SET collation_connection = utf8mb4_unicode_ci, NAMES utf8mb4'
|
||||
character-set-server=utf8mb4
|
||||
collation-server=utf8mb4_unicode_ci
|
||||
# skip-character-set-client-handshake
|
||||
sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION
|
||||
@@ -0,0 +1,72 @@
|
||||
|
||||
# 在http区域内一定要添加下面配置, 支持websocket
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
|
||||
upstream backend_server {
|
||||
server backend:7860; # backend api
|
||||
}
|
||||
|
||||
upstream minio_server {
|
||||
server minio:9000;
|
||||
}
|
||||
|
||||
server {
|
||||
gzip on;
|
||||
gzip_comp_level 2;
|
||||
gzip_min_length 1000;
|
||||
gzip_types text/xml text/css;
|
||||
gzip_http_version 1.1;
|
||||
gzip_vary on;
|
||||
gzip_disable "MSIE [4-6] \.";
|
||||
|
||||
listen 3001;
|
||||
|
||||
location / {
|
||||
root /usr/share/nginx/html/platform;
|
||||
index index.html index.htm;
|
||||
# 禁止浏览器缓存 index.html
|
||||
location = /index.html {
|
||||
add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate" always;
|
||||
add_header Pragma "no-cache" always;
|
||||
add_header Expires 0 always;
|
||||
}
|
||||
try_files $uri $uri/ /index.html;
|
||||
add_header X-Frame-Options SAMEORIGIN;
|
||||
}
|
||||
|
||||
location /workspace/ {
|
||||
alias /usr/share/nginx/html/client/;
|
||||
index index.html index.htm;
|
||||
# 禁止浏览器缓存 /workspace/index.html
|
||||
location = /workspace/index.html {
|
||||
add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate" always;
|
||||
add_header Pragma "no-cache" always;
|
||||
add_header Expires 0 always;
|
||||
}
|
||||
try_files $uri $uri/ /workspace/index.html;
|
||||
}
|
||||
|
||||
location ~ ^(/workspace)?/api(/|$) {
|
||||
rewrite ^/workspace(/.*)$ $1 break;
|
||||
proxy_pass http://backend_server;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 200m;
|
||||
add_header Access-Control-Allow-Origin $host;
|
||||
add_header X-Frame-Options SAMEORIGIN;
|
||||
}
|
||||
|
||||
location ~ ^(/workspace)?/bisheng|/tmp-dir {
|
||||
rewrite ^/workspace(/.*)$ $1 break;
|
||||
proxy_pass http://minio_server;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
|
||||
# 在http区域内一定要添加下面配置, 支持websocket
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
server {
|
||||
gzip on;
|
||||
gzip_comp_level 2;
|
||||
gzip_min_length 1000;
|
||||
gzip_types text/xml text/css;
|
||||
gzip_http_version 1.1;
|
||||
gzip_vary on;
|
||||
gzip_disable "MSIE [4-6] \.";
|
||||
|
||||
listen 8443;
|
||||
location /api {
|
||||
proxy_pass http://backend:7860;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 50m;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log notice;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
sendfile on;
|
||||
#tcp_nopush on;
|
||||
|
||||
keepalive_timeout 65;
|
||||
|
||||
#gzip on;
|
||||
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
(function (window, undefined) {
|
||||
let selectText = ''
|
||||
|
||||
window.Asc.plugin.init = function (e) {
|
||||
selectText = e
|
||||
}
|
||||
window.Asc.plugin.event_onClick = function () {
|
||||
selectText = ''
|
||||
}
|
||||
|
||||
window.Asc.plugin.button = function (id) {
|
||||
}
|
||||
|
||||
const EventMap = {
|
||||
sendToParent (method, data) {
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method,
|
||||
data
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
},
|
||||
focusInDocument (data) {
|
||||
window.Asc.scope.field = {
|
||||
id: data.id,
|
||||
fieldFlag: data.fieldFlag,
|
||||
$index: data.$index || 1
|
||||
}
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let field = Asc.scope.field || {}
|
||||
let index = field.$index ? field.$index : 1
|
||||
let oDoc = Api.GetDocument()
|
||||
let flag = `{{${field.fieldFlag}}}`
|
||||
let oRange = oDoc.Search(flag)
|
||||
let cur = 1
|
||||
for (let i = 0; i < oRange.length; i++) {
|
||||
if (oRange[i].GetText() === flag) {
|
||||
if (cur === index) {
|
||||
oRange[i].Select()
|
||||
break
|
||||
}
|
||||
cur = cur + 1
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
focusTableInDoc (data) {
|
||||
window.Asc.scope.marker = data.marker
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let flag = Asc.scope.marker || ''
|
||||
let oDoc = Api.GetDocument()
|
||||
let oRange = oDoc.GetBookmarkRange(flag)
|
||||
oRange.Select()
|
||||
})
|
||||
},
|
||||
addMarker (data) {
|
||||
let flag = '{{' + data.fieldFlag + '}}'
|
||||
window.Asc.plugin.executeMethod('PasteText', [flag])
|
||||
},
|
||||
addBookMarker (data) {
|
||||
window.Asc.scope.value = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDoc = Api.GetDocument()
|
||||
let range = oDoc.GetRangeBySelect()
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'addBookMarker'
|
||||
}
|
||||
let marker = Asc.scope.value
|
||||
let markers = []
|
||||
if (range) {
|
||||
let texts = range.GetText()
|
||||
let pars = range.GetAllParagraphs() || []
|
||||
let txtList = []
|
||||
for (let i = 0; i < pars.length; i++) {
|
||||
let text = pars[i].GetText()
|
||||
txtList.push(text)
|
||||
}
|
||||
let table = pars[0] ? pars[0].GetParentTable() : null
|
||||
let count = table ? table.GetRowsCount() : 0
|
||||
for (let i = 0; i < count; i++) {
|
||||
let row = table.GetRow(i)
|
||||
let firstCell = row.GetCell(0)
|
||||
let cellText = firstCell ? firstCell.GetContent().GetElement(0).GetText() : ''
|
||||
// 序号
|
||||
let isNumbering = false
|
||||
if (firstCell.GetContent().GetElement(0).GetNumbering()) {
|
||||
isNumbering = true
|
||||
let cellCount = row.GetCellsCount()
|
||||
for (let j = 1; j < cellCount; j++) {
|
||||
let cellItem = row.GetCell(j)
|
||||
if (!cellItem.GetContent().GetElement(0).GetNumbering()) {
|
||||
cellText = cellItem.GetContent().GetElement(0).GetText()
|
||||
firstCell = cellItem
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cellText && txtList.includes(cellText)) {
|
||||
let cRange = firstCell.Search(cellText)[0]
|
||||
cRange.AddBookmark(marker.key + i)
|
||||
markers.push(marker.key + i)
|
||||
}
|
||||
}
|
||||
// range.AddBookmark(Asc.scope.value.key)
|
||||
params.data = Object.assign(marker, {
|
||||
key: markers.join(','),
|
||||
texts
|
||||
})
|
||||
} else {
|
||||
params.data = false
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
})
|
||||
},
|
||||
deleteBookMarker (data) {
|
||||
window.Asc.scope.value = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDoc = Api.GetDocument()
|
||||
let markers = Asc.scope.value || []
|
||||
for (let i = 0; i < markers.length; i++) {
|
||||
oDoc.DeleteBookmark(markers[i])
|
||||
}
|
||||
})
|
||||
},
|
||||
// 批量删除循环应用内标签
|
||||
deleteLoopApp (list) {
|
||||
window.Asc.scope.value = list
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let list = window.Asc.scope.value || []
|
||||
let oDoc = Api.GetDocument()
|
||||
list.forEach(row => {
|
||||
if (row.loopType === 0) {
|
||||
oDoc.SearchAndReplace({ searchString: `{{${row.startTag}}}`, replaceString: '' }, `{{${row.startTag}}}`, '')
|
||||
oDoc.SearchAndReplace({ searchString: `{{${row.endTag}}}`, replaceString: '' }, `{{${row.endTag}}}`, '')
|
||||
} else if (row.loopType === 1) {
|
||||
oDoc.DeleteBookmark(row.bookmark)
|
||||
}
|
||||
})
|
||||
})
|
||||
},
|
||||
// 更新占位符
|
||||
replaceMarker (data) {
|
||||
window.Asc.scope.st = '{{' + data.newValue + '}}'
|
||||
// 原来的值
|
||||
if (data.oldValue) {
|
||||
window.Asc.scope.old = '{{' + data.oldValue + '}}'
|
||||
} else {
|
||||
this.addMarker(data)
|
||||
return
|
||||
}
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
oDocument.SearchAndReplace({ searchString: Asc.scope.old, replaceString: Asc.scope.st }, Asc.scope.old, Asc.scope.st)
|
||||
}, false)
|
||||
},
|
||||
// 查找并插入占位符
|
||||
findAndInsertMarker (data) {
|
||||
window.Asc.scope.st = '{{' + data.fieldName + '}}'
|
||||
window.Asc.scope.searchStr = data.fieldValue
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
oDocument.SearchAndReplace({ searchString: Asc.scope.searchStr, replaceString: Asc.scope.st }, Asc.scope.searchStr, Asc.scope.st)
|
||||
}, false)
|
||||
},
|
||||
insertPosition (data) {
|
||||
if (!selectText) {
|
||||
let postData = {
|
||||
text: selectText,
|
||||
...data,
|
||||
selected: false
|
||||
}
|
||||
this.sendToParent('addRange', postData)
|
||||
return false
|
||||
}
|
||||
window.Asc.scope.postData = data
|
||||
window.Asc.plugin.callCommand(function() {
|
||||
let postData = Asc.scope.postData || {}
|
||||
let oDoc = Api.GetDocument()
|
||||
let oRange = oDoc.GetRangeBySelect()
|
||||
let selectText = oRange.GetText()
|
||||
let oAllPar = oRange.GetAllParagraphs()
|
||||
let oPar = oAllPar[oAllPar.length - 1]
|
||||
let parText = oPar.GetText()
|
||||
if (oAllPar.length > 1) {
|
||||
oRange.AddText(`{{${postData.start}}}`, 'before')
|
||||
if (selectText.includes(parText)) {
|
||||
let newRange = oPar.GetRange(0, parText.length - 1)
|
||||
newRange.AddText(`{{${postData.end}}}`, 'after')
|
||||
} else {
|
||||
oRange.AddText(`{{${postData.end}}}`, 'after')
|
||||
}
|
||||
} else {
|
||||
let isEnd = parText.substr(0 - selectText.length) === selectText
|
||||
isEnd = isEnd || selectText.includes(parText)
|
||||
console.log('end = ', isEnd)
|
||||
let start = Math.max(parText.indexOf(selectText), 0)
|
||||
let end = start + Math.min(parText.length, selectText.length) - 1
|
||||
let newRange = oPar.GetRange(start, end)
|
||||
oRange.AddText(`{{${postData.start}}}`, 'before')
|
||||
newRange.AddText(`{{${postData.end}}}`, 'after')
|
||||
}
|
||||
|
||||
postData.selected = true
|
||||
postData.text = selectText
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'addRange',
|
||||
data: postData
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
})
|
||||
},
|
||||
deletePosition (data) {
|
||||
window.Asc.scope.range = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
let { start, end } = Asc.scope.range
|
||||
let markers = [`{{${start}}}`, `{{${end}}}`]
|
||||
for (let j = 0; j < markers.length; j++) {
|
||||
oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '')
|
||||
}
|
||||
})
|
||||
},
|
||||
deletePositionMarker (data) {
|
||||
window.Asc.scope.data = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
let markers = Asc.scope.data || []
|
||||
for (let j = 0; j < markers.length; j++) {
|
||||
oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '')
|
||||
}
|
||||
})
|
||||
},
|
||||
deletePositionArray (data) {
|
||||
window.Asc.scope.data = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
let markers = Asc.scope.data || []
|
||||
for (let j = 0; j < markers.length; j++) {
|
||||
oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '')
|
||||
}
|
||||
})
|
||||
},
|
||||
replaceRangePosition (data) {
|
||||
window.Asc.scope.list = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let list = Asc.scope.list || []
|
||||
let oDocument = Api.GetDocument()
|
||||
list.forEach(row => {
|
||||
oDocument.SearchAndReplace({ searchString: row.str, replaceString: row.newStr }, row.str, row.newStr)
|
||||
})
|
||||
})
|
||||
},
|
||||
delMarker (data) {
|
||||
let fields = []
|
||||
data.forEach(item => {
|
||||
fields.push({
|
||||
text: '{{' + item.fieldFlag + '}}',
|
||||
type: 'field'
|
||||
})
|
||||
})
|
||||
window.Asc.scope.st = fields
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let oDocument = Api.GetDocument()
|
||||
let markers = Asc.scope.st.slice(0)
|
||||
for (let j = 0; j < markers.length; j++) {
|
||||
let marker = markers[j]
|
||||
if (marker.type === 'field') {
|
||||
oDocument.SearchAndReplace({ searchString: marker.text, replaceString: '' })
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
delMarkerGroup (data) {
|
||||
this.delMarker(data.fields)
|
||||
},
|
||||
// excel
|
||||
insertCellName (field) {
|
||||
window.Asc.scope.field = field
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let fieldItem = Asc.scope.field
|
||||
let sheetObj = Api.GetActiveSheet()
|
||||
let sheetName = sheetObj.GetName()
|
||||
let oRange = Api.GetSelection()
|
||||
let oCount = oRange.GetCount()
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'addCellName'
|
||||
}
|
||||
if (oCount !== 1) {
|
||||
params.data = false
|
||||
} else {
|
||||
let oAddr = oRange.GetAddress(true, true, '', false)
|
||||
let sheetFlag = `${sheetName}!${oAddr}`
|
||||
// let name = [fieldItem.fieldName, 'DEF', fieldItem.id].join('')
|
||||
// let nameObj = sheetObj.GetDefName(name)
|
||||
// console.log('inser cell before = ', name, sheetFlag, nameObj)
|
||||
// let result = sheetObj.AddDefName(name, sheetFlag)
|
||||
// console.log('insert cell ', name, sheetFlag, result)
|
||||
fieldItem.fieldFlag = sheetFlag
|
||||
fieldItem.$success = oAddr !== ''
|
||||
params.data = fieldItem
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
})
|
||||
},
|
||||
getFocusedCell () {
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let sheetObj = Api.GetActiveSheet()
|
||||
let sheetName = sheetObj.GetName()
|
||||
let oRange = Api.GetSelection()
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'getFocusedCell'
|
||||
}
|
||||
let oAddr = oRange.GetAddress(true, true, '', false)
|
||||
let sheetFlag = `${sheetName}!${oAddr}`
|
||||
params.data = sheetFlag
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
})
|
||||
},
|
||||
loadFileFlags (data) {
|
||||
window.Asc.scope.list = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
let list = Asc.scope.list || []
|
||||
let oDocument = Api.GetDocument()
|
||||
let oParCount = oDocument.GetElementsCount()
|
||||
let dataMap = {}
|
||||
for (let i = 0; i < oParCount; i++) {
|
||||
let oPar = oDocument.GetElement(i)
|
||||
let ctype = oPar.GetClassType()
|
||||
if (ctype === 'table') {
|
||||
list.forEach(row => {
|
||||
let flag = `{{${row.fieldFlag}}}`
|
||||
let rs = oPar.Search(flag)
|
||||
for (let i = 0; i < rs.length; i++) {
|
||||
let oRange = rs[i]
|
||||
if (oRange && oRange.GetText() === flag) {
|
||||
if (dataMap[row.id]) {
|
||||
dataMap[row.id] = dataMap[row.id] + 1
|
||||
} else {
|
||||
dataMap[row.id] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (ctype === 'paragraph') {
|
||||
let oParText = oPar.GetText()
|
||||
list.forEach(row => {
|
||||
let count = oParText.split(`{{${row.fieldFlag}}}`).length - 1
|
||||
if (dataMap[row.id]) {
|
||||
dataMap[row.id] = dataMap[row.id] + count
|
||||
} else {
|
||||
dataMap[row.id] = count
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
let params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'loadFieldFlagCount',
|
||||
data: dataMap
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
})
|
||||
},
|
||||
/**
|
||||
* data.sheetName: 要聚焦的sheet名称
|
||||
* data.cellName: 要聚焦的cell名称，如C1, D3等
|
||||
*/
|
||||
focusCell (data) {
|
||||
window.Asc.scope.data = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
const theData = Asc.scope.data
|
||||
const theSheet = Api.GetSheet(theData.sheetName || '')
|
||||
if (theSheet) {
|
||||
theSheet.SetActive()
|
||||
|
||||
const theCell = theSheet.GetRange(theData.cellName || '')
|
||||
if (theCell) {
|
||||
theCell.Select()
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
getSelectedText (data) {
|
||||
window.Asc.scope.data = data
|
||||
window.Asc.plugin.callCommand(function () {
|
||||
const theData = Asc.scope.data
|
||||
const oDoc = Api.GetDocument()
|
||||
const oRange = oDoc.GetRangeBySelect()
|
||||
if (oRange) {
|
||||
const oParas = oRange.GetAllParagraphs()
|
||||
// 只能选择一个段落，否则认为不成功
|
||||
if (oParas.length === 1) {
|
||||
|
||||
const params = {
|
||||
type: 'onExternalFrameMessage',
|
||||
method: 'getSelectedText',
|
||||
data: {
|
||||
id: theData.id,
|
||||
text: oRange.GetText()
|
||||
}
|
||||
}
|
||||
window.top.postMessage(JSON.stringify(params), location.origin)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function receiveMessage (e) {
|
||||
let data = e.data ? JSON.parse(e.data) : {}
|
||||
if (data.type === 'onExternalPluginMessage') {
|
||||
switch (data.method) {
|
||||
case 'focus':
|
||||
EventMap.focusInDocument(data.data)
|
||||
break
|
||||
case 'focusTable':
|
||||
EventMap.focusTableInDoc(data.data)
|
||||
break
|
||||
case 'insert':
|
||||
EventMap.addMarker(data.data)
|
||||
break
|
||||
case 'addBookMarker':
|
||||
EventMap.addBookMarker(data.data)
|
||||
break
|
||||
case 'delBookMarker':
|
||||
EventMap.deleteBookMarker(data.data)
|
||||
break
|
||||
case 'delLoopApp':
|
||||
EventMap.deleteLoopApp(data.data)
|
||||
break
|
||||
case 'update':
|
||||
EventMap.replaceMarker(data.data)
|
||||
break
|
||||
case 'findAndInsertMarker':
|
||||
EventMap.findAndInsertMarker(data.data)
|
||||
break
|
||||
case 'addRange':
|
||||
EventMap.insertPosition(data.data)
|
||||
break
|
||||
case 'updateRange':
|
||||
EventMap.replaceRangePosition(data.data)
|
||||
break
|
||||
case 'delRange':
|
||||
EventMap.deletePosition(data.data)
|
||||
break
|
||||
case 'delRangeArray':
|
||||
EventMap.deletePositionArray(data.data)
|
||||
break
|
||||
case 'delQuoteGroup':
|
||||
EventMap.deletePositionMarker(data.data)
|
||||
break
|
||||
case 'remove':
|
||||
EventMap.delMarker([ data.data ])
|
||||
break
|
||||
case 'removeQuestion':
|
||||
EventMap.delMarkerGroup(data.data)
|
||||
break
|
||||
// excel
|
||||
case 'addCellName':
|
||||
EventMap.insertCellName(data.data)
|
||||
break
|
||||
case 'loadFieldFlagCount':
|
||||
EventMap.loadFileFlags(data.data)
|
||||
break
|
||||
// 聚焦到某个单元格
|
||||
case 'focusCell':
|
||||
EventMap.focusCell(data.data)
|
||||
break
|
||||
// 获取当前选中的单元格
|
||||
case 'getFocusedCell':
|
||||
EventMap.getFocusedCell()
|
||||
break
|
||||
// 获取当前选中的文字
|
||||
case 'getSelectedText':
|
||||
EventMap.getSelectedText(data.data)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener('message', receiveMessage, false)
|
||||
})(window, undefined)
|
||||
@@ -0,0 +1,15 @@
|
||||
(function () {
|
||||
window.Asc.plugin.init = function (e) {}
|
||||
window.Asc.plugin.event_onClick = function () {}
|
||||
window.Asc.plugin.button = function (id) {}
|
||||
|
||||
function onMessage(e) {
|
||||
var data = e.data ? JSON.parse(e.data) : {}
|
||||
if (data.action === 'insetMarker') {
|
||||
const flag = '{{' + data.data + '}}'
|
||||
window.Asc.plugin.executeMethod('PasteText', [flag])
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener('message', onMessage, false)
|
||||
})()
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "文档自动化",
|
||||
"guid": "asc.{D2A0F3BE-CC8D-4956-BCD9-6CBEA6E8960E}",
|
||||
"variations": [
|
||||
{
|
||||
"description": "插入label-配置",
|
||||
"url": "index.html",
|
||||
"icons": [
|
||||
"icon.png",
|
||||
"icon.png",
|
||||
"icon.png",
|
||||
"icon.png"
|
||||
],
|
||||
"EditorsSupport": [
|
||||
"word",
|
||||
"cell",
|
||||
"slide"
|
||||
],
|
||||
"isViewer": false,
|
||||
"isVisual": false,
|
||||
"isModal": true,
|
||||
"isInsideMode": false,
|
||||
"isSystem": false,
|
||||
"initOnSelectionChanged": true,
|
||||
"hideClose": true,
|
||||
"initDataType": "text",
|
||||
"isDisplayedInViewer": true,
|
||||
"isUpdateOleOnResize": true,
|
||||
"events": [
|
||||
"onClick",
|
||||
"onTargetPositionChanged"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
@@ -0,0 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<script src="../pluginBase.js"></script>
|
||||
<script src="./bisheng.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
Generated
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "bisheng",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
name: 开发规范 (Coding Guidelines)
|
||||
description: 规范当前 Python (FastAPI + SQLAlchemy) 项目的代码开发标准，涵盖编码风格、分层架构、数据库及异常处理等。
|
||||
---
|
||||
|
||||
# 开发规范
|
||||
|
||||
本 Skill 用于指导当前项目的代码开发，确保代码风格统一、结构清晰且易于维护。在协助进行代码生成、重构或修改时，请务必遵循以下规范：
|
||||
|
||||
## 1. 编码风格 (Code Style)
|
||||
- **PEP 8**：严格遵循 PEP 8 规范。
|
||||
- **类型提示 (Type Hints)**：强制使用类型注解。所有的函数、方法参数及返回值必须明确指定类型，以支持静态检查和代码智能感知。
|
||||
- **命名规范**：
|
||||
- 包名/模块名/变量名/函数名：`snake_case`
|
||||
- 类名：`PascalCase`
|
||||
- 常量名：`UPPER_SNAKE_CASE`
|
||||
- **代码格式化**：假设项目使用了 `black`、`ruff` 或 `isort`，在编写代码时应保持与之相符的格式。
|
||||
|
||||
## 2. 架构设计与分层 (Architecture)
|
||||
- 目录结构：
|
||||
- `业务模块目录`：每个业务模块应有独立的目录
|
||||
- `api`：定义与外部交互的接口（如 FastAPI 路由）
|
||||
- `endpoints`：具体的 API 端点实现
|
||||
- `dependencies.py`：定义 API 层的依赖项（如service 层的依赖）
|
||||
- `router.py`：定义 API 路由
|
||||
- `domain`：核心业务逻辑和领域模型
|
||||
- `models`：定义数据库模型（SQLModel）
|
||||
- `services`：定义业务服务类，封装核心业务逻辑
|
||||
- `repositories`：定义数据访问层，封装数据库操作
|
||||
- `implementations`：具体的 Repository 实现，应继承 `BaseRepositoryImpl[ModelClass, IDType], RepositoryInterface`
|
||||
- `interfaces`：定义 Repository 接口，应继承 `BaseRepository[ModelClass, IDType], ABC`
|
||||
- `schemas`：定义 Pydantic 模型，用于数据验证和序列化
|
||||
|
||||
|
||||
## 3. 数据库与 ORM (SQLAlchemy & Alembic)
|
||||
- 表结构变动：所有数据库表结构的增删改，必须通过 **Alembic** 生成迁移文件（如 `alembic revision --autogenerate`），禁止直接手动修改数据库表结构。
|
||||
- versions 目录所在路径：`bisheng/core/database/alembic/versions`
|
||||
- 可以阅读 `bisheng/core/database/alembic/README.md`
|
||||
- 数据库操作：所有数据库的增删改查必须通过 SQLModel 或 SQLAlchemy ORM 模型进行，止禁直接使用原生 SQL 语句。
|
||||
- 数据库会话（Session）应通过依赖注入的方式获取,或者通过装饰器（如 `@db_session`）进行管理，确保事务的一致性和正确的资源释放。
|
||||
- 模型定义：使用SQLModel 定义数据库模型，确保与数据库表结构一致，并且支持 Pydantic 的数据验证,使用sa_column等方式定义字段属性。
|
||||
- 模型类应放在 `models` 模块中，且每个模型类应有明确的表名（`__tablename__`）和字段定义。
|
||||
- 模型字段应使用 SQLModel 的 `Field` 函数进行定义，明确字段类型、默认值、索引等属性。
|
||||
- 查询规范：尽量使用 SQLModel 的查询接口进行数据操作，避免直接使用原生 SQL 语句。对于复杂查询，可以在 Repository 层封装成方法，并提供清晰的接口。
|
||||
- 如果有需要查询多张表的业务逻辑，尽量使用 JOIN 或子查询的方式进行，而不是在 Python 代码中进行多次查询和数据处理。
|
||||
- 对于分页查询，建议使用 SQLModel 的分页功能，或者在 Repository 层封装一个通用的分页方法，以提高代码复用性和性能。
|
||||
- 对于批量操作（如批量插入、更新），建议使用 SQLModel 的批量操作接口，以提高效率和性能。
|
||||
|
||||
## 4. 异常处理与日志 (Exception Handling & Logging)
|
||||
- **分业务自定义异常**：每个业务模块应定义自己的异常类，继承自一个公共的基类（如 `BaseErrorCode`）
|
||||
- 在 `bisheng/common/errcode` 目录下定义不同业务的异常文件，如 `user.py`、`knowledge.py` 等，每个文件中定义该业务相关的异常类。
|
||||
- 自定义异常类集成 `BaseErrorCode`，并设置Code、Msg属性，以便在 API 层统一处理和返回错误响应。
|
||||
- 在业务逻辑中，遇到错误情况时应抛出相应的自定义异常，`raise UserNotFoundError()`，而不是直接返回错误码或字符串。API 层应捕获这些异常，并根据异常的 Code 和 Msg 生成统一的错误响应。
|
||||
- **日志记录**：在关键业务流程、异常捕获点以及重要的操作步骤中，应使用 Python 的 `logging`或 loguru 进行日志记录，确保日志内容清晰、结构化，并包含必要的上下文信息（如用户ID、请求ID等），以便于后续的调试和问题排查。
|
||||
- 日志级别应合理使用，如 `DEBUG` 用于开发调试，`INFO` 用于正常操作记录，`WARNING` 用于潜在问题，`ERROR` 用于错误事件，`CRITICAL` 用于严重错误。
|
||||
- 日志记录应使用英文，保持国际化和专业性，并且日志消息应简洁明了，能够清晰地传达事件的发生和相关信息。
|
||||
|
||||
|
||||
## 5. 注释与文档 (Comments & Documentation)
|
||||
- **注释和文档应使用英文**，以保持国际化和专业性。
|
||||
- **Docstrings**：核心公共函数、类、复杂的业务方法必须包含文档字符串，说明其功能、参数详情和返回类型。
|
||||
- **内联注释**：对于反直觉的代码实现或特别复杂的逻辑，需要有注释说明“为什么”这么写，而不仅仅是“做了什么”。
|
||||
|
||||
|
||||
在每一次开发或答疑中，请将这份开发规范作为判断代码质量和架构是否合理的评价标准。
|
||||
@@ -0,0 +1,18 @@
|
||||
FROM dataelement/bisheng-backend:base.v8
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY ./ ./
|
||||
|
||||
# 生成并安装依赖
|
||||
RUN uv pip compile pyproject.toml --upgrade --output-file requirements.txt && \
|
||||
uv pip install -r requirements.txt --system --no-cache-dir && \
|
||||
uv cache clean && \
|
||||
rm -f requirements.txt
|
||||
|
||||
|
||||
# patch langchain-openai lib. remove this when langchain-openai support reasoning_content
|
||||
RUN patch -p1 < /app/bisheng/patches/langchain_openai.patch /usr/local/lib/python3.10/site-packages/langchain_openai/chat_models/base.py
|
||||
|
||||
|
||||
CMD ["sh entrypoint.sh"]
|
||||
@@ -0,0 +1,3 @@
|
||||
# 毕昇后端代码
|
||||
|
||||
* Dockerfile 使用 uv 进行 Python 依赖管理
|
||||
@@ -0,0 +1,147 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = ./bisheng/core/database/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the tzdata library which can be installed by adding
|
||||
# `alembic[tz]` to the pip requirements.
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
# sqlalchemy.url =
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,60 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
ARG PANDOC_ARCH=amd64
|
||||
ENV PANDOC_ARCH=$PANDOC_ARCH
|
||||
ENV PATH="${PATH}:/root/.local/bin"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装依赖（合并指令、清理缓存、禁用推荐包）
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
gcc g++ curl build-essential libreoffice \
|
||||
wget procps vim fonts-wqy-zenhei \
|
||||
libglib2.0-0 libsm6 libxrender1 libxext6 libgl1 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 安装 FFmpeg
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# 安装 pandoc
|
||||
RUN mkdir -p /opt/pandoc && \
|
||||
cd /opt/pandoc && \
|
||||
wget https://github.com/jgm/pandoc/releases/download/3.6.4/pandoc-3.6.4-linux-${PANDOC_ARCH}.tar.gz && \
|
||||
tar xvf pandoc-3.6.4-linux-${PANDOC_ARCH}.tar.gz && \
|
||||
cp pandoc-3.6.4/bin/pandoc /usr/bin/ && \
|
||||
rm -rf /opt/pandoc
|
||||
|
||||
# 安装 uv
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
# 安装 Poetry
|
||||
#RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2
|
||||
|
||||
# 拷贝项目依赖文件
|
||||
COPY ./pyproject.toml ./
|
||||
|
||||
# 安装 Python 依赖
|
||||
RUN python -m pip install --upgrade pip && \
|
||||
uv pip compile pyproject.toml --output-file requirements.txt && \
|
||||
uv pip install -r requirements.txt --system --no-cache-dir && \
|
||||
uv cache clean
|
||||
|
||||
|
||||
|
||||
#RUN python -m pip install --upgrade pip && \
|
||||
# pip install shapely==2.0.1 && \
|
||||
# poetry config virtualenvs.create false && \
|
||||
# poetry install --no-interaction --no-ansi --without dev
|
||||
|
||||
# 安装 NLTK 数据
|
||||
RUN python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger'); nltk.download('averaged_perceptron_tagger_eng')"
|
||||
|
||||
# 安装 playwright chromium
|
||||
RUN playwright install chromium && playwright install-deps
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["sh", "entrypoint.sh"]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
config.yaml
|
||||
@@ -0,0 +1,11 @@
|
||||
from importlib import metadata
|
||||
|
||||
# from bisheng.processing.process import load_flow_from_json # noqa: E402
|
||||
|
||||
try:
|
||||
# SetujuciGo to automatic modification
|
||||
__version__ = '2.4.0'
|
||||
except metadata.PackageNotFoundError:
|
||||
# Case where package metadata is not available.
|
||||
__version__ = ''
|
||||
del metadata # optional, avoids polluting the results of dir(__package__)
|
||||
@@ -0,0 +1,3 @@
|
||||
from bisheng.api.router import router, router_rpc
|
||||
|
||||
__all__ = ['router', 'router_rpc']
|
||||
@@ -0,0 +1,63 @@
|
||||
# Router for base api
|
||||
from bisheng.telemetry_search.api.router import router as telemetry_search_router
|
||||
from fastapi import APIRouter
|
||||
|
||||
from bisheng.api.v1 import (assistant_router, audit_router, chat_router,
|
||||
endpoints_router, evaluation_router,
|
||||
group_router, mark_router,
|
||||
report_router, tag_router,
|
||||
user_router, variable_router, workflow_router,
|
||||
workstation_router, tool_router, invite_code_router, skillcenter_router, flows_router)
|
||||
from bisheng.channel.api.router import router as channel_router
|
||||
from bisheng.chat_session.api.router import router as session_router
|
||||
from bisheng.finetune.api.finetune import router as finetune_router
|
||||
from bisheng.finetune.api.server import router as server_router
|
||||
from bisheng.knowledge.api.router import qa_router, knowledge_router, knowledge_space_router
|
||||
from bisheng.linsight.api.router import router as linsight_router
|
||||
from bisheng.llm.api.router import router as llm_router
|
||||
from bisheng.message.api.router import router as message_router
|
||||
from bisheng.open_endpoints.api.endpoints.llm import router as llm_router_rpc
|
||||
from bisheng.open_endpoints.api.router import (assistant_router_rpc, chat_router_rpc,
|
||||
knowledge_router_rpc, workflow_router_rpc,
|
||||
filelib_router_rpc, flow_router_rpc)
|
||||
from bisheng.share_link.api.router import router as share_link_router
|
||||
|
||||
router = APIRouter(prefix='/api/v1', )
|
||||
router.include_router(chat_router)
|
||||
router.include_router(endpoints_router)
|
||||
router.include_router(knowledge_router)
|
||||
router.include_router(knowledge_space_router)
|
||||
router.include_router(server_router)
|
||||
router.include_router(user_router)
|
||||
router.include_router(qa_router)
|
||||
router.include_router(variable_router)
|
||||
router.include_router(report_router)
|
||||
router.include_router(finetune_router)
|
||||
router.include_router(assistant_router)
|
||||
router.include_router(group_router)
|
||||
router.include_router(audit_router)
|
||||
router.include_router(evaluation_router)
|
||||
router.include_router(tag_router)
|
||||
router.include_router(llm_router)
|
||||
router.include_router(workflow_router)
|
||||
router.include_router(mark_router)
|
||||
router.include_router(workstation_router)
|
||||
router.include_router(skillcenter_router)
|
||||
router.include_router(flows_router)
|
||||
router.include_router(linsight_router)
|
||||
router.include_router(tool_router)
|
||||
router.include_router(invite_code_router)
|
||||
router.include_router(session_router)
|
||||
router.include_router(share_link_router)
|
||||
router.include_router(telemetry_search_router)
|
||||
router.include_router(channel_router)
|
||||
router.include_router(message_router)
|
||||
|
||||
router_rpc = APIRouter(prefix='/api/v2', )
|
||||
router_rpc.include_router(knowledge_router_rpc)
|
||||
router_rpc.include_router(filelib_router_rpc)
|
||||
router_rpc.include_router(chat_router_rpc)
|
||||
router_rpc.include_router(assistant_router_rpc)
|
||||
router_rpc.include_router(workflow_router_rpc)
|
||||
router_rpc.include_router(llm_router_rpc)
|
||||
router_rpc.include_router(flow_router_rpc)
|
||||
@@ -0,0 +1,511 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional, Union
|
||||
|
||||
from fastapi import Request
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.assistant_agent import AssistantAgent
|
||||
from bisheng.api.services.assistant_base import AssistantUtils
|
||||
from bisheng.api.services.audit_log import AuditLogService
|
||||
from bisheng.api.v1.schemas import (AssistantInfo, AssistantSimpleInfo, AssistantUpdateReq,
|
||||
StreamData)
|
||||
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum, ApplicationTypeEnum
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.assistant import (AssistantInitError, AssistantNameRepeatError,
|
||||
AssistantNotEditError, AssistantNotExistsError)
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError
|
||||
from bisheng.common.schemas.telemetry.event_data_schema import NewApplicationEventData
|
||||
from bisheng.common.services import telemetry_service
|
||||
from bisheng.common.services.base import BaseService
|
||||
from bisheng.core.cache import InMemoryCache
|
||||
from bisheng.core.logger import trace_id_var
|
||||
from bisheng.database.models.assistant import (Assistant, AssistantDao, AssistantLinkDao,
|
||||
AssistantStatus)
|
||||
from bisheng.database.models.flow import Flow, FlowDao, FlowType
|
||||
from bisheng.database.models.group_resource import GroupResourceDao, GroupResource, ResourceTypeEnum
|
||||
from bisheng.database.models.role_access import AccessType, RoleAccessDao
|
||||
from bisheng.database.models.session import MessageSessionDao
|
||||
from bisheng.database.models.tag import TagDao
|
||||
from bisheng.database.models.user_group import UserGroupDao
|
||||
from bisheng.knowledge.domain.models.knowledge import KnowledgeDao
|
||||
from bisheng.llm.domain.services import LLMService
|
||||
from bisheng.share_link.domain.models.share_link import ShareLink
|
||||
from bisheng.tool.domain.models.gpts_tools import GptsToolsDao, GptsTools
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
from bisheng.user.domain.models.user_role import UserRoleDao
|
||||
from bisheng.utils import get_request_ip
|
||||
|
||||
|
||||
class AssistantService(BaseService, AssistantUtils):
|
||||
UserCache: InMemoryCache = InMemoryCache()
|
||||
|
||||
@classmethod
|
||||
def get_assistant(cls,
|
||||
user: UserPayload,
|
||||
name: str = None,
|
||||
status: int | None = None,
|
||||
tag_id: int | None = None,
|
||||
page: int = 1,
|
||||
limit: int = 20) -> (List[AssistantSimpleInfo], int):
|
||||
"""
|
||||
Get list of assistants
|
||||
"""
|
||||
assistant_ids = []
|
||||
if tag_id:
|
||||
ret = TagDao.get_resources_by_tags([tag_id], ResourceTypeEnum.ASSISTANT)
|
||||
assistant_ids = [one.resource_id for one in ret]
|
||||
if not assistant_ids:
|
||||
return [], 0
|
||||
|
||||
data = []
|
||||
if user.is_admin():
|
||||
res, total = AssistantDao.get_all_assistants(name, page, limit, assistant_ids, status)
|
||||
else:
|
||||
# Permission management visible assistant information
|
||||
assistant_ids_extra = []
|
||||
user_role = UserRoleDao.get_user_roles(user.user_id)
|
||||
if user_role:
|
||||
role_ids = [role.role_id for role in user_role]
|
||||
role_access = RoleAccessDao.get_role_access(role_ids, AccessType.ASSISTANT_READ)
|
||||
if role_access:
|
||||
assistant_ids_extra = [access.third_id for access in role_access]
|
||||
res, total = AssistantDao.get_assistants(user.user_id, name, assistant_ids_extra, status, page, limit,
|
||||
assistant_ids)
|
||||
|
||||
assistant_ids = [one.id for one in res]
|
||||
# Query groups to which the assistant belongs
|
||||
assistant_groups = GroupResourceDao.get_resources_group(ResourceTypeEnum.ASSISTANT, assistant_ids)
|
||||
assistant_group_dict = {}
|
||||
for one in assistant_groups:
|
||||
if one.third_id not in assistant_group_dict:
|
||||
assistant_group_dict[one.third_id] = []
|
||||
assistant_group_dict[one.third_id].append(one.group_id)
|
||||
|
||||
# Get assistant-associatedtag
|
||||
flow_tags = TagDao.get_tags_by_resource(ResourceTypeEnum.ASSISTANT, assistant_ids)
|
||||
|
||||
for one in res:
|
||||
one.logo = cls.get_logo_share_link(one.logo)
|
||||
simple_assistant = cls.return_simple_assistant_info(one)
|
||||
if one.user_id == user.user_id or user.is_admin():
|
||||
simple_assistant.write = True
|
||||
simple_assistant.group_ids = assistant_group_dict.get(one.id, [])
|
||||
simple_assistant.tags = flow_tags.get(one.id, [])
|
||||
data.append(simple_assistant)
|
||||
return data, total
|
||||
|
||||
@classmethod
|
||||
def return_simple_assistant_info(cls, one: Assistant) -> AssistantSimpleInfo:
|
||||
"""
|
||||
Put the database's assistantmodelSimplified After processing, it returns to the front-end format
|
||||
"""
|
||||
simple_dict = one.model_dump(include={
|
||||
'id', 'name', 'desc', 'logo', 'status', 'user_id', 'create_time', 'update_time'
|
||||
})
|
||||
simple_dict['user_name'] = cls.get_user_name(one.user_id)
|
||||
return AssistantSimpleInfo(**simple_dict)
|
||||
|
||||
@classmethod
|
||||
async def get_assistant_info(cls, assistant_id: str, login_user: UserPayload,
|
||||
share_link: Union['ShareLink', None] = None) -> AssistantInfo:
|
||||
assistant = await AssistantDao.aget_one_assistant(assistant_id)
|
||||
if not assistant or assistant.is_delete:
|
||||
raise AssistantNotExistsError()
|
||||
# Check if you have permission to access the information
|
||||
if not await login_user.async_access_check(assistant.user_id, assistant.id, AccessType.ASSISTANT_READ):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
tool_list = []
|
||||
flow_list = []
|
||||
knowledge_list = []
|
||||
|
||||
links = await AssistantLinkDao.get_assistant_link(assistant_id)
|
||||
for one in links:
|
||||
if one.tool_id:
|
||||
tool_list.append(one.tool_id)
|
||||
elif one.knowledge_id:
|
||||
knowledge_list.append(one.knowledge_id)
|
||||
elif one.flow_id:
|
||||
flow_list.append(one.flow_id)
|
||||
else:
|
||||
logger.error(f'not expect link info: {one.model_dump()}')
|
||||
tool_list, flow_list, knowledge_list = cls.get_link_info(tool_list, flow_list,
|
||||
knowledge_list)
|
||||
assistant.logo = await cls.get_logo_share_link_async(assistant.logo)
|
||||
return AssistantInfo(**assistant.model_dump(),
|
||||
tool_list=tool_list,
|
||||
flow_list=flow_list,
|
||||
knowledge_list=knowledge_list)
|
||||
|
||||
@classmethod
|
||||
async def get_one_assistant(cls, assistant_id: str) -> Optional[Assistant]:
|
||||
assistant = await AssistantDao.aget_one_assistant(assistant_id)
|
||||
return assistant
|
||||
|
||||
# Create Assistant
|
||||
@classmethod
|
||||
async def create_assistant(cls, request: Request, login_user: UserPayload, assistant: Assistant) \
|
||||
-> AssistantInfo:
|
||||
|
||||
# Check if there are any duplicate names under
|
||||
if cls.judge_name_repeat(assistant.name, assistant.user_id):
|
||||
raise AssistantNameRepeatError()
|
||||
|
||||
logger.info(f"assistant original prompt id: {assistant.id}, desc: {assistant.prompt}")
|
||||
|
||||
# Automatically replenish default model configurations
|
||||
assistant_llm = await LLMService.get_assistant_llm()
|
||||
if assistant_llm.llm_list:
|
||||
for one in assistant_llm.llm_list:
|
||||
if one.default:
|
||||
assistant.model_name = str(one.model_id)
|
||||
break
|
||||
|
||||
# Autogenerate Descriptions
|
||||
assistant, _, _ = await cls.get_auto_info(assistant, login_user)
|
||||
assistant = AssistantDao.create_assistant(assistant)
|
||||
|
||||
# RecordTelemetryJournal
|
||||
await telemetry_service.log_event(user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.NEW_APPLICATION,
|
||||
trace_id=trace_id_var.get(),
|
||||
event_data=NewApplicationEventData(
|
||||
app_id=assistant.id,
|
||||
app_name=assistant.name,
|
||||
app_type=ApplicationTypeEnum.ASSISTANT
|
||||
))
|
||||
|
||||
cls.create_assistant_hook(request, assistant, login_user)
|
||||
return AssistantInfo(**assistant.model_dump(),
|
||||
tool_list=[],
|
||||
flow_list=[],
|
||||
knowledge_list=[])
|
||||
|
||||
@classmethod
|
||||
def create_assistant_hook(cls, request: Request, assistant: Assistant, user_payload: UserPayload) -> bool:
|
||||
"""
|
||||
After successful creation of the assistanthook, perform some other business logic
|
||||
"""
|
||||
# Query the user group the user belongs to under
|
||||
user_group = UserGroupDao.get_user_group(user_payload.user_id)
|
||||
if user_group:
|
||||
# Batch Insert Assistant Resources into Correlation Table
|
||||
batch_resource = []
|
||||
for one in user_group:
|
||||
batch_resource.append(GroupResource(
|
||||
group_id=one.group_id,
|
||||
third_id=assistant.id,
|
||||
type=ResourceTypeEnum.ASSISTANT.value))
|
||||
GroupResourceDao.insert_group_batch(batch_resource)
|
||||
|
||||
# Write Audit Log
|
||||
AuditLogService.create_build_assistant(user_payload, get_request_ip(request), assistant.id)
|
||||
|
||||
# WritelogoCeacle
|
||||
cls.get_logo_share_link(assistant.logo)
|
||||
return True
|
||||
|
||||
# Delete Assistant
|
||||
@classmethod
|
||||
def delete_assistant(cls, request: Request, login_user: UserPayload, assistant_id: str) -> bool:
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
|
||||
# Judgment Authorization
|
||||
if not login_user.access_check(assistant.user_id, assistant.id, AccessType.ASSISTANT_WRITE):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
AssistantDao.delete_assistant(assistant)
|
||||
telemetry_service.log_event_sync(user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.DELETE_APPLICATION,
|
||||
trace_id=trace_id_var.get())
|
||||
cls.delete_assistant_hook(request, login_user, assistant)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def delete_assistant_hook(cls, request: Request, login_user: UserPayload, assistant: Assistant) -> bool:
|
||||
""" Clean up associated assistant resources """
|
||||
logger.info(f"delete_assistant_hook id: {assistant.id}, user: {login_user.user_id}")
|
||||
# Write Audit Log
|
||||
AuditLogService.delete_build_assistant(login_user, get_request_ip(request), assistant.id)
|
||||
|
||||
# Clean up associations with user groups
|
||||
GroupResourceDao.delete_group_resource_by_third_id(assistant.id, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
# Update session information
|
||||
MessageSessionDao.update_session_info_by_flow(assistant.name, assistant.desc, assistant.logo,
|
||||
assistant.id, FlowType.ASSISTANT.value)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def auto_update_stream(cls, assistant_id: str, prompt: str, login_user: UserPayload):
|
||||
""" Regenerate Assistant Prompts and Tool Selection, Only call the model capability without modifying the database data """
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
assistant.prompt = prompt
|
||||
|
||||
# Inisialisasillm
|
||||
auto_agent = AssistantAgent(assistant, '', login_user.user_id)
|
||||
await auto_agent.init_auto_update_llm()
|
||||
|
||||
# Streaming Generation Prompts
|
||||
final_prompt = ''
|
||||
async for one_prompt in auto_agent.optimize_assistant_prompt():
|
||||
if one_prompt.content in ('```', 'markdown'):
|
||||
continue
|
||||
yield str(StreamData(event='message', data={'type': 'prompt', 'message': one_prompt.content}))
|
||||
final_prompt += one_prompt.content
|
||||
assistant.prompt = final_prompt
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'message': ""}))
|
||||
|
||||
# Generate opening remarks and opening questions
|
||||
guide_info = auto_agent.generate_guide(assistant.prompt)
|
||||
yield str(StreamData(event='message', data={'type': 'guide_word', 'message': guide_info['opening_lines']}))
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'message': ""}))
|
||||
yield str(StreamData(event='message', data={'type': 'guide_question', 'message': guide_info['questions']}))
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'message': ""}))
|
||||
|
||||
# Automatic selection of tools and skills
|
||||
tool_info = cls.get_auto_tool_info(assistant, auto_agent)
|
||||
tool_info = [one.model_dump() for one in tool_info]
|
||||
yield str(StreamData(event='message', data={'type': 'tool_list', 'message': tool_info}))
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'message': ""}))
|
||||
|
||||
flow_info = cls.get_auto_flow_info(assistant, auto_agent)
|
||||
flow_info = [one.model_dump() for one in flow_info]
|
||||
yield str(StreamData(event='message', data={'type': 'flow_list', 'message': flow_info}))
|
||||
|
||||
@classmethod
|
||||
async def update_assistant(cls, request: Request, login_user: UserPayload, req: AssistantUpdateReq) \
|
||||
-> AssistantInfo:
|
||||
""" Update Assistant Information """
|
||||
assistant = AssistantDao.get_one_assistant(req.id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
|
||||
cls.check_update_permission(assistant, login_user)
|
||||
|
||||
# Update Assistant Data
|
||||
if req.name and req.name != assistant.name:
|
||||
# Check if there are any duplicate names under
|
||||
if cls.judge_name_repeat(req.name, assistant.user_id):
|
||||
raise AssistantNameRepeatError()
|
||||
assistant.name = req.name
|
||||
assistant.desc = req.desc
|
||||
assistant.logo = req.logo if req.logo else assistant.logo
|
||||
assistant.prompt = req.prompt
|
||||
assistant.guide_word = req.guide_word
|
||||
assistant.guide_question = req.guide_question
|
||||
assistant.model_name = req.model_name
|
||||
assistant.temperature = req.temperature
|
||||
assistant.update_time = datetime.now()
|
||||
assistant.max_token = req.max_token
|
||||
AssistantDao.update_assistant(assistant)
|
||||
telemetry_service.log_event_sync(user_id=login_user.user_id, event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get())
|
||||
|
||||
# Update assistant association information
|
||||
if req.tool_list is not None:
|
||||
AssistantLinkDao.update_assistant_tool(assistant.id, tool_list=req.tool_list)
|
||||
if req.flow_list is not None:
|
||||
AssistantLinkDao.update_assistant_flow(assistant.id, flow_list=req.flow_list)
|
||||
if req.knowledge_list is not None:
|
||||
# Using Configuredflow Perform skill replenishment
|
||||
AssistantLinkDao.update_assistant_knowledge(assistant.id,
|
||||
knowledge_list=req.knowledge_list,
|
||||
flow_id='')
|
||||
tool_list, flow_list, knowledge_list = cls.get_link_info(req.tool_list, req.flow_list,
|
||||
req.knowledge_list)
|
||||
cls.update_assistant_hook(request, login_user, assistant)
|
||||
return AssistantInfo(**assistant.model_dump(),
|
||||
tool_list=tool_list,
|
||||
flow_list=flow_list,
|
||||
knowledge_list=knowledge_list)
|
||||
|
||||
@classmethod
|
||||
def update_assistant_hook(cls, request: Request, login_user: UserPayload, assistant: Assistant) -> bool:
|
||||
""" Update Assistant's Hook """
|
||||
logger.info(f"delete_assistant_hook id: {assistant.id}, user: {login_user.user_id}")
|
||||
|
||||
# Write Audit Log
|
||||
AuditLogService.update_build_assistant(login_user, get_request_ip(request), assistant.id)
|
||||
|
||||
# Write cache
|
||||
cls.get_logo_share_link(assistant.logo)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def update_status(cls, request: Request, login_user: UserPayload, assistant_id: str,
|
||||
status: int) -> bool:
|
||||
""" Update Assistant Status """
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
# Determine permissions
|
||||
if not login_user.access_check(assistant.user_id, assistant.id, AccessType.ASSISTANT_WRITE):
|
||||
raise UnAuthorizedError()
|
||||
# Equal status without modification
|
||||
if assistant.status == status:
|
||||
return True
|
||||
|
||||
# Try to initializeagent, go online if initialization is successful, otherwise not go online
|
||||
if status == AssistantStatus.ONLINE.value:
|
||||
tmp_agent = AssistantAgent(assistant, '', login_user.user_id)
|
||||
try:
|
||||
await tmp_agent.init_assistant()
|
||||
except Exception as e:
|
||||
logger.exception('online agent init failed')
|
||||
raise AssistantInitError(exception=e)
|
||||
assistant.status = status
|
||||
AssistantDao.update_assistant(assistant)
|
||||
telemetry_service.log_event_sync(user_id=login_user.user_id, event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get())
|
||||
cls.update_assistant_hook(request, login_user, assistant)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def update_prompt(cls, assistant_id: str, prompt: str, user_payload: UserPayload) -> bool:
|
||||
""" Update assistant prompts """
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
|
||||
cls.check_update_permission(assistant, user_payload)
|
||||
|
||||
assistant.prompt = prompt
|
||||
AssistantDao.update_assistant(assistant)
|
||||
telemetry_service.log_event_sync(user_id=user_payload.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get())
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def update_flow_list(cls, assistant_id: str, flow_list: List[str],
|
||||
user_payload: UserPayload) -> bool:
|
||||
""" Update Assistant Skills List """
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
|
||||
cls.check_update_permission(assistant, user_payload)
|
||||
|
||||
AssistantLinkDao.update_assistant_flow(assistant_id, flow_list=flow_list)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def update_tool_list(cls, assistant_id: str, tool_list: List[int],
|
||||
user_payload: UserPayload) -> bool:
|
||||
""" Update Assistant Tool List """
|
||||
assistant = AssistantDao.get_one_assistant(assistant_id)
|
||||
if not assistant:
|
||||
raise AssistantNotExistsError()
|
||||
|
||||
cls.check_update_permission(assistant, user_payload)
|
||||
|
||||
AssistantLinkDao.update_assistant_tool(assistant_id, tool_list=tool_list)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def check_update_permission(cls, assistant: Assistant, user_payload: UserPayload) -> Any:
|
||||
# Determine permissions
|
||||
if not user_payload.access_check(assistant.user_id, assistant.id, AccessType.ASSISTANT_WRITE):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
# Changes are not allowed when online
|
||||
if assistant.status == AssistantStatus.ONLINE.value:
|
||||
raise AssistantNotEditError()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_link_info(cls,
|
||||
tool_list: List[int],
|
||||
flow_list: List[str],
|
||||
knowledge_list: List[int] = None):
|
||||
tool_list = GptsToolsDao.get_list_by_ids(tool_list) if tool_list else []
|
||||
flow_list = FlowDao.get_flow_by_ids(flow_list) if flow_list else []
|
||||
knowledge_list = KnowledgeDao.get_list_by_ids(knowledge_list) if knowledge_list else []
|
||||
return tool_list, flow_list, knowledge_list
|
||||
|
||||
@classmethod
|
||||
def get_user_name(cls, user_id: int):
|
||||
if not user_id:
|
||||
return 'system'
|
||||
user = cls.UserCache.get(user_id)
|
||||
if user:
|
||||
return user.user_name
|
||||
user = UserDao.get_user(user_id)
|
||||
if not user:
|
||||
return f'{user_id}'
|
||||
cls.UserCache.set(user_id, user)
|
||||
return user.user_name
|
||||
|
||||
@classmethod
|
||||
def judge_name_repeat(cls, name: str, user_id: int) -> bool:
|
||||
""" Determine if the assistant name is a duplicate """
|
||||
assistant = AssistantDao.get_assistant_by_name_user_id(name, user_id)
|
||||
if assistant:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def get_auto_info(cls, assistant: Assistant, login_user: UserPayload) -> (Assistant, List[int], List[int]):
|
||||
"""
|
||||
Auto Generate Assistant'sprompt, Automatically select tools and skills
|
||||
return: Assistant Information, ToolsIDList, SkillsIDVertical
|
||||
"""
|
||||
# Inisialisasiagent
|
||||
auto_agent = AssistantAgent(assistant, '', login_user.user_id)
|
||||
await auto_agent.init_auto_update_llm()
|
||||
|
||||
# Autogenerate Descriptions
|
||||
assistant.desc = auto_agent.generate_description(assistant.prompt)
|
||||
|
||||
return assistant, [], []
|
||||
|
||||
@classmethod
|
||||
def get_auto_tool_info(cls, assistant: Assistant, auto_agent: AssistantAgent) -> List[GptsTools]:
|
||||
# Pagination Auto-Select Tool
|
||||
res = []
|
||||
page = 1
|
||||
page_num = 50
|
||||
while True:
|
||||
all_tool = GptsToolsDao.get_list_by_user(assistant.user_id, page, page_num)
|
||||
if len(all_tool) == 0:
|
||||
break
|
||||
logger.info(f"auto select tools: page: {page}, number: {len(all_tool)}")
|
||||
tool_list = []
|
||||
all_tool_dict = {}
|
||||
for one in all_tool:
|
||||
all_tool_dict[one.name] = one
|
||||
tool_list.append({
|
||||
'name': one.name,
|
||||
'description': one.desc if one.desc else '',
|
||||
})
|
||||
tool_info = []
|
||||
tool_list = auto_agent.choose_tools(tool_list, assistant.prompt)
|
||||
for one in tool_list:
|
||||
if all_tool_dict.get(one):
|
||||
tool_info.append(all_tool_dict[one])
|
||||
res += tool_info
|
||||
page += 1
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def get_auto_flow_info(cls, assistant: Assistant, auto_agent: AssistantAgent) -> List[Flow]:
|
||||
# Automatically select skills, Before picking50skills to make automatic selections
|
||||
all_flow = FlowDao.get_user_access_online_flows(assistant.user_id, 1, 50)
|
||||
flow_dict = {}
|
||||
flow_list = []
|
||||
for one in all_flow:
|
||||
flow_dict[one.name] = one
|
||||
flow_list.append({
|
||||
'name': one.name,
|
||||
'description': one.description if one.description else '',
|
||||
})
|
||||
|
||||
flow_list = auto_agent.choose_tools(flow_list, assistant.prompt)
|
||||
flow_info = []
|
||||
for one in flow_list:
|
||||
if flow_dict.get(one):
|
||||
flow_info.append(flow_dict[one])
|
||||
return flow_info
|
||||
@@ -0,0 +1,380 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from langchain_core.callbacks import Callbacks
|
||||
from langchain_core.language_models import BaseLanguageModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage, BaseMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langchain_core.tools import BaseTool
|
||||
from langchain_core.utils.function_calling import format_tool_to_openai_tool
|
||||
from langgraph.prebuilt import create_react_agent
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.assistant_base import AssistantUtils
|
||||
from bisheng.common.constants.enums.telemetry import ApplicationTypeEnum
|
||||
from bisheng.common.errcode.assistant import AssistantModelEmptyError, AssistantModelNotConfigError, \
|
||||
AssistantAutoLLMError
|
||||
from bisheng.database.models.assistant import Assistant, AssistantLink, AssistantLinkDao
|
||||
from bisheng.llm.domain.services import LLMService
|
||||
from bisheng.tool.domain.services.executor import ToolExecutor
|
||||
from bisheng_langchain.gpts.assistant import ConfigurableAssistant
|
||||
from bisheng_langchain.gpts.auto_optimization import (generate_breif_description,
|
||||
generate_opening_dialog,
|
||||
optimize_assistant_prompt)
|
||||
from bisheng_langchain.gpts.auto_tool_selected import ToolInfo, ToolSelector
|
||||
from bisheng_langchain.gpts.prompts import ASSISTANT_PROMPT_OPT
|
||||
|
||||
|
||||
class AssistantAgent(AssistantUtils):
|
||||
# cohereThe special needs of the model prompt
|
||||
ASSISTANT_PROMPT_COHERE = """{preamble}|<instruct>|Carefully perform the following instructions, in order, starting each with a new line.
|
||||
Firstly, You may need to use complex and advanced reasoning to complete your task and answer the question. Think about how you can use the provided tools to answer the question and come up with a high level plan you will execute.
|
||||
Write 'Plan:' followed by an initial high level plan of how you will solve the problem including the tools and steps required.
|
||||
Secondly, Carry out your plan by repeatedly using actions, reasoning over the results, and re-evaluating your plan. Perform Action, Observation, Reflection steps with the following format. Write 'Action:' followed by a json formatted action containing the "tool_name" and "parameters"
|
||||
Next you will analyze the 'Observation:', this is the result of the action.
|
||||
After that you should always think about what to do next. Write 'Reflection:' followed by what you've figured out so far, any changes you need to make to your plan, and what you will do next including if you know the answer to the question.
|
||||
... (this Action/Observation/Reflection can repeat N times)
|
||||
Thirdly, Decide which of the retrieved documents are relevant to the user's last input by writing 'Relevant Documents:' followed by comma-separated list of document numbers. If none are relevant, you should instead write 'None'.
|
||||
Fourthly, Decide which of the retrieved documents contain facts that should be cited in a good answer to the user's last input by writing 'Cited Documents:' followed a comma-separated list of document numbers. If you dont want to cite any of them, you should instead write 'None'.
|
||||
Fifthly, Write 'Answer:' followed by a response to the user's last input. Use the retrieved documents to help you. Do not insert any citations or grounding markup.
|
||||
Finally, Write 'Grounded answer:' followed by a response to the user's last input in high quality natural english. Use the symbols <co: doc> and </co: doc> to indicate when a fact comes from a document in the search result, e.g <co: 4>my fact</co: 4> for a fact from document 4.
|
||||
|
||||
Additional instructions to note:
|
||||
- If the user's question is in Chinese, please answer it in Chinese.
|
||||
- When there is time information involved in a question, such as recently6Months, yesterday, last year, etc., you need to use the time tool to query the time information.
|
||||
""" # noqa
|
||||
|
||||
def __init__(self, assistant_info: Assistant, chat_id: str, invoke_user_id: int):
|
||||
self.assistant = assistant_info
|
||||
|
||||
# To record the data tracking points
|
||||
self.invoke_user_id = invoke_user_id
|
||||
|
||||
self.chat_id = chat_id
|
||||
self.tools: List[BaseTool] = []
|
||||
self.offline_flows = []
|
||||
self.agent: ConfigurableAssistant | None = None
|
||||
self.agent_executor_dict = {
|
||||
'ReAct': 'get_react_agent_executor',
|
||||
'function call': 'get_openai_functions_agent_executor',
|
||||
}
|
||||
self.current_agent_executor = None
|
||||
self.llm: BaseLanguageModel | None = None
|
||||
self.llm_agent_executor = None
|
||||
# Knowledge Base Retrieval Related Parameters
|
||||
self.knowledge_retriever = {'max_content': 15000, 'sort_by_source_and_index': False}
|
||||
|
||||
async def init_assistant(self, callbacks: Callbacks = None):
|
||||
await self.init_llm()
|
||||
await self.init_tools(callbacks)
|
||||
await self.init_agent()
|
||||
|
||||
async def init_llm(self):
|
||||
# Get a list of configured helper models
|
||||
assistant_llm = await LLMService.get_assistant_llm()
|
||||
if not assistant_llm.llm_list:
|
||||
raise AssistantModelEmptyError()
|
||||
default_llm = None
|
||||
for one in assistant_llm.llm_list:
|
||||
if str(one.model_id) == self.assistant.model_name:
|
||||
default_llm = one
|
||||
break
|
||||
elif not default_llm and one.default:
|
||||
default_llm = one
|
||||
if not default_llm:
|
||||
raise AssistantModelNotConfigError()
|
||||
|
||||
self.llm_agent_executor = default_llm.agent_executor_type
|
||||
self.knowledge_retriever = {
|
||||
'max_content': default_llm.knowledge_max_content,
|
||||
'sort_by_source_and_index': default_llm.knowledge_sort_index
|
||||
}
|
||||
|
||||
# Inisialisasillm
|
||||
self.llm = await LLMService.get_bisheng_llm(model_id=default_llm.model_id,
|
||||
temperature=self.assistant.temperature,
|
||||
streaming=default_llm.streaming,
|
||||
app_id=self.assistant.id,
|
||||
app_name=self.assistant.name,
|
||||
app_type=ApplicationTypeEnum.ASSISTANT,
|
||||
user_id=self.invoke_user_id)
|
||||
|
||||
async def init_auto_update_llm(self):
|
||||
""" Initialize Automatic Optimization prompt and other information.llmInstances """
|
||||
assistant_llm = await LLMService.get_assistant_llm()
|
||||
if not assistant_llm.auto_llm:
|
||||
raise AssistantAutoLLMError()
|
||||
|
||||
self.llm = await LLMService.get_bisheng_llm(model_id=assistant_llm.auto_llm.model_id,
|
||||
temperature=self.assistant.temperature,
|
||||
streaming=assistant_llm.auto_llm.streaming,
|
||||
app_id=self.assistant.id,
|
||||
app_name=self.assistant.name,
|
||||
app_type=ApplicationTypeEnum.ASSISTANT,
|
||||
user_id=self.invoke_user_id)
|
||||
|
||||
async def init_tools(self, callbacks: Callbacks = None):
|
||||
"""Get by nametool Vertical
|
||||
tools_name_param:: {name: params}
|
||||
"""
|
||||
links: List[AssistantLink] = await AssistantLinkDao.get_assistant_link(
|
||||
assistant_id=self.assistant.id)
|
||||
# tool
|
||||
tools: List[BaseTool] = []
|
||||
tool_ids = []
|
||||
flow_links = []
|
||||
for link in links:
|
||||
if link.tool_id:
|
||||
tool_ids.append(link.tool_id)
|
||||
else:
|
||||
flow_links.append(link)
|
||||
if tool_ids:
|
||||
tools = await ToolExecutor.init_by_tool_ids(tool_ids,
|
||||
app_id=self.assistant.id,
|
||||
app_name=self.assistant.name,
|
||||
app_type=ApplicationTypeEnum.ASSISTANT,
|
||||
user_id=self.invoke_user_id,
|
||||
llm=self.llm,
|
||||
callbacks=callbacks)
|
||||
|
||||
for link in flow_links:
|
||||
knowledge_id = link.knowledge_id
|
||||
if knowledge_id:
|
||||
knowledge_tool = await ToolExecutor.init_knowledge_tool(self.invoke_user_id, knowledge_id, llm=self.llm,
|
||||
callbacks=callbacks,
|
||||
**self.knowledge_retriever)
|
||||
tools.append(knowledge_tool)
|
||||
self.tools = tools
|
||||
|
||||
async def init_agent(self):
|
||||
"""
|
||||
Initialize agentagent
|
||||
"""
|
||||
# Introductionagentexecution parameter
|
||||
agent_executor_type = self.llm_agent_executor
|
||||
self.current_agent_executor = agent_executor_type
|
||||
# Do the Conversion
|
||||
agent_executor_type = self.agent_executor_dict.get(agent_executor_type,
|
||||
agent_executor_type)
|
||||
|
||||
prompt = self.assistant.prompt
|
||||
if getattr(self.llm, 'model_name', '').startswith('command-r'):
|
||||
prompt = self.ASSISTANT_PROMPT_COHERE.format(preamble=prompt)
|
||||
if self.current_agent_executor == 'ReAct':
|
||||
# Inisialisasiagent
|
||||
self.agent = ConfigurableAssistant(agent_executor_type=agent_executor_type,
|
||||
tools=self.tools,
|
||||
llm=self.llm,
|
||||
assistant_message=prompt)
|
||||
else:
|
||||
# function-callingpattern, but also add recursive constraints
|
||||
logger.info(f'Creating LangGraph agent with {len(self.tools)} tools, llm type: {type(self.llm)}')
|
||||
logger.info(f'LLM streaming capability: {getattr(self.llm, "streaming", "unknown")}')
|
||||
|
||||
self.agent = create_react_agent(self.llm, self.tools, prompt=prompt, checkpointer=False)
|
||||
logger.info(f'LangGraph agent created: {type(self.agent)}')
|
||||
|
||||
# areagentAdd Recursive Limit Configuration
|
||||
self.agent = self.agent.with_config({'recursion_limit': 100})
|
||||
logger.info(f'Agent config applied: recursion_limit=100')
|
||||
|
||||
async def optimize_assistant_prompt(self):
|
||||
""" Automatically optimize generationprompt """
|
||||
chain = ({
|
||||
'assistant_name': lambda x: x['assistant_name'],
|
||||
'assistant_description': lambda x: x['assistant_description'],
|
||||
}
|
||||
| ASSISTANT_PROMPT_OPT
|
||||
| self.llm)
|
||||
async for one in chain.astream({
|
||||
'assistant_name': self.assistant.name,
|
||||
'assistant_description': self.assistant.prompt,
|
||||
}):
|
||||
yield one
|
||||
|
||||
def sync_optimize_assistant_prompt(self):
|
||||
return optimize_assistant_prompt(self.llm, self.assistant.name, self.assistant.desc)
|
||||
|
||||
def generate_guide(self, prompt: str):
|
||||
""" Generate opening dialogue and opening questions """
|
||||
return generate_opening_dialog(self.llm, prompt)
|
||||
|
||||
def generate_description(self, prompt: str):
|
||||
""" Generate description dialog """
|
||||
return generate_breif_description(self.llm, prompt)
|
||||
|
||||
def choose_tools(self, tool_list: List[Dict[str, str]], prompt: str) -> List[str]:
|
||||
"""
|
||||
Choose A Tool
|
||||
tool_list: [{name: xxx, description: xxx}]
|
||||
"""
|
||||
tool_list = [
|
||||
ToolInfo(tool_name=one['name'], tool_description=one['description'])
|
||||
for one in tool_list
|
||||
]
|
||||
tool_selector = ToolSelector(llm=self.llm, tools=tool_list)
|
||||
return tool_selector.select(self.assistant.name, prompt)
|
||||
|
||||
async def fake_callback(self, callback: Callbacks):
|
||||
if not callback:
|
||||
return
|
||||
# False callback to call back skills that are offline to the front-end
|
||||
for one in self.offline_flows:
|
||||
run_id = uuid.uuid4()
|
||||
await callback[0].on_tool_start({
|
||||
'name': one,
|
||||
},
|
||||
input_str='flow is offline',
|
||||
run_id=run_id)
|
||||
await callback[0].on_tool_end(output='flow is offline', name=one, run_id=run_id)
|
||||
|
||||
async def record_chat_history(self, message: List[Any]):
|
||||
# Record Assistant Chat History
|
||||
if not os.getenv('BISHENG_RECORD_HISTORY'):
|
||||
return
|
||||
try:
|
||||
os.makedirs('/app/data/history', exist_ok=True)
|
||||
with open(f'/app/data/history/{self.assistant.id}_{time.time()}.json',
|
||||
'w',
|
||||
encoding='utf-8') as f:
|
||||
json.dump(
|
||||
{
|
||||
'system': self.assistant.prompt,
|
||||
'message': message,
|
||||
'tools': [format_tool_to_openai_tool(t) for t in self.tools]
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False)
|
||||
except Exception as e:
|
||||
logger.error(f'record assistant history error: {str(e)}')
|
||||
|
||||
async def trim_messages(self, messages: List[Any]) -> List[Any]:
|
||||
# Dapatkanencoding
|
||||
enc = self.cl100k_base()
|
||||
|
||||
def get_finally_message(new_messages: List[Any]) -> List[Any]:
|
||||
# No more processing until only one record has been trimmed
|
||||
if len(new_messages) == 1:
|
||||
return new_messages
|
||||
total_count = 0
|
||||
for one in new_messages:
|
||||
if isinstance(one, HumanMessage):
|
||||
total_count += len(enc.encode(one.content))
|
||||
elif isinstance(one, AIMessage):
|
||||
total_count += len(enc.encode(one.content))
|
||||
if 'tool_calls' in one.additional_kwargs:
|
||||
total_count += len(
|
||||
enc.encode(json.dumps(one.additional_kwargs['tool_calls'], ensure_ascii=False))
|
||||
)
|
||||
else:
|
||||
total_count += len(enc.encode(str(one.content)))
|
||||
if total_count > self.assistant.max_token:
|
||||
return get_finally_message(new_messages[1:])
|
||||
return new_messages
|
||||
|
||||
return get_finally_message(messages)
|
||||
|
||||
async def run(self, query: str, chat_history: List = None, callback: Callbacks = None) -> List[BaseMessage]:
|
||||
"""
|
||||
Run Agent Conversation
|
||||
"""
|
||||
await self.fake_callback(callback)
|
||||
|
||||
if chat_history:
|
||||
chat_history.append(HumanMessage(content=query))
|
||||
inputs = chat_history
|
||||
else:
|
||||
inputs = [HumanMessage(content=query)]
|
||||
|
||||
# trim message
|
||||
inputs = await self.trim_messages(inputs)
|
||||
|
||||
if self.current_agent_executor == 'ReAct':
|
||||
result = await self.react_run(inputs, callback)
|
||||
else:
|
||||
result = await self.agent.ainvoke({'messages': inputs}, config=RunnableConfig(callbacks=callback))
|
||||
result = result['messages']
|
||||
|
||||
# Record Chat History
|
||||
await self.record_chat_history([one.to_json() for one in result])
|
||||
|
||||
return result
|
||||
|
||||
async def astream(self, query: str, chat_history: List = None, callback: Callbacks = None):
|
||||
"""
|
||||
Run Agent Conversation - Streaming version
|
||||
"""
|
||||
await self.fake_callback(callback)
|
||||
|
||||
if chat_history:
|
||||
chat_history.append(HumanMessage(content=query))
|
||||
inputs = chat_history
|
||||
else:
|
||||
inputs = [HumanMessage(content=query)]
|
||||
|
||||
# trim message
|
||||
inputs = await self.trim_messages(inputs)
|
||||
|
||||
if self.current_agent_executor == 'ReAct':
|
||||
# ReActMode temporarily does not support streaming, downgrade to non streaming
|
||||
result = await self.react_run(inputs, callback)
|
||||
# Record Chat History
|
||||
await self.record_chat_history([one.to_json() for one in result])
|
||||
yield result
|
||||
else:
|
||||
# Use Streaming Calls
|
||||
config = RunnableConfig(callbacks=callback)
|
||||
final_messages = []
|
||||
|
||||
logger.info(f'Using function-calling mode, starting astream...')
|
||||
|
||||
chunk_count = 0
|
||||
|
||||
try:
|
||||
# UsemessagesPatternedLangGraph streamingattaintokenLevel of Streaming Output
|
||||
async for chunk in self.agent.astream({'messages': inputs}, config=config, stream_mode="messages"):
|
||||
chunk_count += 1
|
||||
|
||||
# stream_mode="messages" Return (message, metadata) Meta Group
|
||||
message = None
|
||||
if isinstance(chunk, tuple) and len(chunk) >= 2:
|
||||
message, metadata = chunk[:2]
|
||||
elif hasattr(chunk, 'content'):
|
||||
# Directly to the message object
|
||||
message = chunk
|
||||
|
||||
if message:
|
||||
# stream_mode="messages"Returns Independencechunk, use its content directly
|
||||
final_messages = [message] # Save message for history
|
||||
yield [message]
|
||||
|
||||
except Exception as astream_error:
|
||||
logger.exception(f'Error in astream async for loop: {str(astream_error)}')
|
||||
raise astream_error
|
||||
|
||||
logger.info(f'Function calling astream completed, total chunks: {chunk_count}')
|
||||
|
||||
if chunk_count == 0:
|
||||
logger.warning(f'No chunks received from agent.astream()! This indicates a streaming issue.')
|
||||
|
||||
# Record Chat History
|
||||
if final_messages:
|
||||
await self.record_chat_history([one.to_json() for one in final_messages])
|
||||
|
||||
async def react_run(self, inputs: List, callback: Callbacks = None):
|
||||
""" react Mode input and execution """
|
||||
result = await self.agent.ainvoke({
|
||||
'input': inputs[-1].content,
|
||||
'chat_history': inputs[:-1],
|
||||
}, config=RunnableConfig(callbacks=callback))
|
||||
logger.debug(f"react_run result: {result}")
|
||||
output = result['agent_outcome'].return_values['output']
|
||||
if isinstance(output, dict):
|
||||
output = list(output.values())[0]
|
||||
for one in result['intermediate_steps']:
|
||||
inputs.append(one[0])
|
||||
inputs.append(AIMessage(content=output))
|
||||
return inputs
|
||||
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
|
||||
from tiktoken.load import load_tiktoken_bpe
|
||||
from tiktoken.core import Encoding as TikTokenEncoding
|
||||
|
||||
|
||||
class AssistantUtils:
|
||||
# Ignore assistant configuration has been removed from the system configuration, no such method is required for now
|
||||
|
||||
@staticmethod
|
||||
def cl100k_base() -> TikTokenEncoding:
|
||||
ENDOFTEXT = "<|endoftext|>"
|
||||
FIM_PREFIX = "<|fim_prefix|>"
|
||||
FIM_MIDDLE = "<|fim_middle|>"
|
||||
FIM_SUFFIX = "<|fim_suffix|>"
|
||||
ENDOFPROMPT = "<|endofprompt|>"
|
||||
|
||||
tiktoken_file = os.path.join(os.path.dirname(__file__), "tiktoken_file/cl100k_base.tiktoken")
|
||||
|
||||
mergeable_ranks = load_tiktoken_bpe(
|
||||
# "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
|
||||
tiktoken_file,
|
||||
expected_hash="223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7",
|
||||
)
|
||||
special_tokens = {
|
||||
ENDOFTEXT: 100257,
|
||||
FIM_PREFIX: 100258,
|
||||
FIM_MIDDLE: 100259,
|
||||
FIM_SUFFIX: 100260,
|
||||
ENDOFPROMPT: 100276,
|
||||
}
|
||||
return TikTokenEncoding(**{
|
||||
"name": "cl100k_base",
|
||||
"pat_str": r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s""",
|
||||
"mergeable_ranks": mergeable_ranks,
|
||||
"special_tokens": special_tokens,
|
||||
})
|
||||
@@ -0,0 +1,809 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Dict, Union, Tuple
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import func
|
||||
from sqlmodel import col, or_, and_, select
|
||||
|
||||
from bisheng.api.v1.schema.chat_schema import AppChatList
|
||||
from bisheng.api.v1.schema.workflow import WorkflowEventType
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError
|
||||
from bisheng.database.models.assistant import AssistantDao, Assistant
|
||||
from bisheng.database.models.audit_log import AuditLog, SystemId, EventType, ObjectType, AuditLogDao
|
||||
from bisheng.database.models.flow import FlowDao, Flow, FlowType
|
||||
from bisheng.database.models.group import Group
|
||||
from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum
|
||||
from bisheng.database.models.message import ChatMessageDao
|
||||
from bisheng.database.models.role import Role
|
||||
from bisheng.database.models.session import MessageSessionDao, MessageSession
|
||||
from bisheng.database.models.user_group import UserGroupDao
|
||||
from bisheng.knowledge.domain.models.knowledge import KnowledgeDao, Knowledge
|
||||
from bisheng.tool.domain.models.gpts_tools import GptsToolsType
|
||||
from bisheng.user.domain.models.user import UserDao, User
|
||||
|
||||
|
||||
# todo change to async or submit thread pool
|
||||
class AuditLogService:
|
||||
|
||||
@classmethod
|
||||
async def get_audit_log(cls, login_user: UserPayload, group_ids, operator_ids, start_time, end_time,
|
||||
system_id, event_type, page, limit) -> Any:
|
||||
groups = group_ids
|
||||
if not login_user.is_admin():
|
||||
groups = [str(one.group_id) for one in await UserGroupDao.aget_user_admin_group(login_user.user_id)]
|
||||
# Not an administrator of any user groups
|
||||
if not groups:
|
||||
return UnAuthorizedError.return_resp()
|
||||
# Filter bygroup_idand administrator permissionsgroupsDoing Intersections
|
||||
if group_ids:
|
||||
groups = list(set(groups) & set(group_ids))
|
||||
if not groups:
|
||||
return UnAuthorizedError.return_resp()
|
||||
|
||||
data, total = await AuditLogDao.get_audit_logs(groups, operator_ids, start_time, end_time,
|
||||
system_id,
|
||||
event_type,
|
||||
page, limit)
|
||||
return resp_200(data={'data': data, 'total': total})
|
||||
|
||||
@classmethod
|
||||
def get_all_operators(cls, login_user: UserPayload) -> List[Dict]:
|
||||
groups = []
|
||||
if not login_user.is_admin():
|
||||
groups = [one.group_id for one in UserGroupDao.get_user_admin_group(login_user.user_id)]
|
||||
# not any group admin
|
||||
if not groups:
|
||||
raise UnAuthorizedError()
|
||||
|
||||
data = AuditLogDao.get_all_operators(groups)
|
||||
res = {}
|
||||
for one in data:
|
||||
if not one[1]:
|
||||
continue
|
||||
res[one[0]] = {'user_id': one[0], 'user_name': one[1]}
|
||||
return list(res.values())
|
||||
|
||||
@classmethod
|
||||
def _chat_log(cls, user: UserPayload, ip_address: str, event_type: EventType, object_type: ObjectType,
|
||||
object_id: str, object_name: str, resource_type: ResourceTypeEnum):
|
||||
# Get the group to which the resource belongs
|
||||
groups = GroupResourceDao.get_resource_group(resource_type, object_id)
|
||||
group_ids = [one.group_id for one in groups]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.CHAT.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
AuditLogDao.insert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def _chat_log_async(cls, user: UserPayload, ip_address: str, event_type: EventType,
|
||||
object_type: ObjectType,
|
||||
object_id: str, object_name: str, resource_type: ResourceTypeEnum,
|
||||
group_ids: List[int] = None):
|
||||
# Get the group to which the resource belongs
|
||||
if group_ids is None:
|
||||
groups = await GroupResourceDao.aget_resource_group(resource_type, object_id)
|
||||
group_ids = [one.group_id for one in groups]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.CHAT.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
def create_chat_assistant(cls, user: UserPayload, ip_address: str, assistant_id: str):
|
||||
"""
|
||||
New Audit Log for Assistant Session
|
||||
"""
|
||||
logger.info(f"act=create_chat_assistant user={user.user_name} ip={ip_address} assistant={assistant_id}")
|
||||
# Getting Assistant Details
|
||||
assistant_info = AssistantDao.get_one_assistant(assistant_id)
|
||||
cls._chat_log(user, ip_address, EventType.CREATE_CHAT, ObjectType.ASSISTANT,
|
||||
assistant_id, assistant_info.name, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
@classmethod
|
||||
def create_chat_workflow(cls, user: UserPayload, ip_address: str, flow_id: str, flow_info=None):
|
||||
"""
|
||||
New Workflow Session Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_chat_workflow user={user.user_name} ip={ip_address} flow={flow_id}")
|
||||
if not flow_info:
|
||||
flow_info = FlowDao.get_flow_by_id(flow_id)
|
||||
cls._chat_log(user, ip_address, EventType.CREATE_CHAT, ObjectType.WORK_FLOW,
|
||||
flow_id, flow_info.name, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
@classmethod
|
||||
async def delete_chat_workflow(cls, user: UserPayload, ip_address: str, flow_info: Flow):
|
||||
"""
|
||||
Delete Audit Log for Workflow Session
|
||||
"""
|
||||
logger.info(f"act=delete_chat_workflow user={user.user_name} ip={ip_address} flow={flow_info.id}")
|
||||
await cls._chat_log_async(user, ip_address, EventType.DELETE_CHAT, ObjectType.WORK_FLOW,
|
||||
flow_info.id, flow_info.name, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
@classmethod
|
||||
async def delete_chat_assistant(cls, user: UserPayload, ip_address: str, assistant_info: Assistant):
|
||||
"""
|
||||
Delete audit log for assistant session
|
||||
"""
|
||||
logger.info(f"act=delete_assistant_flow user={user.user_name} ip={ip_address} assistant={assistant_info.id}")
|
||||
await cls._chat_log_async(user, ip_address, EventType.DELETE_CHAT, ObjectType.ASSISTANT,
|
||||
assistant_info.id, assistant_info.name, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
@classmethod
|
||||
def _build_log(cls, user: UserPayload, ip_address: str, event_type: EventType, object_type: ObjectType,
|
||||
object_id: str,
|
||||
object_name: str, resource_type: ResourceTypeEnum):
|
||||
"""
|
||||
Build Module Audit Log
|
||||
"""
|
||||
# Get which user groups the resource belongs to
|
||||
groups = GroupResourceDao.get_resource_group(resource_type, object_id)
|
||||
group_ids = [one.group_id for one in groups]
|
||||
|
||||
# Insert Audit Log
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.BUILD.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
AuditLogDao.insert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def _build_log_async(cls, user: UserPayload, ip_address: str, event_type: EventType, object_type: ObjectType,
|
||||
object_id: str,
|
||||
object_name: str, resource_type: ResourceTypeEnum):
|
||||
"""
|
||||
Build Module Audit Log
|
||||
"""
|
||||
# Get which user groups the resource belongs to
|
||||
groups = await GroupResourceDao.aget_resource_group(resource_type, object_id)
|
||||
group_ids = [one.group_id for one in groups]
|
||||
|
||||
# Insert Audit Log
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.BUILD.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
def create_build_workflow(cls, user: UserPayload, ip_address: str, flow_id: str):
|
||||
"""
|
||||
New Workflow Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_build_workflow user={user.user_name} ip={ip_address} flow={flow_id}")
|
||||
flow_info = FlowDao.get_flow_by_id(flow_id)
|
||||
cls._build_log(user, ip_address, EventType.CREATE_BUILD, ObjectType.WORK_FLOW,
|
||||
flow_info.id, flow_info.name, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
@classmethod
|
||||
async def update_build_workflow(cls, user: UserPayload, ip_address: str, flow_id: str):
|
||||
"""
|
||||
Update Workflow Audit Log
|
||||
"""
|
||||
logger.info(f"act=update_build_workflow user={user.user_name} ip={ip_address} flow={flow_id}")
|
||||
flow_info = await FlowDao.aget_flow_by_id(flow_id)
|
||||
await cls._build_log_async(user, ip_address, EventType.UPDATE_BUILD, ObjectType.WORK_FLOW, flow_info.id,
|
||||
flow_info.name, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
@classmethod
|
||||
def delete_build_workflow(cls, user: UserPayload, ip_address: str, flow_info: Flow):
|
||||
"""
|
||||
Delete Workflow Audit Log
|
||||
"""
|
||||
logger.info(f"act=delete_build_workflow user={user.user_name} ip={ip_address} flow={flow_info.id}")
|
||||
cls._build_log(user, ip_address, EventType.DELETE_BUILD, ObjectType.WORK_FLOW,
|
||||
flow_info.id, flow_info.name, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
@classmethod
|
||||
def create_build_assistant(cls, user: UserPayload, ip_address: str, assistant_id: str):
|
||||
"""
|
||||
New Assistant Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_build_assistant user={user.user_name} ip={ip_address} assistant={assistant_id}")
|
||||
assistant_info = AssistantDao.get_one_assistant(assistant_id)
|
||||
cls._build_log(user, ip_address, EventType.CREATE_BUILD, ObjectType.ASSISTANT,
|
||||
assistant_info.id, assistant_info.name, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
@classmethod
|
||||
def update_build_assistant(cls, user: UserPayload, ip_address: str, assistant_id: str):
|
||||
"""
|
||||
Update the assistant's audit log
|
||||
"""
|
||||
logger.info(f"act=update_build_assistant user={user.user_name} ip={ip_address} assistant={assistant_id}")
|
||||
assistant_info = AssistantDao.get_one_assistant(assistant_id)
|
||||
|
||||
cls._build_log(user, ip_address, EventType.UPDATE_BUILD, ObjectType.ASSISTANT,
|
||||
assistant_info.id, assistant_info.name, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
@classmethod
|
||||
def delete_build_assistant(cls, user: UserPayload, ip_address: str, assistant_id: str):
|
||||
"""
|
||||
Delete Audit Log for Assistant
|
||||
"""
|
||||
logger.info(f"act=delete_build_assistant user={user.user_name} ip={ip_address} assistant={assistant_id}")
|
||||
assistant_info = AssistantDao.get_one_assistant(assistant_id)
|
||||
|
||||
cls._build_log(user, ip_address, EventType.DELETE_BUILD, ObjectType.ASSISTANT,
|
||||
assistant_info.id, assistant_info.name, ResourceTypeEnum.ASSISTANT)
|
||||
|
||||
@classmethod
|
||||
async def create_chat_message(cls, user: UserPayload, ip_address: str, message: Union[str, MessageSession]):
|
||||
"""
|
||||
New Chat Message Audit Log for Build Module
|
||||
"""
|
||||
if isinstance(message, MessageSession):
|
||||
message_session = message
|
||||
else:
|
||||
message_session = await MessageSessionDao.async_get_one(message)
|
||||
|
||||
logger.info(f"act=create_chat_message user={user.user_name} ip={ip_address} session={message_session.chat_id}")
|
||||
|
||||
user_groups = await UserGroupDao.aget_user_group(message_session.user_id)
|
||||
group_ids = [ug.group_id for ug in user_groups]
|
||||
|
||||
await cls._chat_log_async(user, ip_address, EventType.CREATE_CHAT, ObjectType.WORKSTATION,
|
||||
message_session.chat_id, message_session.name, ResourceTypeEnum.WORKSTATION,
|
||||
group_ids)
|
||||
|
||||
@classmethod
|
||||
async def delete_chat_message(cls, user: UserPayload, ip_address: str, message: Union[str, MessageSession]):
|
||||
"""
|
||||
Delete Chat Message Audit Log for Build Module
|
||||
"""
|
||||
if isinstance(message, MessageSession):
|
||||
message_session = message
|
||||
else:
|
||||
message_session = await MessageSessionDao.async_get_one(message)
|
||||
|
||||
logger.info(f"act=delete_chat_message user={user.user_name} ip={ip_address} session={message_session.chat_id}")
|
||||
|
||||
await cls._chat_log_async(user, ip_address, EventType.DELETE_CHAT, ObjectType.WORKSTATION,
|
||||
message_session.chat_id, message_session.name, ResourceTypeEnum.WORKSTATION)
|
||||
|
||||
@classmethod
|
||||
def _knowledge_log(cls, user: UserPayload, ip_address: str, event_type: EventType, object_type: ObjectType,
|
||||
object_id: str, object_name: str, resource_type: ResourceTypeEnum, resource_id: str):
|
||||
"""
|
||||
Logs of Knowledge Base Modules
|
||||
"""
|
||||
# Get which user groups the resource belongs to
|
||||
groups = GroupResourceDao.get_resource_group(resource_type, resource_id)
|
||||
group_ids = [one.group_id for one in groups]
|
||||
|
||||
# Insert Audit Log
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.KNOWLEDGE.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
AuditLogDao.insert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
def create_knowledge(cls, user: UserPayload, ip_address: str, knowledge_id: int):
|
||||
"""
|
||||
New Knowledge Base Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_knowledge user={user.user_name} ip={ip_address} knowledge={knowledge_id}")
|
||||
knowledge_info = KnowledgeDao.query_by_id(knowledge_id)
|
||||
cls._knowledge_log(user, ip_address, EventType.CREATE_KNOWLEDGE, ObjectType.KNOWLEDGE,
|
||||
str(knowledge_id), knowledge_info.name, ResourceTypeEnum.KNOWLEDGE, str(knowledge_id))
|
||||
|
||||
@classmethod
|
||||
def delete_knowledge(cls, user: UserPayload, ip_address: str, knowledge: Knowledge):
|
||||
"""
|
||||
Delete Knowledge Base Audit Log
|
||||
"""
|
||||
logger.info(f"act=delete_knowledge user={user.user_name} ip={ip_address} knowledge={knowledge.id}")
|
||||
cls._knowledge_log(user, ip_address, EventType.DELETE_KNOWLEDGE, ObjectType.KNOWLEDGE,
|
||||
str(knowledge.id), knowledge.name, ResourceTypeEnum.KNOWLEDGE, str(knowledge.id))
|
||||
|
||||
@classmethod
|
||||
async def create_knowledge_space(cls, user: UserPayload, ip_address: str, knowledge: Knowledge):
|
||||
"""
|
||||
New Knowledge Space Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_knowledge_space user={user.user_name} ip={ip_address} knowledge={knowledge.id}")
|
||||
user_group = await UserGroupDao.aget_user_group(user.user_id)
|
||||
group_ids = [one.group_id for one in user_group]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.KNOWLEDGE_SPACE.value,
|
||||
event_type=EventType.CREATE_KNOWLEDGE_SPACE.value,
|
||||
object_type=ObjectType.KNOWLEDGE_SPACE.value,
|
||||
object_id=str(knowledge.id),
|
||||
object_name=knowledge.name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def delete_knowledge_space(cls, user: UserPayload, ip_address: str, knowledge: Knowledge):
|
||||
"""
|
||||
Delete Knowledge Space Audit Log
|
||||
"""
|
||||
logger.info(f"act=delete_knowledge_space user={user.user_name} ip={ip_address} knowledge={knowledge.id}")
|
||||
user_group = await UserGroupDao.aget_user_group(user.user_id)
|
||||
group_ids = [one.group_id for one in user_group]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.KNOWLEDGE_SPACE.value,
|
||||
event_type=EventType.DELETE_KNOWLEDGE_SPACE.value,
|
||||
object_type=ObjectType.KNOWLEDGE_SPACE.value,
|
||||
object_id=str(knowledge.id),
|
||||
object_name=knowledge.name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
def upload_knowledge_file(cls, user: UserPayload, ip_address: str, knowledge_id: int, file_name: str):
|
||||
"""
|
||||
Audit Logs for Knowledge Base Upload Files
|
||||
"""
|
||||
logger.info(f"act=upload_knowledge_file user={user.user_name} ip={ip_address}"
|
||||
f" knowledge={knowledge_id} file={file_name}")
|
||||
cls._knowledge_log(user, ip_address, EventType.UPLOAD_FILE, ObjectType.FILE,
|
||||
str(knowledge_id), file_name, ResourceTypeEnum.KNOWLEDGE, str(knowledge_id))
|
||||
|
||||
@classmethod
|
||||
def delete_knowledge_file(cls, user: UserPayload, ip_address: str, knowledge_id: int, file_name: str):
|
||||
"""
|
||||
Audit Logs for Knowledge Base Deletion Files
|
||||
"""
|
||||
logger.info(f"act=delete_knowledge_file user={user.user_name} ip={ip_address}"
|
||||
f" knowledge={knowledge_id} file={file_name}")
|
||||
cls._knowledge_log(user, ip_address, EventType.DELETE_FILE, ObjectType.FILE,
|
||||
str(knowledge_id), file_name, ResourceTypeEnum.KNOWLEDGE, str(knowledge_id))
|
||||
|
||||
@classmethod
|
||||
def _system_log(cls, user: UserPayload, ip_address: str, group_ids: List[int], event_type: EventType,
|
||||
object_type: ObjectType, object_id: str, object_name: str, note: str = ''):
|
||||
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.SYSTEM.value,
|
||||
event_type=event_type.value,
|
||||
object_type=object_type.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
note=note,
|
||||
)
|
||||
AuditLogDao.insert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
def update_user(cls, user: UserPayload, ip_address: str, user_id: int, group_ids: List[int], note: str):
|
||||
"""
|
||||
Modify a user's user groups and roles
|
||||
"""
|
||||
logger.info(f"act=update_system_user user={user.user_name} ip={ip_address} user_id={user_id} note={note}")
|
||||
user_info = UserDao.get_user(user_id)
|
||||
cls._system_log(user, ip_address, group_ids, EventType.UPDATE_USER,
|
||||
ObjectType.USER_CONF, str(user_id), user_info.user_name, note)
|
||||
|
||||
@classmethod
|
||||
def forbid_user(cls, user: UserPayload, ip_address: str, user_info: User):
|
||||
"""
|
||||
user: Action User
|
||||
user_info: Operated by user
|
||||
"""
|
||||
logger.info(f"act=forbid_user user={user.user_name} ip={ip_address} user_id={user.user_id}")
|
||||
# Get the group to which the user belongs
|
||||
user_group = UserGroupDao.get_user_group(user_info.user_id)
|
||||
user_group = [one.group_id for one in user_group]
|
||||
cls._system_log(user, ip_address, user_group, EventType.FORBID_USER,
|
||||
ObjectType.USER_CONF, str(user_info.user_id), user_info.user_name)
|
||||
|
||||
@classmethod
|
||||
def recover_user(cls, user: UserPayload, ip_address: str, user_info: User):
|
||||
logger.info(f"act=recover_user user={user.user_name} ip={ip_address} user_id={user_info.user_id}")
|
||||
# Get the group to which the user belongs
|
||||
user_group = UserGroupDao.get_user_group(user_info.user_id)
|
||||
user_group = [one.group_id for one in user_group]
|
||||
cls._system_log(user, ip_address, user_group, EventType.RECOVER_USER,
|
||||
ObjectType.USER_CONF, str(user_info.user_id), user_info.user_name)
|
||||
|
||||
@classmethod
|
||||
def create_user_group(cls, user: UserPayload, ip_address: str, group_info: Group):
|
||||
logger.info(f"act=create_user_group user={user.user_name} ip={ip_address} group_id={group_info.id}")
|
||||
cls._system_log(user, ip_address, [group_info.id], EventType.CREATE_USER_GROUP,
|
||||
ObjectType.USER_GROUP_CONF, str(group_info.id), group_info.group_name)
|
||||
|
||||
@classmethod
|
||||
def update_user_group(cls, user: UserPayload, ip_address: str, group_info: Group):
|
||||
logger.info(f"act=update_user_group user={user.user_name} ip={ip_address} group_id={group_info.id}")
|
||||
# Get user group information
|
||||
cls._system_log(user, ip_address, [group_info.id], EventType.UPDATE_USER_GROUP,
|
||||
ObjectType.USER_GROUP_CONF, str(group_info.id), group_info.group_name)
|
||||
|
||||
@classmethod
|
||||
def delete_user_group(cls, user: UserPayload, ip_address: str, group_info: Group):
|
||||
logger.info(f"act=delete_user_group user={user.user_name} ip={ip_address} group_id={group_info.id}")
|
||||
# Get user group information
|
||||
cls._system_log(user, ip_address, [group_info.id], EventType.DELETE_USER_GROUP,
|
||||
ObjectType.USER_GROUP_CONF, str(group_info.id), group_info.group_name)
|
||||
|
||||
@classmethod
|
||||
def create_role(cls, user: UserPayload, ip_address: str, role: Role):
|
||||
logger.info(f"act=create_role user={user.user_name} ip={ip_address} role_id={role.id}")
|
||||
|
||||
cls._system_log(user, ip_address, [role.group_id], EventType.CREATE_ROLE,
|
||||
ObjectType.ROLE_CONF, str(role.id), role.role_name)
|
||||
|
||||
@classmethod
|
||||
def update_role(cls, user: UserPayload, ip_address: str, role: Role):
|
||||
logger.info(f"act=update_role user={user.user_name} ip={ip_address} role_id={role.id}")
|
||||
|
||||
cls._system_log(user, ip_address, [role.group_id], EventType.UPDATE_ROLE,
|
||||
ObjectType.ROLE_CONF, str(role.id), role.role_name)
|
||||
|
||||
@classmethod
|
||||
def delete_role(cls, user: UserPayload, ip_address: str, role: Role):
|
||||
logger.info(f"act=delete_role user={user.user_name} ip={ip_address} role_id={role.id}")
|
||||
|
||||
cls._system_log(user, ip_address, [role.group_id], EventType.DELETE_ROLE,
|
||||
ObjectType.ROLE_CONF, str(role.id), role.role_name)
|
||||
|
||||
@classmethod
|
||||
def create_tool(cls, user: UserPayload, ip_address: str, group_ids: List[int], tool_type: GptsToolsType):
|
||||
logger.info(f"act=create_tool user={user.user_name} ip={ip_address} tool_type_id={tool_type.id}")
|
||||
|
||||
cls._system_log(user, ip_address, group_ids, EventType.ADD_TOOL, ObjectType.TOOL, str(tool_type.id),
|
||||
tool_type.name)
|
||||
|
||||
@classmethod
|
||||
def update_tool(cls, user: UserPayload, ip_address: str, group_ids: List[int], tool_type: GptsToolsType):
|
||||
logger.info(f"act=update_tool user={user.user_name} ip={ip_address} tool_type_id={tool_type.id}")
|
||||
|
||||
cls._system_log(user, ip_address, group_ids, EventType.UPDATE_TOOL, ObjectType.TOOL, str(tool_type.id),
|
||||
tool_type.name)
|
||||
|
||||
@classmethod
|
||||
def delete_tool(cls, user: UserPayload, ip_address: str, group_ids: List[int], tool_type: GptsToolsType):
|
||||
logger.info(f"act=delete_tool user={user.user_name} ip={ip_address} tool_type_id={tool_type.id}")
|
||||
cls._system_log(user, ip_address, group_ids, EventType.DELETE_TOOL, ObjectType.TOOL, str(tool_type.id),
|
||||
tool_type.name)
|
||||
|
||||
@classmethod
|
||||
def user_login(cls, user: UserPayload, ip_address: str):
|
||||
logger.info(f"act=user_login user={user.user_name} ip={ip_address} user_id={user.user_id}")
|
||||
# Get the group to which the user belongs
|
||||
user_group = UserGroupDao.get_user_group(user.user_id)
|
||||
user_group = [one.group_id for one in user_group]
|
||||
cls._system_log(user, ip_address, user_group, EventType.USER_LOGIN,
|
||||
ObjectType.NONE, '', '')
|
||||
|
||||
@classmethod
|
||||
async def _dashboard_log(cls, user: UserPayload, ip_address: str, group_ids: List[int], event_type: EventType,
|
||||
object_id: str, object_name: str):
|
||||
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.DASHBOARD.value,
|
||||
event_type=event_type.value,
|
||||
object_type=ObjectType.DASHBOARD.value,
|
||||
object_id=object_id,
|
||||
object_name=object_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def create_dashboard(cls, user: UserPayload, ip_address: str, dashboard_id: str, dashboard_name: str,
|
||||
group_ids: List[int]):
|
||||
logger.info(f"act=create_dashboard user={user.user_name} ip={ip_address} dashboard_id={dashboard_id}")
|
||||
await cls._dashboard_log(user, ip_address, group_ids, EventType.CREATE_DASHBOARD, dashboard_id, dashboard_name)
|
||||
|
||||
@classmethod
|
||||
async def update_dashboard(cls, user: UserPayload, ip_address: str, dashboard_id: str, dashboard_name: str,
|
||||
group_ids: List[int]):
|
||||
logger.info(f"act=update_dashboard user={user.user_name} ip={ip_address} dashboard_id={dashboard_id}")
|
||||
await cls._dashboard_log(user, ip_address, group_ids, EventType.UPDATE_DASHBOARD, dashboard_id, dashboard_name)
|
||||
|
||||
@classmethod
|
||||
async def delete_dashboard(cls, user: UserPayload, ip_address: str, dashboard_id: str, dashboard_name: str,
|
||||
group_ids: List[int]):
|
||||
logger.info(f"act=delete_dashboard user={user.user_name} ip={ip_address} dashboard_id={dashboard_id}")
|
||||
await cls._dashboard_log(user, ip_address, group_ids, EventType.DELETE_DASHBOARD, dashboard_id, dashboard_name)
|
||||
|
||||
@classmethod
|
||||
async def create_channel(cls, user: UserPayload, ip_address: str, channel_id: str, channel_name: str):
|
||||
"""
|
||||
New Channel Audit Log
|
||||
"""
|
||||
logger.info(f"act=create_channel user={user.user_name} ip={ip_address} channel={channel_id}")
|
||||
user_group = await UserGroupDao.aget_user_group(user.user_id)
|
||||
group_ids = [one.group_id for one in user_group]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.SUBSCRIPTION.value,
|
||||
event_type=EventType.CREATE_CHANNEL.value,
|
||||
object_type=ObjectType.CHANNEL.value,
|
||||
object_id=channel_id,
|
||||
object_name=channel_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def delete_channel(cls, user: UserPayload, ip_address: str, channel_id: str, channel_name: str):
|
||||
"""
|
||||
Delete Channel Audit Log
|
||||
"""
|
||||
logger.info(f"act=delete_channel user={user.user_name} ip={ip_address} channel={channel_id}")
|
||||
user_group = await UserGroupDao.aget_user_group(user.user_id)
|
||||
group_ids = [one.group_id for one in user_group]
|
||||
audit_log = AuditLog(
|
||||
operator_id=user.user_id,
|
||||
operator_name=user.user_name,
|
||||
group_ids=group_ids,
|
||||
system_id=SystemId.SUBSCRIPTION.value,
|
||||
event_type=EventType.DELETE_CHANNEL.value,
|
||||
object_type=ObjectType.CHANNEL.value,
|
||||
object_id=channel_id,
|
||||
object_name=channel_name,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
await AuditLogDao.ainsert_audit_logs([audit_log])
|
||||
|
||||
@classmethod
|
||||
async def get_filter_flow_ids(cls, user: UserPayload, flow_ids: List[str], group_ids: List[int]) -> (bool, List):
|
||||
"""Filter workflow, assistant and workstation ids by visible groups."""
|
||||
flow_ids = [one for one in flow_ids]
|
||||
group_admins = []
|
||||
if not user.is_admin():
|
||||
user_groups = await UserGroupDao.aget_user_admin_group(user.user_id)
|
||||
# Not a user group administrator, no permissions
|
||||
if not user_groups:
|
||||
raise UnAuthorizedError.http_exception()
|
||||
group_admins = [one.group_id for one in user_groups]
|
||||
# GroupingidDoing Intersections
|
||||
if group_ids:
|
||||
if group_admins:
|
||||
# Query user group not belonging to user management, return empty
|
||||
group_admins = list(set(group_admins) & set(group_ids))
|
||||
if len(group_admins) == 0:
|
||||
return False, []
|
||||
else:
|
||||
group_admins = group_ids
|
||||
|
||||
# Get all apps under groupingID
|
||||
group_flows = []
|
||||
if group_admins:
|
||||
group_flows = await GroupResourceDao.get_groups_resource(group_admins,
|
||||
resource_types=[ResourceTypeEnum.WORK_FLOW,
|
||||
ResourceTypeEnum.ASSISTANT,
|
||||
ResourceTypeEnum.WORKSTATION])
|
||||
# User group under user management has no resources
|
||||
if not group_flows:
|
||||
return False, []
|
||||
group_flows = [one.third_id for one in group_flows]
|
||||
|
||||
# Acquire the final skillIDRestrict to list
|
||||
filter_flow_ids = []
|
||||
if flow_ids and group_flows:
|
||||
filter_flow_ids = list(set(group_flows) & set(flow_ids))
|
||||
if not filter_flow_ids:
|
||||
return False, []
|
||||
elif flow_ids:
|
||||
filter_flow_ids = flow_ids
|
||||
elif group_flows:
|
||||
filter_flow_ids = group_flows
|
||||
return True, filter_flow_ids
|
||||
|
||||
@classmethod
|
||||
async def get_session_list(cls, user: UserPayload, flow_ids: List[str], user_ids: List[int], group_ids: List[int],
|
||||
start_date: datetime, end_date: datetime,
|
||||
feedback: str, sensitive_status: int, page: int, page_size: int) -> Tuple[
|
||||
List[AppChatList], int]:
|
||||
|
||||
if user.is_admin():
|
||||
# Administrator: The frontend sends out what it needs to retrieve; if nothing is sent, it retrieves all (an empty list usually means there are no restrictions or the decision is made by the business logic in subsequent logic).
|
||||
search_group_ids = group_ids or []
|
||||
else:
|
||||
# Regular users: Administrative privileges must be verified
|
||||
user_managed_groups = await UserGroupDao.aget_user_admin_group(user.user_id)
|
||||
if not user_managed_groups:
|
||||
raise UnAuthorizedError.http_exception()
|
||||
|
||||
managed_group_ids = {one.group_id for one in user_managed_groups}
|
||||
|
||||
if group_ids:
|
||||
# Find the intersection: the intersection of the frontend requests
|
||||
valid_group_ids = list(set(group_ids) & managed_group_ids)
|
||||
if not valid_group_ids:
|
||||
return [], 0
|
||||
search_group_ids = valid_group_ids
|
||||
else:
|
||||
# Default: All managed groups
|
||||
search_group_ids = list(managed_group_ids)
|
||||
|
||||
conditions = []
|
||||
|
||||
# Basic equality/range filtering
|
||||
if sensitive_status:
|
||||
conditions.append(MessageSession.sensitive_status == sensitive_status)
|
||||
|
||||
if user_ids:
|
||||
conditions.append(col(MessageSession.user_id).in_(user_ids))
|
||||
|
||||
if start_date:
|
||||
conditions.append(col(MessageSession.create_time) >= start_date)
|
||||
if end_date:
|
||||
conditions.append(col(MessageSession.create_time) <= end_date)
|
||||
|
||||
if flow_ids:
|
||||
conditions.append(col(MessageSession.flow_id).in_(flow_ids))
|
||||
|
||||
# Process type filtering (fixed enumeration)
|
||||
conditions.append(col(MessageSession.flow_type).in_([
|
||||
FlowType.WORKFLOW.value,
|
||||
FlowType.ASSISTANT.value,
|
||||
FlowType.WORKSTATION.value
|
||||
]))
|
||||
|
||||
# Feedback status filtering
|
||||
feedback_map = {
|
||||
'like': col(MessageSession.like) > 0,
|
||||
'dislike': col(MessageSession.dislike) > 0,
|
||||
'copied': col(MessageSession.copied) > 0
|
||||
}
|
||||
if feedback in feedback_map:
|
||||
conditions.append(feedback_map[feedback])
|
||||
|
||||
# Group membership filtering
|
||||
if search_group_ids:
|
||||
group_filters = [
|
||||
func.json_contains(MessageSession.group_ids, str(gid))
|
||||
for gid in search_group_ids
|
||||
]
|
||||
conditions.append(or_(*group_filters))
|
||||
|
||||
# build query statement
|
||||
statement = select(MessageSession).where(and_(*conditions)).order_by(col(MessageSession.create_time).desc())
|
||||
|
||||
res_task = MessageSessionDao.get_statement_results(statement, page=page, limit=page_size)
|
||||
total_task = MessageSessionDao.get_statement_count(statement)
|
||||
|
||||
res, total = await asyncio.gather(res_task, total_task)
|
||||
|
||||
if not res:
|
||||
return [], total
|
||||
|
||||
target_user_ids = set()
|
||||
target_flow_ids = set() # Flow/Workflow
|
||||
target_assistant_ids = set() # Assistant
|
||||
|
||||
for session in res:
|
||||
target_user_ids.add(session.user_id)
|
||||
if session.flow_type in [FlowType.WORKFLOW.value, FlowType.WORKSTATION.value]:
|
||||
target_flow_ids.add(session.flow_id)
|
||||
elif session.flow_type == FlowType.ASSISTANT.value:
|
||||
target_assistant_ids.add(session.flow_id)
|
||||
|
||||
target_user_ids_list = list(target_user_ids)
|
||||
|
||||
async def get_users_groups_map(u_ids: List[int]):
|
||||
# get user groups for multiple users
|
||||
if not u_ids: return {}
|
||||
tasks = [user.get_user_groups(uid) for uid in u_ids]
|
||||
results = await asyncio.gather(*tasks)
|
||||
return dict(zip(u_ids, results))
|
||||
|
||||
users_data, flows_data, assistants_data, user_groups_map = await asyncio.gather(
|
||||
UserDao.aget_user_by_ids(target_user_ids_list),
|
||||
FlowDao.aget_flow_by_ids(list(target_flow_ids)),
|
||||
AssistantDao.aget_assistants_by_ids(list(target_assistant_ids)),
|
||||
get_users_groups_map(target_user_ids_list)
|
||||
)
|
||||
|
||||
user_map = {u.user_id: u.user_name for u in users_data}
|
||||
flow_map = {f.id: f.name for f in flows_data}
|
||||
assistant_map = {a.id: a.name for a in assistants_data}
|
||||
|
||||
# Construct the return object
|
||||
result: List[AppChatList] = []
|
||||
|
||||
for session in res:
|
||||
# Determine the current name
|
||||
current_name = session.flow_name
|
||||
if session.flow_type in [FlowType.WORKFLOW.value, FlowType.WORKSTATION.value]:
|
||||
current_name = flow_map.get(session.flow_id, current_name)
|
||||
elif session.flow_type == FlowType.ASSISTANT.value:
|
||||
current_name = assistant_map.get(session.flow_id, current_name)
|
||||
|
||||
# Append to the result set
|
||||
result.append(AppChatList(
|
||||
**session.model_dump(exclude={'flow_name'}),
|
||||
flow_name=current_name,
|
||||
like_count=session.like,
|
||||
dislike_count=session.dislike,
|
||||
copied_count=session.copied,
|
||||
user_name=user_map.get(session.user_id, ""), # get user name
|
||||
user_groups=user_groups_map.get(session.user_id, [])
|
||||
))
|
||||
|
||||
return result, total
|
||||
|
||||
@classmethod
|
||||
async def get_session_messages(cls, user: UserPayload, flow_ids: List[str], user_ids: List[int],
|
||||
group_ids: List[int],
|
||||
start_date: datetime, end_date: datetime, feedback: str,
|
||||
sensitive_status: int) -> List[AppChatList]:
|
||||
page = 1
|
||||
page_size = 50
|
||||
res = []
|
||||
while True:
|
||||
result, total = await cls.get_session_list(user, flow_ids, user_ids, group_ids, start_date, end_date,
|
||||
feedback,
|
||||
sensitive_status, page, page_size)
|
||||
if not result:
|
||||
break
|
||||
page += 1
|
||||
res.extend(await cls.get_chat_messages(result))
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def get_chat_messages(cls, chat_list: List[AppChatList]) -> List[AppChatList]:
|
||||
chat_ids = [chat.chat_id for chat in chat_list]
|
||||
|
||||
chat_messages = await ChatMessageDao.get_all_message_by_chat_ids(chat_ids)
|
||||
chat_messages_map = {}
|
||||
for one in chat_messages:
|
||||
if one.chat_id not in chat_messages_map:
|
||||
chat_messages_map[one.chat_id] = []
|
||||
chat_messages_map[one.chat_id].append(one)
|
||||
for chat in chat_list:
|
||||
chat_messages = chat_messages_map.get(chat.chat_id, [])
|
||||
# remove workflow input event, because it's not show in web
|
||||
chat.messages = [message for message in chat_messages
|
||||
if message.category != WorkflowEventType.UserInput.value]
|
||||
return chat_list
|
||||
@@ -0,0 +1,155 @@
|
||||
import asyncio
|
||||
import json
|
||||
# Pengaturan websockets Log level is NONE
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from pydantic import BaseModel
|
||||
from websockets import connect
|
||||
|
||||
from bisheng.common.errcode.http_error import ServerError
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.database.models.message import ChatMessage
|
||||
|
||||
# Maintain a connection pool
|
||||
connection_pool = defaultdict(asyncio.Queue)
|
||||
logging.getLogger('websockets').setLevel(logging.ERROR)
|
||||
|
||||
expire = 600 # reids 60s Overdue
|
||||
|
||||
|
||||
class TimedQueue:
|
||||
|
||||
def __init__(self):
|
||||
self.queue = asyncio.Queue()
|
||||
self.last_active = datetime.now()
|
||||
|
||||
async def put_nowait(self, item):
|
||||
self.last_active = datetime.now()
|
||||
await self.queue.put(item)
|
||||
|
||||
async def get_nowait(self):
|
||||
self.last_active = datetime.now()
|
||||
return await self.queue.get()
|
||||
|
||||
def empty(self):
|
||||
return self.queue.empty()
|
||||
|
||||
def qsize(self):
|
||||
return self.queue.qsize()
|
||||
|
||||
|
||||
async def clean_inactive_queues(queue: defaultdict, timeout_threshold: timedelta):
|
||||
while True:
|
||||
current_time = datetime.now()
|
||||
for key, timed_queue in list(queue.items()):
|
||||
# If the queue is not active beyond the set threshold time, clear the queue
|
||||
if current_time - timed_queue.last_active > timeout_threshold:
|
||||
while not timed_queue.empty():
|
||||
timed_queue.get_nowait() # Remove task from queue
|
||||
del queue[key] # Delete queue
|
||||
await asyncio.sleep(timeout_threshold.total_seconds())
|
||||
|
||||
|
||||
# Maintain a connection pool
|
||||
connection_pool = defaultdict(TimedQueue)
|
||||
|
||||
|
||||
# clean_inactive_queues(connection_pool, timedelta(minutes=5))
|
||||
|
||||
|
||||
async def get_connection(uri, identifier):
|
||||
"""
|
||||
DapatkanWebSocketConnections. Returns directly if there are connections available in the connection pool;
|
||||
Otherwise, create a new connection and add it to the connection pool.
|
||||
"""
|
||||
if connection_pool[identifier].empty():
|
||||
# build newWebSocketCONNECT
|
||||
websocket = await connect(uri)
|
||||
|
||||
await connection_pool[identifier].put_nowait(websocket)
|
||||
|
||||
# Get Connection from Connection Pool
|
||||
websocket = await connection_pool[identifier].get_nowait()
|
||||
return websocket
|
||||
|
||||
|
||||
async def release_connection(identifier, websocket):
|
||||
"""
|
||||
releaseWebSocketConnect and put it back into the connection pool.
|
||||
"""
|
||||
await connection_pool[identifier].put_nowait(websocket)
|
||||
|
||||
|
||||
def comment_answer(message_id: int, comment: str):
|
||||
with get_sync_db_session() as session:
|
||||
message = session.get(ChatMessage, message_id)
|
||||
if message:
|
||||
message.remark = comment[:4096]
|
||||
session.add(message)
|
||||
session.commit()
|
||||
|
||||
|
||||
class ContentStreamResp(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChoiceStreamResp(BaseModel):
|
||||
index: int = 0
|
||||
delta: ContentStreamResp = 0
|
||||
session_id: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
jsonData = '{"index": "%s", "delta": %s, "session_id": "%s"}' % (
|
||||
self.index, json.dumps(self.delta.dict(), ensure_ascii=False), self.session_id)
|
||||
return '{"choices":[%s]}\n\n' % (jsonData)
|
||||
|
||||
|
||||
async def event_stream(
|
||||
webosocket: connect,
|
||||
message: str,
|
||||
session_id: str,
|
||||
model: str,
|
||||
streaming: bool,
|
||||
):
|
||||
payload = {'inputs': message, 'flow_id': model, 'chat_id': session_id}
|
||||
try:
|
||||
await webosocket.send(json.dumps(payload, ensure_ascii=False))
|
||||
except Exception as e:
|
||||
yield ServerError(exception=e).to_sse_event_instance_str()
|
||||
return
|
||||
sync = ''
|
||||
while True:
|
||||
try:
|
||||
msg = await webosocket.recv()
|
||||
except Exception as e:
|
||||
yield ServerError(exception=e).to_sse_event_instance_str()
|
||||
break
|
||||
if msg is None:
|
||||
continue
|
||||
# Judgingmsg of income they generate.
|
||||
res = json.loads(msg)
|
||||
if streaming:
|
||||
if res.get('type') != 'end' and res.get('message'):
|
||||
delta = ContentStreamResp(role='assistant', content=res.get('message'))
|
||||
yield str(ChoiceStreamResp(index=0, session_id=session_id, delta=delta))
|
||||
else:
|
||||
# Control the following via thecloseWhether to send a message
|
||||
if res.get('type') == 'end':
|
||||
sync = res.get('message')
|
||||
|
||||
if res.get('type') == 'close':
|
||||
if not streaming and sync:
|
||||
delta = ContentStreamResp(role='assistant', content=sync)
|
||||
msg = ChoiceStreamResp(index=0,
|
||||
session_id=session_id,
|
||||
delta=delta,
|
||||
finish_reason='stop')
|
||||
yield '{"choices":[%s]}' % (json.dumps(msg.dict()))
|
||||
# Release Connection
|
||||
elif streaming:
|
||||
yield 'data: [DONE]'
|
||||
await release_connection(session_id, webosocket)
|
||||
break
|
||||
@@ -0,0 +1,79 @@
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from bisheng.api.v1.schema.dataset_param import CreateDatasetParam
|
||||
from bisheng.common.errcode.dataset import DatasetNameExistsError
|
||||
from bisheng.common.services.base import BaseService
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage_sync
|
||||
from bisheng.database.models.dataset import Dataset, DatasetCreate, DatasetDao, DatasetRead
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
|
||||
|
||||
class DatasetService(BaseService):
|
||||
|
||||
@classmethod
|
||||
def build_dataset_list(cls,
|
||||
page: int,
|
||||
limit: int,
|
||||
keyword: Optional[str] = None) -> (List[Dict], int):
|
||||
"""completelist DATA"""
|
||||
|
||||
dataset_list = DatasetDao.filter_dataset_by_ids(dataset_ids=[],
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
limit=limit)
|
||||
count_filter = []
|
||||
if keyword:
|
||||
count_filter.append(Dataset.name.like('%{}%'.format(keyword)))
|
||||
total_count = DatasetDao.get_count_by_filter(count_filter)
|
||||
|
||||
user_ids = [one.user_id for one in dataset_list]
|
||||
user_list = UserDao.get_user_by_ids(user_ids)
|
||||
user_dict = {one.user_id: one for one in user_list}
|
||||
res = [DatasetRead.model_validate(one) for one in dataset_list]
|
||||
for one in res:
|
||||
one.user_name = user_dict[one.user_id].user_name
|
||||
if one.object_name:
|
||||
one.url = one.object_name
|
||||
|
||||
return res, total_count
|
||||
|
||||
@classmethod
|
||||
def create_dataset(cls, user_id: int, data: CreateDatasetParam):
|
||||
"""Create Dataset"""
|
||||
dataset_insert = DatasetCreate.validate(data)
|
||||
dataset_insert.user_id = user_id
|
||||
isExist = DatasetDao.get_dataset_by_name(data.name)
|
||||
if isExist:
|
||||
raise DatasetNameExistsError()
|
||||
dataset = DatasetDao.insert(dataset_insert)
|
||||
# Conditioning Documentation
|
||||
object_name = f'/dataset/{dataset.id}/{dataset.name}'
|
||||
if data.file_url:
|
||||
# MinioClient().upload_minio()
|
||||
dataset.object_name = object_name
|
||||
if data.qa_list:
|
||||
for qa in data.qa_list:
|
||||
qa.dataset_id = dataset.id
|
||||
# QADao.insert(qa)
|
||||
|
||||
dataset = DatasetDao.update(dataset)
|
||||
return dataset
|
||||
|
||||
@classmethod
|
||||
def delete_dataset(cls, dataset_id: int):
|
||||
dataset = DatasetDao.get_dataset_by_id(dataset_id)
|
||||
if not dataset:
|
||||
raise HTTPException(status_code=404, detail='Dataset not found')
|
||||
# <g id="Bold">Medical Treatment:</g>minio
|
||||
object_name = dataset.object_name
|
||||
if object_name:
|
||||
minio_client = get_minio_storage_sync()
|
||||
minio_client.remove_object_sync(object_name=object_name)
|
||||
DatasetDao.delete(dataset)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def get_one_by_object_name(cls, object_name: str) -> Optional[Dataset]:
|
||||
dataset = await DatasetDao.aget_dataset_by_object_name(object_name)
|
||||
@@ -0,0 +1,335 @@
|
||||
# flake8: noqa
|
||||
"""Loads PDF with semantic splilter."""
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
import aiofiles
|
||||
import cv2
|
||||
import fitz
|
||||
import requests
|
||||
from PIL import Image
|
||||
from aiohttp import ClientTimeout
|
||||
from langchain_community.docstore.document import Document
|
||||
from langchain_community.document_loaders.pdf import BasePDFLoader
|
||||
|
||||
from bisheng.core.external.http_client.http_client_manager import get_http_client
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage_sync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_image_tag(results, part):
|
||||
element_id = part.get("element_id", None)
|
||||
url = results.get(element_id)
|
||||
return f""
|
||||
|
||||
|
||||
def get_image_parts(partitions):
|
||||
page_dict = {}
|
||||
for part in partitions:
|
||||
label = part["type"]
|
||||
if label == "Image":
|
||||
bboxes = part.get("metadata", {}).get("extra_data", {}).get("bboxes", [])
|
||||
page = part.get("metadata", {}).get("extra_data", {}).get("pages", -1)
|
||||
element_id = part.get("element_id", None)
|
||||
if len(bboxes) == 0 or page == -1 or not element_id:
|
||||
continue
|
||||
item = {}
|
||||
item["bboxes"] = bboxes[0]
|
||||
item["element_id"] = element_id
|
||||
page_id = page[0]
|
||||
if page_id not in page_dict:
|
||||
page_dict[page_id] = []
|
||||
page_dict[page_id].append(item)
|
||||
return page_dict
|
||||
|
||||
|
||||
def crop_image(image_file, item, cropped_imag_base_dir):
|
||||
element_id = item.get("element_id")
|
||||
bbox = item.get("bboxes")
|
||||
img = cv2.imread(image_file)
|
||||
x1, y1, x2, y2 = bbox
|
||||
cropped_img = img[y1:y2, x1:x2]
|
||||
file_name = f"{element_id}.png"
|
||||
cv2.imwrite(os.path.join(cropped_imag_base_dir, file_name), cropped_img)
|
||||
return file_name
|
||||
|
||||
|
||||
def extract_pdf_images(file_name, page_dict, doc_id, knowledge_id):
|
||||
from bisheng.api.services.knowledge_imp import put_images_to_minio
|
||||
from bisheng.api.services.knowledge_imp import KnowledgeUtils
|
||||
from bisheng.core.cache.utils import CACHE_DIR
|
||||
|
||||
result = {}
|
||||
base_dir = f"{CACHE_DIR}/{doc_id}"
|
||||
cropped_image_base_dir = f"{base_dir}/images"
|
||||
pdf_page_base_dir = f"{base_dir}/images"
|
||||
|
||||
if not os.path.exists(pdf_page_base_dir):
|
||||
os.makedirs(pdf_page_base_dir)
|
||||
if not os.path.exists(cropped_image_base_dir):
|
||||
os.makedirs(cropped_image_base_dir)
|
||||
|
||||
pdf_document = fitz.open(file_name)
|
||||
|
||||
minio_client = get_minio_storage_sync()
|
||||
|
||||
for page_number, items in page_dict.items():
|
||||
page = pdf_document[page_number]
|
||||
pix = page.get_pixmap()
|
||||
image = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
pdf_image_file_name = f"{pdf_page_base_dir}/{page_number}.png"
|
||||
image.save(pdf_image_file_name)
|
||||
for item in items:
|
||||
cropped_image_file = crop_image(
|
||||
pdf_image_file_name, item, cropped_image_base_dir
|
||||
)
|
||||
result[item["element_id"]] = (
|
||||
f"/{minio_client.bucket}/{KnowledgeUtils.get_knowledge_file_image_dir(doc_id, knowledge_id)}/{cropped_image_file}"
|
||||
)
|
||||
put_images_to_minio(cropped_image_base_dir, knowledge_id, doc_id)
|
||||
return result
|
||||
|
||||
|
||||
def pre_handle(partitions, file_name, knowledge_id):
|
||||
doc_id = str(uuid4())
|
||||
image_parts = get_image_parts(partitions=partitions)
|
||||
if len(image_parts) == 0:
|
||||
return []
|
||||
return extract_pdf_images(file_name, image_parts, doc_id, knowledge_id)
|
||||
|
||||
|
||||
def merge_partitions(file_name, partitions, knowledge_id=None):
|
||||
# Pre-processingpdf, Extracting Images
|
||||
pre_handle_results = pre_handle(
|
||||
partitions=partitions, file_name=file_name, knowledge_id=knowledge_id
|
||||
)
|
||||
text_elem_sep = "\n"
|
||||
doc_content = []
|
||||
is_first_elem = True
|
||||
last_label = ""
|
||||
prev_length = 0
|
||||
metadata = dict(bboxes=[], pages=[], indexes=[], types=[])
|
||||
|
||||
for part in partitions:
|
||||
label, text = part["type"], part["text"]
|
||||
extra_data = part["metadata"]["extra_data"]
|
||||
if label == "Image":
|
||||
part["text"] = get_image_tag(pre_handle_results, part)
|
||||
text = part["text"]
|
||||
|
||||
if is_first_elem:
|
||||
f_text = text + "\n" if label == "Title" else text
|
||||
doc_content.append(f_text)
|
||||
is_first_elem = False
|
||||
else:
|
||||
if last_label == "Title" and label == "Title":
|
||||
doc_content.append("\n" + text)
|
||||
elif label == "Title":
|
||||
doc_content.append("\n\n" + text)
|
||||
elif label == "Table":
|
||||
doc_content.append("\n\n" + text)
|
||||
else:
|
||||
if last_label == "Table":
|
||||
doc_content.append(text_elem_sep * 2 + text)
|
||||
else:
|
||||
doc_content.append(text_elem_sep + text)
|
||||
|
||||
last_label = label
|
||||
metadata["bboxes"].extend(
|
||||
list(map(lambda x: list(map(int, x)), extra_data["bboxes"]))
|
||||
)
|
||||
metadata["pages"].extend(extra_data["pages"])
|
||||
metadata["types"].extend(extra_data["types"])
|
||||
|
||||
indexes = extra_data["indexes"]
|
||||
up_indexes = [[s + prev_length, e + prev_length] for (s, e) in indexes]
|
||||
metadata["indexes"].extend(up_indexes)
|
||||
prev_length += len(doc_content[-1])
|
||||
|
||||
content = "".join(doc_content)
|
||||
return content, metadata
|
||||
|
||||
|
||||
class Etl4lmLoader(BasePDFLoader):
|
||||
"""Loads a PDF with pypdf and chunks at character level. dummy version
|
||||
|
||||
Loader also stores page numbers in metadata.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_name: str,
|
||||
file_path: str,
|
||||
unstructured_api_key: str = None,
|
||||
unstructured_api_url: str = None,
|
||||
force_ocr: bool = False,
|
||||
enable_formular: bool = True,
|
||||
filter_page_header_footer: bool = False,
|
||||
ocr_sdk_url: str = None,
|
||||
timeout: int = 60,
|
||||
knowledge_id: int = None,
|
||||
start: int = 0,
|
||||
n: int = None,
|
||||
verbose: bool = False,
|
||||
kwargs: dict = {},
|
||||
) -> None:
|
||||
"""Initialize with a file path."""
|
||||
self.unstructured_api_url = unstructured_api_url
|
||||
self.unstructured_api_key = unstructured_api_key
|
||||
self.force_ocr = force_ocr
|
||||
self.enable_formular = enable_formular
|
||||
self.filter_page_header_footer = filter_page_header_footer
|
||||
self.ocr_sdk_url = ocr_sdk_url
|
||||
self.headers = {"Content-Type": "application/json"}
|
||||
self.file_name = file_name
|
||||
self.timemout = timeout
|
||||
self.start = start
|
||||
self.n = n
|
||||
self.extra_kwargs = kwargs
|
||||
self.partitions = None
|
||||
self.knowledge_id = knowledge_id
|
||||
super().__init__(file_path)
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Load given path as pages."""
|
||||
b64_data = base64.b64encode(open(self.file_path, "rb").read()).decode()
|
||||
parameters = {"start": self.start, "n": self.n}
|
||||
parameters.update(self.extra_kwargs)
|
||||
# TODO: add filter_page_header_footer into payload when elt4llm is ready.
|
||||
payload = dict(
|
||||
filename=os.path.basename(self.file_name),
|
||||
b64_data=[b64_data],
|
||||
mode="partition",
|
||||
force_ocr=self.force_ocr,
|
||||
enable_formula=self.enable_formular,
|
||||
ocr_sdk_url=self.ocr_sdk_url,
|
||||
parameters=parameters,
|
||||
)
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.unstructured_api_url, headers=self.headers, json=payload, timeout=self.timemout
|
||||
)
|
||||
except requests.Timeout as e:
|
||||
logger.error(f"Request to etl4lm API timed out: {e}")
|
||||
raise Exception("etl4lm server timeout")
|
||||
except Exception as e:
|
||||
if str(e).find("Timeout") != -1:
|
||||
logger.error(f"Request to etl4lm API timed out: {e}")
|
||||
raise Exception("etl4lm server timeout")
|
||||
raise e
|
||||
if resp.status_code != 200:
|
||||
raise Exception(
|
||||
f"file partition {os.path.basename(self.file_name)} failed resp={resp.text}"
|
||||
)
|
||||
|
||||
resp = resp.json()
|
||||
if 200 != resp.get("status_code"):
|
||||
logger.info(
|
||||
f"file partition {os.path.basename(self.file_name)} error resp={resp}"
|
||||
)
|
||||
raise Exception(
|
||||
f"file partition error {os.path.basename(self.file_name)} error resp={resp}"
|
||||
)
|
||||
partitions = resp["partitions"]
|
||||
if partitions:
|
||||
logger.info(f"content_from_partitions")
|
||||
self.partitions = partitions
|
||||
content, metadata = merge_partitions(
|
||||
self.file_path, partitions, self.knowledge_id
|
||||
)
|
||||
elif resp.get("text"):
|
||||
logger.info(f"content_from_text")
|
||||
content = resp["text"]
|
||||
metadata = {
|
||||
"bboxes": [],
|
||||
"pages": [],
|
||||
"indexes": [],
|
||||
"types": [],
|
||||
}
|
||||
else:
|
||||
logger.warning(f"content_is_empty resp={resp}")
|
||||
content = ""
|
||||
metadata = {}
|
||||
|
||||
logger.info(f'unstruct_return code={resp.get("status_code")}')
|
||||
|
||||
if resp.get("b64_pdf"):
|
||||
with open(self.file_path, "wb") as f:
|
||||
f.write(base64.b64decode(resp["b64_pdf"]))
|
||||
|
||||
metadata["source"] = self.file_name
|
||||
doc = Document(page_content=content, metadata=metadata)
|
||||
return [doc]
|
||||
|
||||
async def aload(self) -> List[Document]:
|
||||
"""Asynchronously load given path as pages."""
|
||||
async with aiofiles.open(self.file_path, "rb") as f:
|
||||
file_data = await f.read()
|
||||
b64_data = base64.b64encode(file_data).decode()
|
||||
parameters = {"start": self.start, "n": self.n}
|
||||
parameters.update(self.extra_kwargs)
|
||||
# TODO: add filter_page_header_footer into payload when elt4llm is ready.
|
||||
payload = dict(
|
||||
filename=os.path.basename(self.file_name),
|
||||
b64_data=[b64_data],
|
||||
mode="partition",
|
||||
force_ocr=self.force_ocr,
|
||||
enable_formula=self.enable_formular,
|
||||
ocr_sdk_url=self.ocr_sdk_url,
|
||||
parameters=parameters,
|
||||
)
|
||||
try:
|
||||
|
||||
http_client = await get_http_client()
|
||||
|
||||
resp = await http_client.post(
|
||||
url=self.unstructured_api_url, headers=self.headers, body=payload,
|
||||
timeout=ClientTimeout(total=self.timemout)
|
||||
)
|
||||
except Exception as e:
|
||||
if str(e).find("Timeout") != -1:
|
||||
logger.error(f"Request to etl4lm API timed out: {e}")
|
||||
raise Exception("etl4lm server timeout")
|
||||
raise e
|
||||
if (resp.status_code != 200) or (resp.body and resp.body.get("status_code") != 200):
|
||||
logger.info(
|
||||
f"file partition {os.path.basename(self.file_name)} error resp={resp}"
|
||||
)
|
||||
raise Exception(
|
||||
f"file partition error {os.path.basename(self.file_name)} error resp={resp}"
|
||||
)
|
||||
|
||||
partitions = resp.body.get("partitions")
|
||||
if partitions:
|
||||
logger.info(f"content_from_partitions")
|
||||
self.partitions = partitions
|
||||
content, metadata = merge_partitions(
|
||||
self.file_path, partitions, self.knowledge_id
|
||||
)
|
||||
elif resp.body.get("text"):
|
||||
logger.info(f"content_from_text")
|
||||
content = resp.body["text"]
|
||||
metadata = {
|
||||
"bboxes": [],
|
||||
"pages": [],
|
||||
"indexes": [],
|
||||
"types": [],
|
||||
}
|
||||
else:
|
||||
logger.warning(f"content_is_empty resp={resp.body}")
|
||||
content = ""
|
||||
metadata = {}
|
||||
|
||||
logger.info(f'unstruct_return code={resp.body.get("status_code")}')
|
||||
|
||||
if resp.body.get("b64_pdf"):
|
||||
with open(self.file_path, "wb") as f:
|
||||
f.write(base64.b64decode(resp.body["b64_pdf"]))
|
||||
|
||||
metadata["source"] = self.file_name
|
||||
doc = Document(page_content=content, metadata=metadata)
|
||||
return [doc]
|
||||
@@ -0,0 +1,355 @@
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from bisheng_ragas import evaluate
|
||||
from bisheng_ragas.llms.langchain import LangchainLLM
|
||||
from bisheng_ragas.metrics import AnswerCorrectnessBisheng
|
||||
from datasets import Dataset
|
||||
from fastapi import UploadFile, HTTPException
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.assistant_agent import AssistantAgent
|
||||
from bisheng.api.v1.schema.workflow import WorkflowEventType
|
||||
from bisheng.api.v1.schemas import (UnifiedResponseModel, resp_200)
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.core.cache import InMemoryCache
|
||||
from bisheng.core.cache.redis_manager import get_redis_client_sync
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage_sync
|
||||
from bisheng.database.models.assistant import AssistantDao
|
||||
from bisheng.database.models.evaluation import (Evaluation, EvaluationDao, ExecType, EvaluationTaskStatus)
|
||||
from bisheng.database.models.flow import FlowDao
|
||||
from bisheng.database.models.flow_version import FlowVersionDao, FlowVersion
|
||||
from bisheng.llm.domain.services import LLMService
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
from bisheng.utils import generate_uuid
|
||||
from bisheng.worker.workflow.redis_callback import RedisCallback
|
||||
from bisheng.worker.workflow.tasks import execute_workflow, continue_workflow, workflow_stateful_worker
|
||||
from bisheng.workflow.common.workflow import WorkflowStatus
|
||||
|
||||
expire = 600
|
||||
|
||||
|
||||
class EvaluationService:
|
||||
UserCache: InMemoryCache = InMemoryCache()
|
||||
|
||||
@classmethod
|
||||
def get_evaluation(cls,
|
||||
user: UserPayload,
|
||||
page: int = 1,
|
||||
limit: int = 20) -> UnifiedResponseModel[List[Evaluation]]:
|
||||
"""
|
||||
Get a list of assessment tasks
|
||||
"""
|
||||
data = []
|
||||
res_evaluations, total = EvaluationDao.get_my_evaluations(user.user_id, page, limit)
|
||||
|
||||
# SkillIDVertical
|
||||
flow_ids = []
|
||||
# assistantIDVertical
|
||||
assistant_ids = []
|
||||
# VersionIDVertical
|
||||
flow_version_ids = []
|
||||
|
||||
for one in res_evaluations:
|
||||
if one.exec_type in [ExecType.FLOW.value, ExecType.WORKFLOW.value]:
|
||||
flow_ids.append(one.unique_id)
|
||||
if one.version:
|
||||
flow_version_ids.append(one.version)
|
||||
if one.exec_type == ExecType.ASSISTANT.value:
|
||||
assistant_ids.append(one.unique_id)
|
||||
|
||||
flow_names = {}
|
||||
flow_versions = {}
|
||||
assistant_names = {}
|
||||
|
||||
if flow_ids:
|
||||
flows = FlowDao.get_flow_by_ids(flow_ids=flow_ids)
|
||||
flow_names = {str(one.id): one.name for one in flows}
|
||||
|
||||
if flow_version_ids:
|
||||
versions = FlowVersionDao.get_list_by_ids(ids=flow_version_ids)
|
||||
flow_versions = {one.id: one.name for one in versions}
|
||||
|
||||
if assistant_ids:
|
||||
assistants = AssistantDao.get_assistants_by_ids(assistant_ids=assistant_ids)
|
||||
assistant_names = {str(one.id): one.name for one in assistants}
|
||||
|
||||
redis_client = get_redis_client_sync()
|
||||
|
||||
for one in res_evaluations:
|
||||
evaluation_item = jsonable_encoder(one)
|
||||
if one.exec_type in [ExecType.FLOW.value, ExecType.WORKFLOW.value]:
|
||||
evaluation_item['unique_name'] = flow_names.get(one.unique_id)
|
||||
if one.exec_type == ExecType.ASSISTANT.value:
|
||||
evaluation_item['unique_name'] = assistant_names.get(one.unique_id)
|
||||
if one.version:
|
||||
evaluation_item['version_name'] = flow_versions.get(one.version)
|
||||
if one.result_score:
|
||||
evaluation_item['result_score'] = json.loads(one.result_score) if isinstance(one.result_score,
|
||||
str) else one.result_score
|
||||
|
||||
# Processing Task Progress
|
||||
if one.status != EvaluationTaskStatus.running.value:
|
||||
evaluation_item['progress'] = f'100%'
|
||||
elif redis_client.exists(EvaluationService.get_redis_key(one.id)):
|
||||
evaluation_item['progress'] = f'{redis_client.get(EvaluationService.get_redis_key(one.id))}%'
|
||||
else:
|
||||
evaluation_item['progress'] = f'0%'
|
||||
|
||||
# Make sure the error description is returned to the front-end
|
||||
evaluation_item['description'] = one.description or ''
|
||||
evaluation_item['user_name'] = cls.get_user_name(one.user_id)
|
||||
data.append(evaluation_item)
|
||||
|
||||
return resp_200(data={'data': data, 'total': total})
|
||||
|
||||
@classmethod
|
||||
def delete_evaluation(cls, evaluation_id: int, user_payload: UserPayload) -> UnifiedResponseModel:
|
||||
evaluation = EvaluationDao.get_user_one_evaluation(user_payload.user_id, evaluation_id)
|
||||
if not evaluation:
|
||||
raise HTTPException(status_code=404, detail='Evaluation not found')
|
||||
|
||||
EvaluationDao.delete_evaluation(evaluation)
|
||||
return resp_200()
|
||||
|
||||
@classmethod
|
||||
def get_user_name(cls, user_id: int):
|
||||
if not user_id:
|
||||
return 'system'
|
||||
user = cls.UserCache.get(user_id)
|
||||
if user:
|
||||
return user.user_name
|
||||
user = UserDao.get_user(user_id)
|
||||
if not user:
|
||||
return f'{user_id}'
|
||||
cls.UserCache.set(user_id, user)
|
||||
return user.user_name
|
||||
|
||||
@classmethod
|
||||
def upload_file(cls, file: UploadFile):
|
||||
minio_client = get_minio_storage_sync()
|
||||
file_id = generate_uuid()
|
||||
file_name = file.filename
|
||||
|
||||
file_ext = os.path.basename(file.filename).split('.')[-1]
|
||||
file_path = f'evaluation/dataset/{file_id}.{file_ext}'
|
||||
minio_client.put_object_sync(bucket_name=minio_client.bucket, object_name=file_path, file=file.file,
|
||||
content_type=file.content_type)
|
||||
return file_name, file_path
|
||||
|
||||
@classmethod
|
||||
def upload_result_file(cls, df: pd.DataFrame):
|
||||
minio_client = get_minio_storage_sync()
|
||||
file_id = generate_uuid()
|
||||
|
||||
csv_buffer = io.BytesIO()
|
||||
df.to_csv(csv_buffer, index=False)
|
||||
csv_buffer.seek(0)
|
||||
|
||||
file_path = f'evaluation/result/{file_id}.csv'
|
||||
minio_client.put_object_sync(
|
||||
bucket_name=minio_client.bucket,
|
||||
object_name=file_path,
|
||||
file=csv_buffer.read(),
|
||||
content_type='application/csv')
|
||||
return file_path
|
||||
|
||||
@classmethod
|
||||
def read_csv_file(cls, file_path: str):
|
||||
minio_client = get_minio_storage_sync()
|
||||
resp = minio_client.get_object_sync(bucket_name=minio_client.bucket, object_name=file_path)
|
||||
if resp is None:
|
||||
return None
|
||||
return BytesIO(resp)
|
||||
|
||||
@classmethod
|
||||
def parse_csv(cls, file_data: io.BytesIO):
|
||||
df = pd.read_csv(file_data)
|
||||
df = df.dropna(axis=0, how='all').dropna(axis=1, how='all')
|
||||
if df.shape[1] < 2:
|
||||
raise ValueError("CSV file must have at least two columns")
|
||||
if df.columns[0] != 'question' or df.columns[1] != 'ground_truth':
|
||||
raise ValueError(
|
||||
"CSV file must have 'question' as the first column and 'ground_truth' as the second column")
|
||||
formatted_data = [{"question": row[0], "ground_truth": row[1]} for row in df.values]
|
||||
return formatted_data
|
||||
|
||||
@classmethod
|
||||
def get_redis_key(cls, evaluation_id: int):
|
||||
return f'evaluation_task_progress_{evaluation_id}'
|
||||
|
||||
|
||||
def execute_workflow_get_answer(workflow_info: FlowVersion, evaluation: Evaluation, question: str) -> str:
|
||||
# Initialize workflow
|
||||
unique_id = generate_uuid()
|
||||
workflow_id = evaluation.unique_id
|
||||
chat_id = ""
|
||||
user_id = evaluation.user_id
|
||||
workflow = RedisCallback(unique_id, workflow_id, chat_id, user_id)
|
||||
workflow.set_workflow_data(workflow_info.data)
|
||||
workflow.set_workflow_status(WorkflowStatus.WAITING.value)
|
||||
hash_key = generate_uuid()
|
||||
worker_node = workflow_stateful_worker.find_task_node_sync(hash_key)
|
||||
|
||||
execute_workflow.apply_async([unique_id, workflow_id, chat_id, user_id], queue=worker_node)
|
||||
|
||||
# Listen for execution results of workflows
|
||||
input_event = None
|
||||
for event in workflow.sync_get_response_until_break():
|
||||
input_event = event
|
||||
|
||||
status_info = workflow.get_workflow_status()
|
||||
if status_info["status"] == WorkflowStatus.FAILED.value:
|
||||
raise Exception(status_info.get("reason", "workflow run failed"))
|
||||
elif status_info['status'] == WorkflowStatus.SUCCESS.value:
|
||||
raise Exception("Only Q&A type workflows are currently supported")
|
||||
elif status_info['status'] == WorkflowStatus.INPUT.value:
|
||||
if not input_event or input_event.message.get('input_schema', {}).get("tab") == "form_input":
|
||||
raise Exception("Only Q&A type workflows are currently supported")
|
||||
# Only workflows entered in dialog boxes are entered by default
|
||||
workflow.set_user_input({input_event.message.get('node_id'): {"user_input": question}})
|
||||
workflow.set_workflow_status(WorkflowStatus.INPUT_OVER.value)
|
||||
worker_node = workflow_stateful_worker.find_task_node_sync(hash_key)
|
||||
continue_workflow.apply_async([unique_id, workflow_id, chat_id, user_id], queue=worker_node)
|
||||
events = []
|
||||
for event in workflow.sync_get_response_until_break():
|
||||
events.append(event)
|
||||
status_info = workflow.get_workflow_status()
|
||||
if status_info['status'] == WorkflowStatus.FAILED.value:
|
||||
raise Exception(status_info.get("reason", "workflow run failed"))
|
||||
elif status_info['status'] in [WorkflowStatus.SUCCESS.value, WorkflowStatus.INPUT.value]:
|
||||
workflow.set_workflow_stop()
|
||||
# Get the content of the first output event as an answer, if not, report an error
|
||||
if not events:
|
||||
raise Exception("Only Q&A type workflows are currently supported")
|
||||
answer = None
|
||||
for event in events:
|
||||
if event.category in [WorkflowEventType.OutputMsg.value, WorkflowEventType.OutputWithInput.value,
|
||||
WorkflowEventType.OutputWithChoose.value]:
|
||||
answer = event.message.get('msg', "")
|
||||
break
|
||||
elif event.category == WorkflowEventType.StreamMsg.value and event.type != 'stream':
|
||||
answer = event.message.get('msg', "")
|
||||
break
|
||||
if answer is None:
|
||||
raise Exception("Only Q&A type workflows are currently supported")
|
||||
return answer
|
||||
else:
|
||||
workflow.set_workflow_stop()
|
||||
raise Exception(f"workflow status is unknown: {status_info}")
|
||||
else:
|
||||
raise Exception(f"workflow status is unknown: {status_info}")
|
||||
|
||||
|
||||
async def add_evaluation_task(evaluation_id: int):
|
||||
evaluation = EvaluationDao.get_one_evaluation(evaluation_id=evaluation_id)
|
||||
if not evaluation:
|
||||
return
|
||||
|
||||
redis_key = EvaluationService.get_redis_key(evaluation_id)
|
||||
redis_client = get_redis_client_sync()
|
||||
try:
|
||||
file_data = EvaluationService.read_csv_file(evaluation.file_path)
|
||||
csv_data = EvaluationService.parse_csv(file_data)
|
||||
progress_increment = 80 / len(csv_data)
|
||||
current_progress = 0
|
||||
|
||||
if evaluation.exec_type == ExecType.FLOW.value:
|
||||
raise ValueError("unsupport flow")
|
||||
|
||||
elif evaluation.exec_type == ExecType.ASSISTANT.value:
|
||||
assistant = await AssistantDao.aget_one_assistant(evaluation.unique_id)
|
||||
if not assistant:
|
||||
raise Exception("Assistant not found")
|
||||
gpts_agent = AssistantAgent(assistant_info=assistant, chat_id="", invoke_user_id=evaluation.user_id)
|
||||
await gpts_agent.init_assistant()
|
||||
for index, one in enumerate(csv_data):
|
||||
messages = await gpts_agent.run(one.get('question'))
|
||||
if len(messages):
|
||||
one["answer"] = messages[-1].content
|
||||
current_progress += progress_increment
|
||||
redis_client.set(redis_key, round(current_progress))
|
||||
elif evaluation.exec_type == ExecType.WORKFLOW.value:
|
||||
workflow_info = FlowVersionDao.get_version_by_id(version_id=evaluation.version)
|
||||
if not workflow_info or workflow_info.flow_id != evaluation.unique_id:
|
||||
raise Exception("workflow version info not found")
|
||||
for index, one in enumerate(csv_data):
|
||||
one["answer"] = await asyncio.to_thread(execute_workflow_get_answer, workflow_info, evaluation,
|
||||
one.get('question', ""))
|
||||
|
||||
_llm = await LLMService.get_evaluation_llm_object(evaluation.user_id)
|
||||
llm = LangchainLLM(_llm)
|
||||
data_samples = {
|
||||
"question": [one.get('question') for one in csv_data],
|
||||
"answer": [one.get('answer') for one in csv_data],
|
||||
"ground_truths": [[one.get('ground_truth')] for one in csv_data]
|
||||
}
|
||||
|
||||
dataset = Dataset.from_dict(data_samples)
|
||||
answer_correctness_bisheng = AnswerCorrectnessBisheng(llm=llm, human_prompt=evaluation.prompt)
|
||||
score = await asyncio.to_thread(evaluate, dataset, [answer_correctness_bisheng])
|
||||
df = score.to_pandas()
|
||||
result = df.to_dict(orient="list")
|
||||
logger.debug(f'evaluation id = {evaluation_id} result: {result}')
|
||||
|
||||
question = result.get('question', [])
|
||||
columns = [
|
||||
# Data field:Title:Type(1:Text 2:Numbers 3:%)
|
||||
("question", "question", 1),
|
||||
("ground_truths", "ground_truth", 1),
|
||||
("answer", "answer", 1),
|
||||
("statements_num_gt_only", "statements_num_gt_only", 2),
|
||||
("statements_num_answer_only", "statements_num_answer_only", 2),
|
||||
("statements_num_overlap", "statements_num_overlap", 2),
|
||||
("answer_recall", "recall", 3),
|
||||
("answer_precision", "precision", 3),
|
||||
("answer_f1", "F1", 3)
|
||||
]
|
||||
row_list = []
|
||||
tmp_dict = defaultdict(int)
|
||||
total_dict = {}
|
||||
|
||||
for index, one in enumerate(question):
|
||||
row_data = {}
|
||||
for field, title, unit_type in columns:
|
||||
value = result.get(field)[index]
|
||||
if unit_type != 1:
|
||||
tmp_dict[field] += value
|
||||
if unit_type == 3:
|
||||
value = f'{value * 100:.2f}%' if value not in ["nan", np.nan] else value
|
||||
row_data[title] = value
|
||||
row_list.append(row_data)
|
||||
|
||||
total_row_data = {}
|
||||
for field, title, unit_type in columns:
|
||||
value = tmp_dict.get(field)
|
||||
if unit_type == 3:
|
||||
value = f'{(value / len(row_list)) * 100:.2f}%'
|
||||
total_dict[field] = value
|
||||
total_row_data[title] = value
|
||||
row_list.append(total_row_data)
|
||||
|
||||
df = pd.DataFrame(data=row_list, columns=[one[1] for one in columns])
|
||||
result_file_path = EvaluationService.upload_result_file(df)
|
||||
|
||||
evaluation.result_score = total_dict
|
||||
evaluation.status = EvaluationTaskStatus.success.value
|
||||
evaluation.result_file_path = result_file_path
|
||||
EvaluationDao.update_evaluation(evaluation=evaluation)
|
||||
redis_client.delete(redis_key)
|
||||
logger.info(f'evaluation task success id={evaluation_id}')
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f'evaluation task failed id={evaluation_id} {str(e)}')
|
||||
evaluation.status = EvaluationTaskStatus.failed.value
|
||||
evaluation.description = str(e)[-500:] # Limit the length of the error description to avoid being too long
|
||||
EvaluationDao.update_evaluation(evaluation=evaluation)
|
||||
redis_client.delete(redis_key)
|
||||
@@ -0,0 +1,324 @@
|
||||
import asyncio
|
||||
import copy
|
||||
from typing import List, Dict, AsyncGenerator, Union
|
||||
|
||||
from fastapi import Request
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.audit_log import AuditLogService
|
||||
from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200, FlowVersionCreate, FlowCompareReq, resp_500, \
|
||||
StreamData
|
||||
from bisheng.common.chat.utils import process_node_data
|
||||
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.flow import NotFoundVersionError, CurVersionDelError, VersionNameExistsError, \
|
||||
WorkFlowOnlineEditError
|
||||
from bisheng.common.errcode.http_error import NotFoundError, UnAuthorizedError
|
||||
from bisheng.common.services import telemetry_service
|
||||
from bisheng.common.services.base import BaseService
|
||||
from bisheng.core.logger import trace_id_var
|
||||
from bisheng.database.models.flow import FlowDao, FlowStatus, Flow, FlowType
|
||||
from bisheng.database.models.flow_version import FlowVersionDao, FlowVersionRead, FlowVersion
|
||||
from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum, GroupResource
|
||||
from bisheng.database.models.role_access import AccessType
|
||||
from bisheng.database.models.session import MessageSessionDao
|
||||
from bisheng.database.models.user_group import UserGroupDao
|
||||
from bisheng.share_link.domain.models.share_link import ShareLink
|
||||
from bisheng.utils import get_request_ip
|
||||
|
||||
|
||||
class FlowService(BaseService):
|
||||
|
||||
@classmethod
|
||||
def get_version_list_by_flow(cls, user: UserPayload, flow_id: str) -> UnifiedResponseModel[List[FlowVersionRead]]:
|
||||
"""
|
||||
By SkillID Get all versions of a skill
|
||||
"""
|
||||
data = FlowVersionDao.get_list_by_flow(flow_id)
|
||||
# Include Deleted Versions
|
||||
all_version_num = FlowVersionDao.count_list_by_flow(flow_id, include_delete=True)
|
||||
return resp_200(data={
|
||||
'data': data,
|
||||
'total': all_version_num
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def get_version_info(cls, user: UserPayload, version_id: int) -> UnifiedResponseModel[FlowVersion]:
|
||||
"""
|
||||
According to versionIDGet version details
|
||||
"""
|
||||
data = FlowVersionDao.get_version_by_id(version_id)
|
||||
return resp_200(data=data)
|
||||
|
||||
@classmethod
|
||||
def delete_version(cls, user: UserPayload, version_id: int) -> UnifiedResponseModel[None]:
|
||||
"""
|
||||
According to versionIDRemove Version
|
||||
"""
|
||||
telemetry_service.log_event_sync(
|
||||
user_id=user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
version_info = FlowVersionDao.get_version_by_id(version_id)
|
||||
if not version_info:
|
||||
return NotFoundVersionError.return_resp()
|
||||
|
||||
flow_info = FlowDao.get_flow_by_id(version_info.flow_id)
|
||||
if not flow_info or flow_info.flow_type != FlowType.WORKFLOW.value:
|
||||
return NotFoundError.return_resp()
|
||||
|
||||
# Determine permissions
|
||||
if not user.access_check(flow_info.user_id, flow_info.id, AccessType.WORKFLOW_WRITE):
|
||||
return UnAuthorizedError.return_resp()
|
||||
|
||||
if version_info.is_current == 1:
|
||||
return CurVersionDelError.return_resp()
|
||||
|
||||
FlowVersionDao.delete_flow_version(version_id)
|
||||
return resp_200()
|
||||
|
||||
@classmethod
|
||||
async def judge_flow_write_permission(cls, user: UserPayload, flow_id: str) -> Flow:
|
||||
flow_info = await FlowDao.aget_flow_by_id(flow_id)
|
||||
if not flow_info or flow_info.flow_type != FlowType.WORKFLOW.value:
|
||||
raise NotFoundError.http_exception()
|
||||
|
||||
# Determine permissions
|
||||
if not await user.async_access_check(flow_info.user_id, flow_info.id, AccessType.WORKFLOW_WRITE):
|
||||
raise UnAuthorizedError.http_exception()
|
||||
return flow_info
|
||||
|
||||
@classmethod
|
||||
async def change_current_version(cls, request: Request, login_user: UserPayload, flow_id: str, version_id: int) \
|
||||
-> UnifiedResponseModel[None]:
|
||||
"""
|
||||
Modify Current Version
|
||||
"""
|
||||
await telemetry_service.log_event(
|
||||
user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
flow_info = await cls.judge_flow_write_permission(login_user, flow_id)
|
||||
|
||||
if flow_info.status == FlowStatus.ONLINE.value:
|
||||
return WorkFlowOnlineEditError.return_resp()
|
||||
|
||||
# Switch versions
|
||||
version_info = await FlowVersionDao.aget_version_by_id(version_id)
|
||||
if not version_info:
|
||||
return NotFoundVersionError.return_resp()
|
||||
if version_info.is_current == 1:
|
||||
return resp_200()
|
||||
|
||||
# Modify the version selected by the user for the current version
|
||||
await FlowVersionDao.change_current_version(flow_id, version_info)
|
||||
|
||||
await cls.update_flow_hook(request, login_user, flow_info)
|
||||
return resp_200()
|
||||
|
||||
@classmethod
|
||||
async def create_new_version(cls, user: UserPayload, flow_id: str, flow_version: FlowVersionCreate) \
|
||||
-> UnifiedResponseModel[FlowVersion]:
|
||||
"""
|
||||
Create New Version
|
||||
"""
|
||||
await telemetry_service.log_event(
|
||||
user_id=user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
await cls.judge_flow_write_permission(user, flow_id)
|
||||
|
||||
exist_version = FlowVersionDao.get_version_by_name(flow_id, flow_version.name)
|
||||
if exist_version:
|
||||
return VersionNameExistsError.return_resp()
|
||||
|
||||
flow_version = FlowVersion(flow_id=flow_id, name=flow_version.name, description=flow_version.description,
|
||||
user_id=user.user_id, data=flow_version.data,
|
||||
original_version_id=flow_version.original_version_id,
|
||||
flow_type=flow_version.flow_type)
|
||||
|
||||
# Create New Version
|
||||
flow_version = FlowVersionDao.create_version(flow_version)
|
||||
|
||||
return resp_200(data=flow_version)
|
||||
|
||||
@classmethod
|
||||
async def update_version_info(cls, request: Request, user: UserPayload, version_id: int,
|
||||
flow_version: FlowVersionCreate) \
|
||||
-> UnifiedResponseModel[FlowVersion]:
|
||||
"""
|
||||
It updates version information.
|
||||
"""
|
||||
await telemetry_service.log_event(
|
||||
user_id=user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
# Contains the deleted version. If the version is deleted, revert to this version
|
||||
version_info = await FlowVersionDao.aget_version_by_id(version_id, include_delete=True)
|
||||
if not version_info:
|
||||
return NotFoundVersionError.return_resp()
|
||||
flow_info = await cls.judge_flow_write_permission(user, version_info.flow_id)
|
||||
|
||||
if version_info.is_current == 1 and flow_info.status == FlowStatus.ONLINE.value and flow_version.data:
|
||||
return WorkFlowOnlineEditError.return_resp()
|
||||
|
||||
version_info.name = flow_version.name if flow_version.name else version_info.name
|
||||
version_info.description = flow_version.description if flow_version.description else version_info.description
|
||||
version_info.data = flow_version.data if flow_version.data else version_info.data
|
||||
version_info.is_delete = 0
|
||||
|
||||
flow_version = await FlowVersionDao.aupdate_version(version_info)
|
||||
|
||||
await cls.update_flow_hook(request, user, flow_info)
|
||||
return resp_200(data=flow_version)
|
||||
|
||||
@classmethod
|
||||
async def get_one_flow(cls, login_user: UserPayload, flow_id: str, share_link: Union['ShareLink', None] = None) -> \
|
||||
UnifiedResponseModel[Flow]:
|
||||
flow_info = await FlowDao.aget_flow_by_id(flow_id)
|
||||
if not flow_info or flow_info.flow_type != FlowType.WORKFLOW.value:
|
||||
raise NotFoundError()
|
||||
if not await login_user.async_access_check(flow_info.user_id, flow_info.id, AccessType.WORKFLOW):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
flow_info.logo = await cls.get_logo_share_link_async(flow_info.logo)
|
||||
|
||||
return resp_200(data=flow_info)
|
||||
|
||||
@classmethod
|
||||
async def get_compare_tasks(cls, user: UserPayload, req: FlowCompareReq) -> List:
|
||||
"""
|
||||
Get Comparison Tasks
|
||||
"""
|
||||
if req.question_list is None or len(req.question_list) == 0:
|
||||
return []
|
||||
if req.version_list is None or len(req.version_list) == 0:
|
||||
return []
|
||||
if req.node_id is None:
|
||||
return []
|
||||
|
||||
# Get version data
|
||||
version_infos = FlowVersionDao.get_list_by_ids(req.version_list)
|
||||
# Start a new event loop
|
||||
tasks = []
|
||||
for index, question in enumerate(req.question_list):
|
||||
question_index = index
|
||||
tmp_inputs = copy.deepcopy(req.inputs)
|
||||
tmp_inputs, tmp_tweaks = cls.parse_compare_inputs(tmp_inputs, question)
|
||||
for version in version_infos:
|
||||
task = asyncio.create_task(cls.exec_flow_node(
|
||||
copy.deepcopy(tmp_inputs), tmp_tweaks, question_index, [version]))
|
||||
tasks.append(task)
|
||||
return tasks
|
||||
|
||||
@classmethod
|
||||
def parse_compare_inputs(cls, inputs: Dict, question) -> (Dict, Dict):
|
||||
# Under special treatmentinputs, Hold and PasswebsocketSessions are formatted consistently
|
||||
if inputs.get('data', None):
|
||||
for one in inputs['data']:
|
||||
one['id'] = one['nodeId']
|
||||
if 'InputFile' in one['id']:
|
||||
one['file_path'] = one['value']
|
||||
|
||||
# Paddingquestion and Generate Replacementtweaks
|
||||
for key, val in inputs.items():
|
||||
if key != 'data' and key != 'id':
|
||||
# Default inputkey, replace the firstkey
|
||||
logger.info(f"replace_inputs {key} replace to {question}")
|
||||
inputs[key] = question
|
||||
break
|
||||
if 'id' in inputs:
|
||||
inputs.pop('id')
|
||||
# Replacement Node Parameters, GantiinputFileNodeAndVariableNodeParameters
|
||||
tweaks = {}
|
||||
if 'data' in inputs:
|
||||
node_data = inputs.pop('data')
|
||||
if node_data:
|
||||
tweaks = process_node_data(node_data)
|
||||
return inputs, tweaks
|
||||
|
||||
@classmethod
|
||||
async def compare_flow_node(cls, user: UserPayload, req: FlowCompareReq) -> UnifiedResponseModel[Dict]:
|
||||
"""
|
||||
Compare nodes in two versions Output Results
|
||||
"""
|
||||
tasks = await cls.get_compare_tasks(user, req)
|
||||
if len(tasks) == 0:
|
||||
return resp_200(data=[])
|
||||
res = [{} for _ in range(len(req.question_list))]
|
||||
try:
|
||||
for one in asyncio.as_completed(tasks):
|
||||
index, answer = await one
|
||||
if res[index]:
|
||||
res[index].update(answer)
|
||||
else:
|
||||
res[index] = answer
|
||||
except Exception as e:
|
||||
return resp_500(message="Workflow comparison error:{}".format(str(e)))
|
||||
return resp_200(data=res)
|
||||
|
||||
@classmethod
|
||||
async def compare_flow_stream(cls, user: UserPayload, req: FlowCompareReq) -> AsyncGenerator:
|
||||
"""
|
||||
Compare nodes in two versions Output Results
|
||||
"""
|
||||
tasks = await cls.get_compare_tasks(user, req)
|
||||
if len(tasks) == 0:
|
||||
return
|
||||
for one in asyncio.as_completed(tasks):
|
||||
index, answer_dict = await one
|
||||
for version_id, answer in answer_dict.items():
|
||||
yield str(StreamData(event='message',
|
||||
data={'question_index': index,
|
||||
'version_id': version_id,
|
||||
'answer': answer}))
|
||||
|
||||
@classmethod
|
||||
async def exec_flow_node(cls, inputs: Dict, tweaks: Dict, index: int, versions: List[FlowVersion]):
|
||||
# Gantianswer
|
||||
raise ValueError("flow is not supported")
|
||||
|
||||
@classmethod
|
||||
def create_flow_hook(cls, request: Request, login_user: UserPayload, flow_info: Flow) -> bool:
|
||||
logger.info(f'create_flow_hook flow: {flow_info.id}, user_payload: {login_user.user_id}')
|
||||
user_group = UserGroupDao.get_user_group(login_user.user_id)
|
||||
if user_group:
|
||||
batch_resource = []
|
||||
for one in user_group:
|
||||
batch_resource.append(
|
||||
GroupResource(group_id=one.group_id,
|
||||
third_id=flow_info.id,
|
||||
type=ResourceTypeEnum.WORK_FLOW.value))
|
||||
GroupResourceDao.insert_group_batch(batch_resource)
|
||||
AuditLogService.create_build_workflow(login_user, get_request_ip(request), flow_info.id)
|
||||
|
||||
cls.get_logo_share_link(flow_info.logo)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
async def update_flow_hook(cls, request: Request, login_user: UserPayload, flow_info: Flow) -> bool:
|
||||
# Write Audit Log
|
||||
await AuditLogService.update_build_workflow(login_user, get_request_ip(request), flow_info.id)
|
||||
|
||||
# WritelogoCeacle
|
||||
await cls.get_logo_share_link_async(flow_info.logo)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def delete_flow_hook(cls, request: Request, login_user: UserPayload, flow_info: Flow) -> bool:
|
||||
logger.info(f'delete_flow_hook flow: {flow_info.id}, user_payload: {login_user.user_id}')
|
||||
|
||||
# Write Audit Log
|
||||
AuditLogService.delete_build_workflow(login_user, get_request_ip(request), flow_info)
|
||||
|
||||
# Delete Skills Associated Under User Group
|
||||
GroupResourceDao.delete_group_resource_by_third_id(flow_info.id, ResourceTypeEnum.WORK_FLOW)
|
||||
|
||||
# Update session information
|
||||
MessageSessionDao.update_session_info_by_flow(flow_info.name, flow_info.description, flow_info.logo,
|
||||
flow_info.id, flow_info.flow_type)
|
||||
return True
|
||||
@@ -0,0 +1,52 @@
|
||||
import random
|
||||
import string
|
||||
|
||||
|
||||
class VoucherGenerator:
|
||||
def __init__(self, length=10):
|
||||
self.length = length
|
||||
# Exclude similar letters and numbers: 'I', 'l', 'O', '0', '1'
|
||||
self.characters = ''.join(set(string.ascii_letters + string.digits) - set('IlOo01'))
|
||||
self.weights = [7, 9, 10, 5, 8, 4, 2, 1, 3] # Weighting Factor
|
||||
self.check_digits = ['1', '0', 'X', '9', '8', '7', '6', '5', '4', '3', '2'] # Checksum Correspondence Form
|
||||
|
||||
def generate_voucher(self):
|
||||
voucher_base = ''.join(random.choices(self.characters, k=self.length - 1))
|
||||
check_digit = self.calculate_check_digit(voucher_base)
|
||||
return voucher_base + check_digit
|
||||
|
||||
def calculate_check_digit(self, voucher_base):
|
||||
total = sum(self.weights[i] * (ord(char) - ord('A') if char.isalpha() else int(char)) for i, char in
|
||||
enumerate(voucher_base))
|
||||
remainder = total % 11
|
||||
return self.check_digits[remainder]
|
||||
|
||||
def validate_voucher(self, voucher):
|
||||
if len(voucher) != 10:
|
||||
return False, "Invalid voucher length"
|
||||
|
||||
voucher_base = voucher[:-1]
|
||||
provided_check_digit = voucher[-1]
|
||||
|
||||
calculated_check_digit = self.calculate_check_digit(voucher_base)
|
||||
|
||||
if provided_check_digit == calculated_check_digit:
|
||||
return True, "Valid voucher"
|
||||
else:
|
||||
return False, "Invalid voucher"
|
||||
|
||||
|
||||
# Example Usage
|
||||
if __name__ == "__main__":
|
||||
generator = VoucherGenerator()
|
||||
voucher_code = generator.generate_voucher() # Generate a unique redemption code
|
||||
print(f"Generated voucher code: {voucher_code}")
|
||||
|
||||
# Verify Redeem Code
|
||||
is_valid, info = generator.validate_voucher(voucher_code)
|
||||
print(f"Is valid: {is_valid}, Info: {info}")
|
||||
|
||||
# Try to validate an invalid redemption code
|
||||
invalid_voucher_code = 'ABCDEFGHJK967'
|
||||
is_valid, info = generator.validate_voucher(invalid_voucher_code)
|
||||
print(f"Is valid: {is_valid}, Info: {info}")
|
||||
@@ -0,0 +1,115 @@
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.invite_code.code_validator import VoucherGenerator
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.linsight import InviteCodeBindError, InviteCodeInvalidError
|
||||
from bisheng.database.models.invite_code import InviteCode, InviteCodeDao
|
||||
from bisheng.utils import generate_uuid
|
||||
|
||||
|
||||
class InviteCodeService:
|
||||
|
||||
@classmethod
|
||||
async def use_invite_code(cls, user_id: int) -> bool:
|
||||
"""
|
||||
using referral code
|
||||
:param user_id: UsersID
|
||||
:return: Invitation code results
|
||||
"""
|
||||
logger.debug(f"use_invite_code {user_id}")
|
||||
|
||||
codes = await InviteCodeDao.get_user_bind_code(user_id)
|
||||
for one in codes:
|
||||
flag = await InviteCodeDao.use_invite_code(user_id, one.code)
|
||||
if flag:
|
||||
logger.debug(f"use_invite_code {user_id}, {one.code} success")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def revoke_invite_code(cls, user_id: int) -> bool:
|
||||
"""
|
||||
Revoke Invitation Code
|
||||
:param user_id: UsersID
|
||||
:return: Invitation code revocation result
|
||||
"""
|
||||
logger.debug(f"revoke_invite_code {user_id}")
|
||||
|
||||
codes = await InviteCodeDao.get_user_all_code(user_id)
|
||||
for one in codes:
|
||||
# Description is a brand new invite code and has not been used
|
||||
if one.used <= 0:
|
||||
continue
|
||||
flag = await InviteCodeDao.revoke_invite_code_used(user_id, one.code)
|
||||
if flag:
|
||||
logger.debug(f"revoke_invite_code {user_id}, {one.code} success")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def create_batch_invite_codes(cls, login_user: UserPayload, name: str, num: int, limit: int) -> list[str]:
|
||||
"""
|
||||
Bulk create invite codes
|
||||
:param login_user: Action user information
|
||||
:param name: Invitation code name
|
||||
:param num: How many codes
|
||||
:param limit: Number of uses per invite code
|
||||
:return: Invitation code list created
|
||||
"""
|
||||
generator = VoucherGenerator()
|
||||
code_list = []
|
||||
batch_id = generate_uuid()
|
||||
for i in range(num):
|
||||
code_list.append(InviteCode(
|
||||
code=generator.generate_voucher(),
|
||||
batch_id=batch_id,
|
||||
batch_name=name,
|
||||
limit=limit,
|
||||
created_id=login_user.user_id,
|
||||
))
|
||||
# Check if the generated invite code is a duplicate
|
||||
unique_codes = []
|
||||
for code in code_list:
|
||||
if code.code in unique_codes:
|
||||
raise ValueError(f"Duplicate invite code found: {code.code}")
|
||||
unique_codes.append(code.code)
|
||||
|
||||
# Call the database operation to save the invite code
|
||||
await InviteCodeDao.insert_invite_code(code_list)
|
||||
return unique_codes
|
||||
|
||||
@classmethod
|
||||
async def get_invite_code_num(cls, login_user: UserPayload) -> int:
|
||||
"""
|
||||
Get the number of times a user can use an invite code
|
||||
:param login_user: Action user information
|
||||
:return: Invitation code usage
|
||||
"""
|
||||
nums = 0
|
||||
codes = await InviteCodeDao.get_user_bind_code(login_user.user_id)
|
||||
for one in codes:
|
||||
nums += one.limit - one.used
|
||||
return nums
|
||||
|
||||
@classmethod
|
||||
async def bind_invite_code(cls, login_user: UserPayload, code: str) -> bool:
|
||||
"""
|
||||
Binding Invitation Code
|
||||
:param login_user: Action user information
|
||||
:param code: Invitation Code
|
||||
:return: Binding Results
|
||||
"""
|
||||
generator = VoucherGenerator()
|
||||
flag, _ = generator.validate_voucher(code)
|
||||
if not flag:
|
||||
raise InviteCodeInvalidError()
|
||||
codes = await InviteCodeDao.get_user_bind_code(login_user.user_id)
|
||||
if codes:
|
||||
raise InviteCodeBindError()
|
||||
|
||||
flag = await InviteCodeDao.bind_invite_code(login_user.user_id, code)
|
||||
if not flag:
|
||||
raise InviteCodeInvalidError()
|
||||
return flag
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
import os
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from bisheng.core.cache.utils import CACHE_DIR
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage_sync
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_from_docx import handler as docx_handler
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_from_excel import handler as excel_handler
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_from_html import handler as html_handler
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_from_pdf import handler as pdf_handler
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_from_pptx import handler as pptx_handler
|
||||
from bisheng.knowledge.rag.pipeline.loader.utils.md_post_processing import post_processing
|
||||
|
||||
|
||||
def combine_multiple_md_files_to_raw_texts(
|
||||
path,
|
||||
) -> tuple[list[Document], list[Document]]:
|
||||
"""
|
||||
combine multiple md file to raw texts including meta-data list.
|
||||
Args:
|
||||
path: the directory containing the md files.
|
||||
Returns:
|
||||
0: split raw texts, each text is a Document object.
|
||||
1: a single Document object containing all the texts combined.
|
||||
"""
|
||||
|
||||
files = sorted([f for f in os.listdir(path)])
|
||||
raw_texts = []
|
||||
|
||||
# A file corresponds to only one complete Document Objects, texts It is only after cuttingchunkContents
|
||||
documents = [Document(page_content="", metadata={})]
|
||||
|
||||
for file_name in files:
|
||||
full_file_name = f"{path}/{file_name}"
|
||||
with open(full_file_name, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
raw_texts.append(Document(page_content=content, metadata={}))
|
||||
documents[0].page_content += content
|
||||
return raw_texts, documents
|
||||
|
||||
|
||||
def convert_file_to_md(
|
||||
file_name,
|
||||
input_file_name,
|
||||
header_rows=[0, 1],
|
||||
data_rows=10,
|
||||
append_header=True,
|
||||
knowledge_id=None,
|
||||
retain_images=True,
|
||||
):
|
||||
"""
|
||||
The main function that handles file conversions.
|
||||
Args:
|
||||
file_name:
|
||||
input_file_name:
|
||||
header_rows:
|
||||
data_rows:
|
||||
append_header:
|
||||
knowledge_id:
|
||||
"""
|
||||
file_name = file_name.lower()
|
||||
md_file_name = None
|
||||
local_image_dir = None
|
||||
include_cache_dir = True
|
||||
doc_id = None
|
||||
if file_name.endswith(".docx") or file_name.endswith(".doc"):
|
||||
md_file_name, local_image_dir, doc_id = docx_handler(CACHE_DIR, input_file_name)
|
||||
elif file_name.endswith(".pptx") or file_name.endswith(".ppt"):
|
||||
md_file_name, local_image_dir, doc_id = pptx_handler(CACHE_DIR, input_file_name)
|
||||
include_cache_dir = False
|
||||
elif (
|
||||
file_name.endswith(".xlsx")
|
||||
or file_name.endswith(".xls")
|
||||
or file_name.endswith(".csv")
|
||||
):
|
||||
md_file_name, local_image_dir, doc_id = excel_handler(
|
||||
CACHE_DIR, input_file_name, header_rows, data_rows, append_header
|
||||
)
|
||||
local_image_dir = None
|
||||
return md_file_name, local_image_dir, doc_id
|
||||
elif (
|
||||
file_name.endswith(".html")
|
||||
or file_name.endswith(".htm")
|
||||
or file_name.endswith(".mhtml")
|
||||
):
|
||||
(
|
||||
md_file_name,
|
||||
local_image_dir,
|
||||
doc_id,
|
||||
) = html_handler(CACHE_DIR, input_file_name)
|
||||
include_cache_dir = False
|
||||
elif file_name.endswith("pdf"):
|
||||
md_file_name, local_image_dir, doc_id = pdf_handler(CACHE_DIR, input_file_name)
|
||||
include_cache_dir = True
|
||||
else:
|
||||
raise ValueError(f"unsupported file type {file_name} for conversion to markdown.")
|
||||
|
||||
return replace_image_url(
|
||||
md_file_name,
|
||||
local_image_dir,
|
||||
doc_id,
|
||||
include_cache_dir,
|
||||
knowledge_id=knowledge_id,
|
||||
retain_images=retain_images,
|
||||
)
|
||||
|
||||
|
||||
def replace_image_url(
|
||||
md_file_name,
|
||||
local_image_dir,
|
||||
doc_id,
|
||||
include_cache_dir,
|
||||
knowledge_id=None,
|
||||
retain_images=True,
|
||||
):
|
||||
"""
|
||||
Usage:
|
||||
user the same bucket as origin file located.
|
||||
Args:
|
||||
md_file_name:
|
||||
local_image_dir:
|
||||
doc_id:
|
||||
knowledge_id:
|
||||
if the knowledge_id is None, this process will be interrupted,
|
||||
because the image files wouldn't be put into minio
|
||||
"""
|
||||
from bisheng.api.services.knowledge_imp import KnowledgeUtils
|
||||
|
||||
minio_image_path = f"/{get_minio_storage_sync().bucket}/{KnowledgeUtils.get_knowledge_file_image_dir(doc_id, knowledge_id)}"
|
||||
url_for_replacement = local_image_dir
|
||||
if not include_cache_dir:
|
||||
url_for_replacement = doc_id
|
||||
|
||||
if md_file_name and local_image_dir and doc_id:
|
||||
with open(md_file_name, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
content = content.replace(url_for_replacement, minio_image_path)
|
||||
|
||||
with open(md_file_name, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
post_processing(md_file_name, retain_images)
|
||||
return md_file_name, local_image_dir, doc_id
|
||||
@@ -0,0 +1,399 @@
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import List, Any, Dict, Optional
|
||||
|
||||
from fastapi import Request, HTTPException
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.assistant import AssistantService
|
||||
from bisheng.api.services.audit_log import AuditLogService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError
|
||||
from bisheng.common.errcode.user import UserGroupNotDeleteError, AdminUserUpdateForbiddenError
|
||||
from bisheng.core.cache.redis_manager import get_redis_client_sync
|
||||
from bisheng.database.constants import AdminRole
|
||||
from bisheng.database.models.assistant import AssistantDao
|
||||
from bisheng.database.models.flow import FlowDao, FlowType
|
||||
from bisheng.database.models.group import Group, GroupCreate, GroupDao, GroupRead, DefaultGroup
|
||||
from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum
|
||||
from bisheng.database.models.role import RoleDao
|
||||
from bisheng.database.models.user_group import UserGroupCreate, UserGroupDao, UserGroupRead
|
||||
from bisheng.knowledge.domain.models.knowledge import KnowledgeDao
|
||||
from bisheng.telemetry_search.domain.services.dashboard import DashboardService
|
||||
from bisheng.tool.domain.models.gpts_tools import GptsToolsDao
|
||||
from bisheng.user.domain.models.user import User, UserDao
|
||||
from bisheng.user.domain.models.user_role import UserRoleDao
|
||||
from bisheng.user.domain.services.user import UserService
|
||||
from bisheng.utils import get_request_ip
|
||||
|
||||
|
||||
class RoleGroupService():
|
||||
|
||||
def get_group_list(self, group_ids: List[int]) -> List[GroupRead]:
|
||||
"""Get the full amountgroupVertical"""
|
||||
|
||||
# Inquirygroup
|
||||
if group_ids:
|
||||
groups = GroupDao.get_group_by_ids(group_ids)
|
||||
else:
|
||||
groups = GroupDao.get_all_group()
|
||||
# Inquiryuser
|
||||
user_admin = UserGroupDao.get_groups_admins([group.id for group in groups])
|
||||
users_dict = {}
|
||||
if user_admin:
|
||||
user_ids = [user.user_id for user in user_admin]
|
||||
users = UserDao.get_user_by_ids(user_ids)
|
||||
users_dict = {user.user_id: user for user in users}
|
||||
|
||||
groupReads = [GroupRead.validate(group) for group in groups]
|
||||
for group in groupReads:
|
||||
group.group_admins = [
|
||||
self._dump_user_with_avatar_share_link(users_dict.get(user.user_id)) for user in user_admin
|
||||
if user.group_id == group.id
|
||||
]
|
||||
return groupReads
|
||||
|
||||
def create_group(self, request: Request, login_user: UserPayload, group: GroupCreate) -> Group:
|
||||
"""Add Usergroup"""
|
||||
group_admin = group.group_admins
|
||||
group.create_user = login_user.user_id
|
||||
group.update_user = login_user.user_id
|
||||
group = GroupDao.insert_group(group)
|
||||
if group_admin:
|
||||
logger.info('set_admin group_admins={} group_id={}', group_admin, group.id)
|
||||
self.set_group_admin(request, login_user, group_admin, group.id)
|
||||
self.create_group_hook(request, login_user, group)
|
||||
return group
|
||||
|
||||
def create_group_hook(self, request: Request, login_user: UserPayload, group: Group) -> bool:
|
||||
""" New User Group Post Action """
|
||||
logger.info(f'act=create_group_hook user={login_user.user_name} group_id={group.id}')
|
||||
# Log Audit Logs
|
||||
AuditLogService.create_user_group(login_user, get_request_ip(request), group)
|
||||
return True
|
||||
|
||||
def update_group(self, request: Request, login_user: UserPayload, group: Group) -> Group:
|
||||
"""Update User"""
|
||||
exist_group = GroupDao.get_user_group(group.id)
|
||||
if not exist_group:
|
||||
raise ValueError('User group does not exist')
|
||||
exist_group.group_name = group.group_name
|
||||
exist_group.remark = group.group_name
|
||||
exist_group.update_user = login_user.user_id
|
||||
exist_group.update_time = datetime.now()
|
||||
|
||||
group = GroupDao.update_group(exist_group)
|
||||
self.update_group_hook(request, login_user, group)
|
||||
return group
|
||||
|
||||
def update_group_hook(self, request: Request, login_user: UserPayload, group: Group):
|
||||
logger.info(f'act=update_group_hook user={login_user.user_name} group_id={group.id}')
|
||||
# Log Audit Logs
|
||||
AuditLogService.update_user_group(login_user, get_request_ip(request), group)
|
||||
|
||||
def delete_group(self, request: Request, login_user: UserPayload, group_id: int):
|
||||
"""Can delete existing usergroups"""
|
||||
if group_id == DefaultGroup:
|
||||
raise HTTPException(status_code=500, detail='Default group cannot be deleted')
|
||||
group_info = GroupDao.get_user_group(group_id)
|
||||
if not group_info:
|
||||
return resp_200()
|
||||
|
||||
# Determine if there are still users in the group
|
||||
user_group_list = UserGroupDao.get_group_user(group_id)
|
||||
if user_group_list:
|
||||
return UserGroupNotDeleteError.return_resp()
|
||||
GroupDao.delete_group(group_id)
|
||||
self.delete_group_hook(request, login_user, group_info)
|
||||
return resp_200()
|
||||
|
||||
def delete_group_hook(self, request: Request, login_user: UserPayload, group_info: Group):
|
||||
logger.info(f'act=delete_group_hook user={login_user.user_name} group_id={group_info.id}')
|
||||
# Log Audit Logs
|
||||
AuditLogService.delete_user_group(login_user, get_request_ip(request), group_info)
|
||||
# Move resources under a group to the default user group
|
||||
# Get all resources under a group
|
||||
all_resource = GroupResourceDao.get_group_all_resource(group_info.id)
|
||||
need_move_resource = []
|
||||
for one in all_resource:
|
||||
# Getting resources belongs to several groups,If you belong to more than one group, you don't have, Otherwise, transfer the resource to the default user group
|
||||
resource_groups = GroupResourceDao.get_resource_group(ResourceTypeEnum(one.type), one.third_id)
|
||||
if len(resource_groups) > 1:
|
||||
continue
|
||||
else:
|
||||
one.group_id = DefaultGroup
|
||||
need_move_resource.append(one)
|
||||
if need_move_resource:
|
||||
GroupResourceDao.update_group_resource(need_move_resource)
|
||||
GroupResourceDao.delete_group_resource_by_group_id(group_info.id)
|
||||
# Delete role list under user group
|
||||
RoleDao.delete_role_by_group_id(group_info.id)
|
||||
# Delete administrators of user groups
|
||||
UserGroupDao.delete_group_all_admin(group_info.id)
|
||||
# Send delete event toredisQueued
|
||||
delete_message = json.dumps({"id": group_info.id})
|
||||
redis_client = get_redis_client_sync()
|
||||
redis_client.rpush('delete_group', delete_message, expiration=86400)
|
||||
redis_client.publish('delete_group', delete_message)
|
||||
|
||||
def get_group_user_list(self, group_id: int, page_size: int, page_num: int) -> Optional[List[Dict]]:
|
||||
"""Get the full amountgroupVertical"""
|
||||
|
||||
# Inquiryuser
|
||||
user_group_list = UserGroupDao.get_group_user(group_id, page_size, page_num)
|
||||
if user_group_list:
|
||||
user_ids = [user.user_id for user in user_group_list]
|
||||
users = UserDao.get_user_by_ids(user_ids)
|
||||
return [self._dump_user_with_avatar_share_link(user) for user in users]
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _dump_user_with_avatar_share_link(user: User | None) -> Dict:
|
||||
if not user:
|
||||
return {}
|
||||
user_data = user.model_dump()
|
||||
user_data['avatar'] = UserService.get_avatar_share_link_sync(user_data.get('avatar'))
|
||||
return user_data
|
||||
|
||||
def insert_user_group(self, user_group: UserGroupCreate) -> UserGroupRead:
|
||||
"""Insert User Group"""
|
||||
|
||||
user_groups = UserGroupDao.get_user_group(user_group.user_id)
|
||||
if user_groups and user_group.group_id in [ug.group_id for ug in user_groups]:
|
||||
raise ValueError('Duplicate setup user group')
|
||||
|
||||
return UserGroupDao.insert_user_group(user_group)
|
||||
|
||||
def replace_user_groups(self, request: Request, login_user: UserPayload, user_id: int, group_ids: List[int]):
|
||||
""" Overwrite the user group the user belongs to """
|
||||
# Determine if the Operated User is a Super Admin
|
||||
user_role_list = UserRoleDao.get_user_roles(user_id)
|
||||
if any(one.role_id == AdminRole for one in user_role_list):
|
||||
raise AdminUserUpdateForbiddenError()
|
||||
|
||||
# Get all previous groupings of users
|
||||
old_group = UserGroupDao.get_user_group(user_id)
|
||||
old_group = [one.group_id for one in old_group]
|
||||
if not login_user.is_admin():
|
||||
# Get Operator Managed Groups
|
||||
admin_group = UserGroupDao.get_user_admin_group(login_user.user_id)
|
||||
admin_group = [one.group_id for one in admin_group]
|
||||
# Filter the group where the operator is located, only groups with permission management are processed
|
||||
old_group = [one for one in old_group if one in admin_group]
|
||||
# Describe this user Not in a user group administered by this user group administrator
|
||||
if not old_group:
|
||||
raise UnAuthorizedError()
|
||||
need_delete_group = old_group.copy()
|
||||
need_add_group = []
|
||||
for one in group_ids:
|
||||
if one not in old_group:
|
||||
# User groups to join
|
||||
need_add_group.append(one)
|
||||
else:
|
||||
# Remaining in the old user group is the user group to be moved out
|
||||
need_delete_group.remove(one)
|
||||
if need_delete_group:
|
||||
UserGroupDao.delete_user_groups(user_id, need_delete_group)
|
||||
if need_add_group:
|
||||
UserGroupDao.add_user_groups(user_id, need_add_group)
|
||||
|
||||
# Log Audit Logs
|
||||
group_infos = GroupDao.get_group_by_ids(old_group + group_ids)
|
||||
group_dict: Dict[int, str] = {}
|
||||
for one in group_infos:
|
||||
group_dict[one.id] = one.group_name
|
||||
note = "Pre-edit user groups:"
|
||||
for one in old_group:
|
||||
note += f'{group_dict.get(one, one)}、'
|
||||
note = note.rstrip('、')
|
||||
note += "Post-edit user groups:"
|
||||
for one in group_ids:
|
||||
note += f'{group_dict.get(one, one)}、'
|
||||
note = note.rstrip('、')
|
||||
AuditLogService.update_user(login_user, get_request_ip(request), user_id, list(group_dict.keys()), note)
|
||||
return None
|
||||
|
||||
def get_user_groups_list(self, user_id: int) -> List[GroupRead]:
|
||||
"""Get a list of user groups"""
|
||||
user_groups = UserGroupDao.get_user_group(user_id)
|
||||
if not user_groups:
|
||||
return []
|
||||
group_ids = [ug.group_id for ug in user_groups]
|
||||
return GroupDao.get_group_by_ids(group_ids)
|
||||
|
||||
def set_group_admin(self, request: Request, login_user: UserPayload, user_ids: List[int], group_id: int):
|
||||
"""Set up user group administrators"""
|
||||
# Get the list of administrators of the current user group
|
||||
user_group_admins = UserGroupDao.get_groups_admins([group_id])
|
||||
res = []
|
||||
need_delete_admin = []
|
||||
need_add_admin = user_ids
|
||||
if user_group_admins:
|
||||
for user in user_group_admins:
|
||||
if user.user_id in need_add_admin:
|
||||
res.append(user)
|
||||
need_add_admin.remove(user.user_id)
|
||||
else:
|
||||
need_delete_admin.append(user.user_id)
|
||||
if need_add_admin:
|
||||
# Users who are not in the group can be assigned as administrators. Do user creation
|
||||
for user_id in need_add_admin:
|
||||
res.append(UserGroupDao.insert_user_group_admin(user_id, group_id))
|
||||
if need_delete_admin:
|
||||
UserGroupDao.delete_group_admins(group_id, need_delete_admin)
|
||||
# Modified by the most recent modifier for the user group
|
||||
GroupDao.update_group_update_user(group_id, login_user.user_id)
|
||||
|
||||
group_info = GroupDao.get_user_group(group_id)
|
||||
self.update_group_hook(request, login_user, group_info)
|
||||
return res
|
||||
|
||||
def set_group_update_user(self, login_user: UserPayload, group_id: int):
|
||||
"""Set up user group administrators"""
|
||||
GroupDao.update_group_update_user(group_id, login_user.user_id)
|
||||
|
||||
async def get_group_resources(self, group_id: int, resource_type: ResourceTypeEnum, name: str,
|
||||
page_size: int, page_num: int) -> (List[Any], int):
|
||||
""" Get resources under user """
|
||||
if resource_type.value == ResourceTypeEnum.KNOWLEDGE.value:
|
||||
return await asyncio.to_thread(self.get_group_knowledge, group_id, name, page_size, page_num)
|
||||
elif resource_type.value == ResourceTypeEnum.WORK_FLOW.value:
|
||||
return await asyncio.to_thread(self.get_group_flow, group_id, name, page_size, page_num, FlowType.WORKFLOW)
|
||||
elif resource_type.value == ResourceTypeEnum.ASSISTANT.value:
|
||||
return await asyncio.to_thread(self.get_group_assistant, group_id, name, page_size, page_num)
|
||||
elif resource_type.value == ResourceTypeEnum.GPTS_TOOL.value:
|
||||
return await asyncio.to_thread(self.get_group_tool, group_id, name, page_size, page_num)
|
||||
elif resource_type.value == ResourceTypeEnum.DASHBOARD.value:
|
||||
return await self.get_group_dashboards(group_id, name, page_size, page_num)
|
||||
logger.warning('not support resource type: %s', resource_type)
|
||||
return [], 0
|
||||
|
||||
def get_user_map(self, user_ids: set[int]):
|
||||
user_list = UserDao.get_user_by_ids(list(user_ids))
|
||||
user_map = {user.user_id: user.user_name for user in user_list}
|
||||
return user_map
|
||||
|
||||
async def aget_user_map(self, user_ids: set[int]):
|
||||
user_list = await UserDao.aget_user_by_ids(list(user_ids))
|
||||
user_map = {user.user_id: user.user_name for user in user_list}
|
||||
return user_map
|
||||
|
||||
def get_group_flow(self, group_id: int, keyword: str, page_size: int, page_num: int,
|
||||
flow_type: FlowType = FlowType.WORKFLOW) -> (List[Any], int):
|
||||
""" Get a list of knowledge bases under user groups """
|
||||
resource_list = GroupResourceDao.get_group_resource(group_id, ResourceTypeEnum.WORK_FLOW)
|
||||
if not resource_list:
|
||||
return [], 0
|
||||
res = []
|
||||
flow_ids = [resource.third_id for resource in resource_list]
|
||||
data, total = FlowDao.filter_flows_by_ids(flow_ids, keyword, page_num, page_size, flow_type.value)
|
||||
db_user_ids = {one.user_id for one in data}
|
||||
user_map = self.get_user_map(db_user_ids)
|
||||
for one in data:
|
||||
one_dict = jsonable_encoder(one)
|
||||
one_dict["user_name"] = user_map.get(one.user_id, one.user_id)
|
||||
res.append(one_dict)
|
||||
|
||||
return res, total
|
||||
|
||||
def get_group_knowledge(self, group_id: int, keyword: str, page_size: int, page_num: int) -> (List[Any], int):
|
||||
""" Get a list of knowledge bases under user groups """
|
||||
# Query Knowledge Base under User GroupsIDVertical
|
||||
resource_list = GroupResourceDao.get_group_resource(group_id, ResourceTypeEnum.KNOWLEDGE)
|
||||
if not resource_list:
|
||||
return [], 0
|
||||
res = []
|
||||
knowledge_ids = [int(resource.third_id) for resource in resource_list]
|
||||
# Query Knowledge Base
|
||||
data, total = KnowledgeDao.filter_knowledge_by_ids(knowledge_ids, keyword, page_num, page_size)
|
||||
db_user_ids = {one.user_id for one in data}
|
||||
user_map = self.get_user_map(db_user_ids)
|
||||
for one in data:
|
||||
one_dict = jsonable_encoder(one)
|
||||
one_dict["user_name"] = user_map.get(one.user_id, one.user_id)
|
||||
res.append(one_dict)
|
||||
return res, total
|
||||
|
||||
def get_group_assistant(self, group_id: int, keyword: str, page_size: int, page_num: int) -> (List[Any], int):
|
||||
""" Get a list of helpers under a user group """
|
||||
# Query Assistant under User GroupsIDVertical
|
||||
resource_list = GroupResourceDao.get_group_resource(group_id, ResourceTypeEnum.ASSISTANT)
|
||||
if not resource_list:
|
||||
return [], 0
|
||||
res = []
|
||||
assistant_ids = [resource.third_id for resource in resource_list] # Query Assistant
|
||||
data, total = AssistantDao.filter_assistant_by_id(assistant_ids, keyword, page_num, page_size)
|
||||
for one in data:
|
||||
simple_one = AssistantService.return_simple_assistant_info(one)
|
||||
res.append(simple_one)
|
||||
return res, total
|
||||
|
||||
def get_group_tool(self, group_id: int, keyword: str, page_size: int, page_num: int) -> (List[Any], int):
|
||||
""" Get a list of tools under user groups """
|
||||
# Query Tools under User GroupsIDVertical
|
||||
resource_list = GroupResourceDao.get_group_resource(group_id, ResourceTypeEnum.GPTS_TOOL)
|
||||
if not resource_list:
|
||||
return [], 0
|
||||
res = []
|
||||
tool_ids = [int(resource.third_id) for resource in resource_list]
|
||||
# Query Tools
|
||||
data, total = GptsToolsDao.filter_tool_types_by_ids(tool_ids, keyword, page_num, page_size)
|
||||
db_user_ids = {one.user_id for one in data}
|
||||
user_map = self.get_user_map(db_user_ids)
|
||||
for one in data:
|
||||
one_dict = jsonable_encoder(one)
|
||||
one_dict["user_name"] = user_map.get(one.user_id, one.user_id)
|
||||
res.append(one_dict)
|
||||
return res, total
|
||||
|
||||
async def get_group_dashboards(self, group_id: int, keyword: str, page_size: int, page_num: int) -> (List[Any],
|
||||
int):
|
||||
|
||||
""" Get a list of dashboards under a user group """
|
||||
# Query the dashboard under the user groupIDVertical
|
||||
resource_list = await GroupResourceDao.aget_group_resources(group_id=group_id,
|
||||
resource_type=ResourceTypeEnum.DASHBOARD)
|
||||
if not resource_list:
|
||||
return [], 0
|
||||
res = []
|
||||
dashboard_ids = [int(resource.third_id) for resource in resource_list]
|
||||
# Query Dashboard
|
||||
data = await DashboardService.get_simple_dashboards(keyword=keyword, filter_ids=dashboard_ids)
|
||||
|
||||
user_map = await self.aget_user_map(set([one.user_id for one in data]))
|
||||
for one in data:
|
||||
one_dict = one.model_dump(exclude={"layout_config", "style_config"})
|
||||
one_dict["name"] = one.title
|
||||
one_dict["user_name"] = user_map.get(one.user_id, one.user_id)
|
||||
res.append(one_dict)
|
||||
if page_size and page_num:
|
||||
start_index = (page_num - 1) * page_size
|
||||
end_index = start_index + page_size
|
||||
paged_res = res[start_index:end_index]
|
||||
return paged_res, len(res)
|
||||
return res, len(res)
|
||||
|
||||
async def get_manage_resources(self, login_user: UserPayload, keyword: str, page: int, page_size: int) -> (list, int):
|
||||
""" Get a list of apps under a user group managed by a user Contains skills, assistants, workflows"""
|
||||
groups = []
|
||||
if not login_user.is_admin():
|
||||
groups = [str(one.group_id) for one in await UserGroupDao.aget_user_admin_group(login_user.user_id)]
|
||||
if not groups:
|
||||
return [], 0
|
||||
|
||||
resource_ids = []
|
||||
# Description is a user group administrator, need to filter to get the resources under the corresponding group
|
||||
if groups:
|
||||
group_resources = await GroupResourceDao.get_groups_resource(groups, resource_types=[
|
||||
ResourceTypeEnum.ASSISTANT,
|
||||
ResourceTypeEnum.WORK_FLOW,
|
||||
])
|
||||
if not group_resources:
|
||||
return [], 0
|
||||
resource_ids = [one.third_id for one in group_resources]
|
||||
|
||||
return await FlowDao.aget_all_apps(keyword, id_list=resource_ids, page=page, limit=page_size)
|
||||
@@ -0,0 +1,163 @@
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
from fastapi import Request
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError, NotFoundError
|
||||
from bisheng.common.errcode.tag import TagExistError, TagNotExistError
|
||||
from bisheng.common.models.config import ConfigDao, ConfigKeyEnum, Config
|
||||
from bisheng.database.models.assistant import AssistantDao
|
||||
from bisheng.database.models.flow import FlowDao
|
||||
from bisheng.database.models.group_resource import ResourceTypeEnum, GroupResourceDao
|
||||
from bisheng.database.models.role_access import AccessType
|
||||
from bisheng.database.models.tag import TagDao, Tag, TagLink, TagBusinessTypeEnum
|
||||
|
||||
|
||||
class TagService:
|
||||
|
||||
@classmethod
|
||||
def get_all_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
keyword: str = None, page: int = 0, limit: int = 10) -> (List[Tag], int):
|
||||
""" Get all tags """
|
||||
result = TagDao.search_tags(keyword, page, limit, business_type=TagBusinessTypeEnum.APPLICATION,
|
||||
business_id=TagBusinessTypeEnum.APPLICATION.value)
|
||||
return result, TagDao.count_tags(keyword, business_type=TagBusinessTypeEnum.APPLICATION,
|
||||
business_id=TagBusinessTypeEnum.APPLICATION.value)
|
||||
|
||||
@classmethod
|
||||
def create_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
name: str) -> Tag:
|
||||
# Query if there is a renaming of the label name
|
||||
exist_tag = TagDao.get_tag_by_name(name)
|
||||
if exist_tag:
|
||||
raise TagExistError.http_exception()
|
||||
new_tag = Tag(name=name, user_id=login_user.user_id, business_type=TagBusinessTypeEnum.APPLICATION,
|
||||
business_id=TagBusinessTypeEnum.APPLICATION.value)
|
||||
new_tag = TagDao.insert_tag(new_tag)
|
||||
return new_tag
|
||||
|
||||
@classmethod
|
||||
def update_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
tag_id: int,
|
||||
name: str) -> Tag:
|
||||
tag_info = TagDao.get_tag_by_id(tag_id)
|
||||
if not tag_info:
|
||||
raise TagNotExistError.http_exception()
|
||||
# Query if there is a renaming of the label name
|
||||
exist_tag = TagDao.get_tag_by_name(name)
|
||||
if exist_tag and exist_tag.id != tag_id:
|
||||
raise TagExistError.http_exception()
|
||||
|
||||
tag_info.name = name
|
||||
new_tag = TagDao.update_tag(tag_info)
|
||||
return new_tag
|
||||
|
||||
@classmethod
|
||||
def delete_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
tag_id: int) -> bool:
|
||||
""" NO NAME SPACE NO KEY VALUE!! """
|
||||
return TagDao.delete_tag(tag_id)
|
||||
|
||||
@classmethod
|
||||
def check_tag_link_permission(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
resource_id: str,
|
||||
resource_type: ResourceTypeEnum) -> bool:
|
||||
""" Check if labeling of resources is allowed """
|
||||
if login_user.is_admin():
|
||||
return True
|
||||
resource_info = None
|
||||
access_type: AccessType
|
||||
if resource_type == ResourceTypeEnum.ASSISTANT:
|
||||
resource_info = AssistantDao.get_one_assistant(resource_id)
|
||||
access_type = AccessType.ASSISTANT_WRITE
|
||||
elif resource_type == ResourceTypeEnum.WORK_FLOW:
|
||||
resource_info = FlowDao.get_flow_by_id(resource_id)
|
||||
access_type = AccessType.WORKFLOW_WRITE
|
||||
else:
|
||||
raise NotFoundError()
|
||||
if not resource_info:
|
||||
raise NotFoundError()
|
||||
|
||||
if login_user.access_check(resource_info.user_id, resource_id, access_type):
|
||||
return True
|
||||
|
||||
# Get user groups to which the resource belongs
|
||||
resource_groups = GroupResourceDao.get_resource_group(resource_type, resource_id)
|
||||
resource_groups = [int(one.group_id) for one in resource_groups]
|
||||
# Determine if the operator under is an administrator of a user group
|
||||
if not login_user.check_groups_admin(resource_groups):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def create_tag_link(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
tag_id: int,
|
||||
resource_id: str,
|
||||
resource_type: ResourceTypeEnum) -> TagLink:
|
||||
""" Associate resources with tags """
|
||||
cls.check_tag_link_permission(request, login_user, resource_id, resource_type)
|
||||
|
||||
new_link = TagLink(tag_id=tag_id, resource_id=resource_id, resource_type=resource_type.value,
|
||||
user_id=login_user.user_id)
|
||||
try:
|
||||
new_link = TagDao.insert_tag_link(new_link)
|
||||
except Exception as e:
|
||||
logger.error(f'tag_link_error: {e}')
|
||||
raise TagExistError.http_exception()
|
||||
return new_link
|
||||
|
||||
@classmethod
|
||||
def delete_tag_link(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
tag_id: int,
|
||||
resource_id: str,
|
||||
resource_type: ResourceTypeEnum) -> bool:
|
||||
""" Remove association of resources and tags """
|
||||
cls.check_tag_link_permission(request, login_user, resource_id, resource_type)
|
||||
|
||||
return TagDao.delete_resource_tag(tag_id, resource_id, resource_type)
|
||||
|
||||
@classmethod
|
||||
def get_home_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload) -> List[Tag]:
|
||||
""" Get a list of tags to show on the homepage """
|
||||
home_tags = ConfigDao.get_config(ConfigKeyEnum.HOME_TAGS)
|
||||
if not home_tags:
|
||||
return []
|
||||
home_tags = json.loads(home_tags.value)
|
||||
tags = TagDao.get_tags_by_ids(home_tags)
|
||||
|
||||
tags = sorted(tags, key=lambda x: home_tags.index(x.id))
|
||||
return tags
|
||||
|
||||
@classmethod
|
||||
def update_home_tag(cls,
|
||||
request: Request,
|
||||
login_user: UserPayload,
|
||||
tag_ids: List[int]) -> bool:
|
||||
""" Update the list of tags displayed on the homepage """
|
||||
home_tags = ConfigDao.get_config(ConfigKeyEnum.HOME_TAGS)
|
||||
if not home_tags:
|
||||
home_tags = Config(key=ConfigKeyEnum.HOME_TAGS.value, value=json.dumps(tag_ids))
|
||||
else:
|
||||
home_tags.value = json.dumps(tag_ids)
|
||||
|
||||
ConfigDao.insert_config(home_tags)
|
||||
return True
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
from bisheng.template.field.base import TemplateField
|
||||
from bisheng.template.template.base import Template
|
||||
from pydantic import BaseModel
|
||||
from langchain_core.language_models import BaseLanguageModel
|
||||
|
||||
|
||||
def set_flow_knowledge_id(graph_data: dict, knowledge_id: int):
|
||||
|
||||
for node in graph_data['nodes']:
|
||||
if 'VectorStore' in node['data']['node']['base_classes']:
|
||||
if 'collection_name' in node['data'].get('node').get('template').keys():
|
||||
node['data']['node']['template']['collection_name']['collection_id'] = knowledge_id
|
||||
if 'index_name' in node['data'].get('node').get('template').keys():
|
||||
node['data']['node']['template']['index_name']['collection_id'] = knowledge_id
|
||||
return graph_data
|
||||
|
||||
|
||||
def replace_flow_llm(graph_data: dict, llm: BaseLanguageModel, llm_param: dict):
|
||||
# Ganticlass, Gantitemplate， Others do not move.
|
||||
for node in graph_data['nodes']:
|
||||
if 'BaseLanguageModel' in node['data']['node']['base_classes']:
|
||||
node['data']['type'] = type(llm).__name__
|
||||
node['data']['node']['template'] = trans_obj_to_json(llm, llm_param)
|
||||
|
||||
return graph_data
|
||||
|
||||
|
||||
def trans_obj_to_json(obj: BaseModel, llm_param: dict):
|
||||
# template Build.
|
||||
template = []
|
||||
field_json = obj.__dict__
|
||||
for k, v in field_json.items():
|
||||
if k in llm_param:
|
||||
template.append(
|
||||
TemplateField(field_type=type(v).__name__, name=k,
|
||||
value=llm_param.get(k)).to_dict())
|
||||
return Template(type_name=type(obj).__name__, fields=template).to_dict()
|
||||
@@ -0,0 +1,493 @@
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from langchain.memory import ConversationBufferWindowMemory
|
||||
|
||||
from bisheng.api.v1.schema.workflow import WorkflowEvent, WorkflowEventType, WorkflowInputSchema, WorkflowInputItem, \
|
||||
WorkflowOutputSchema
|
||||
from bisheng.api.v1.schemas import ChatResponse
|
||||
from bisheng.common.chat.utils import SourceType
|
||||
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.flow import WorkFlowInitError
|
||||
from bisheng.common.errcode.http_error import NotFoundError, UnAuthorizedError
|
||||
from bisheng.common.services import telemetry_service
|
||||
from bisheng.common.services.base import BaseService
|
||||
from bisheng.core.logger import trace_id_var
|
||||
from bisheng.database.models.flow import FlowDao, FlowStatus, FlowType, Flow
|
||||
from bisheng.database.models.flow import UserLinkType
|
||||
from bisheng.database.models.flow_version import FlowVersionDao
|
||||
from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum
|
||||
from bisheng.database.models.role_access import AccessType, RoleAccessDao
|
||||
from bisheng.database.models.tag import TagDao, TagBusinessTypeEnum
|
||||
from bisheng.database.models.user_link import UserLinkDao
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
from bisheng.user.domain.models.user_role import UserRoleDao
|
||||
from bisheng.utils import generate_uuid
|
||||
from bisheng.workflow.callback.base_callback import BaseCallback
|
||||
from bisheng.workflow.common.node import BaseNodeData, NodeType
|
||||
from bisheng.workflow.graph.graph_state import GraphState
|
||||
from bisheng.workflow.graph.workflow import Workflow
|
||||
from bisheng.workflow.nodes.node_manage import NodeFactory
|
||||
|
||||
|
||||
class WorkFlowService(BaseService):
|
||||
SUPPORTED_APP_TYPES = {FlowType.WORKFLOW.value, FlowType.ASSISTANT.value}
|
||||
|
||||
@classmethod
|
||||
def filter_supported_apps(cls, data: list[dict]) -> list[dict]:
|
||||
return [one for one in data if one.get('flow_type') in cls.SUPPORTED_APP_TYPES]
|
||||
|
||||
@classmethod
|
||||
def add_extra_field(cls, user: UserPayload, data: list[dict], managed: bool = False) -> list[dict]:
|
||||
""" Add some extra fields for app list """
|
||||
data = cls.filter_supported_apps(data)
|
||||
# ApplicationsIDVertical
|
||||
resource_ids = []
|
||||
# Skill Creation User'sIDVertical
|
||||
user_ids = []
|
||||
for one in data:
|
||||
one['id'] = one['id']
|
||||
resource_ids.append(one['id'])
|
||||
user_ids.append(one['user_id'])
|
||||
# Get user information in the list
|
||||
user_infos = UserDao.get_user_by_ids(user_ids)
|
||||
user_dict = {one.user_id: one.user_name for one in user_infos}
|
||||
|
||||
# Get version information in the list
|
||||
version_infos = FlowVersionDao.get_list_by_flow_ids(resource_ids)
|
||||
flow_versions = {}
|
||||
for one in version_infos:
|
||||
if one.flow_id not in flow_versions:
|
||||
flow_versions[one.flow_id] = []
|
||||
flow_versions[one.flow_id].append(jsonable_encoder(one))
|
||||
|
||||
resource_groups = GroupResourceDao.get_resources_group(None, resource_ids)
|
||||
resource_group_dict = {}
|
||||
for one in resource_groups:
|
||||
if one.third_id not in resource_group_dict:
|
||||
resource_group_dict[one.third_id] = []
|
||||
resource_group_dict[one.third_id].append(one.group_id)
|
||||
|
||||
resource_tag_dict = TagDao.get_tags_by_resource(None, resource_ids)
|
||||
|
||||
# Add additional information
|
||||
for one in data:
|
||||
if one['flow_type'] == FlowType.WORKFLOW.value:
|
||||
access_type = AccessType.WORKFLOW_WRITE
|
||||
else:
|
||||
access_type = AccessType.ASSISTANT_WRITE
|
||||
|
||||
one['user_name'] = user_dict.get(one['user_id'], one['user_id'])
|
||||
one['write'] = True if managed else user.access_check(one['user_id'], one['id'], access_type)
|
||||
one['version_list'] = flow_versions.get(one['id'], [])
|
||||
one['group_ids'] = resource_group_dict.get(one['id'], [])
|
||||
one['tags'] = resource_tag_dict.get(one['id'], [])
|
||||
one['logo'] = cls.get_logo_share_link(one['logo'])
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def get_all_flows(cls, user: UserPayload, name: str, status: int, tag_id: Optional[int], flow_type: Optional[int],
|
||||
page: int = 1, page_size: int = 10, managed: bool = False,
|
||||
skip_pagination: bool = False) -> (list[dict], int):
|
||||
"""
|
||||
Get all the skills
|
||||
"""
|
||||
if flow_type is not None and flow_type not in cls.SUPPORTED_APP_TYPES:
|
||||
return [], 0
|
||||
|
||||
# SetujutagDapatkanidVertical
|
||||
flow_ids = []
|
||||
if tag_id:
|
||||
ret = TagDao.get_resources_by_tags_batch([tag_id], [ResourceTypeEnum.WORK_FLOW, ResourceTypeEnum.ASSISTANT])
|
||||
if not ret:
|
||||
return [], 0
|
||||
flow_ids = [one.resource_id for one in ret]
|
||||
|
||||
query_page = page
|
||||
query_page_size = page_size
|
||||
if flow_type is None:
|
||||
query_page = 0
|
||||
query_page_size = 0
|
||||
|
||||
# Get a list of skills visible to the user
|
||||
if user.is_admin():
|
||||
data, total = FlowDao.get_all_apps(name, status, flow_ids, flow_type, None, None, None, query_page,
|
||||
query_page_size)
|
||||
else:
|
||||
access_list = [AccessType.WORKFLOW, AccessType.ASSISTANT_READ]
|
||||
if managed:
|
||||
access_list = [AccessType.WORKFLOW_WRITE, AccessType.ASSISTANT_WRITE]
|
||||
flow_id_extra = user.get_user_access_resource_ids(access_list)
|
||||
data, total = FlowDao.get_all_apps(name, status, flow_ids, flow_type, user.user_id, flow_id_extra, None,
|
||||
query_page, query_page_size)
|
||||
data = cls.filter_supported_apps(data)
|
||||
if flow_type is None and not skip_pagination:
|
||||
total = len(data)
|
||||
start_index = (page - 1) * page_size
|
||||
end_index = start_index + page_size
|
||||
data = data[start_index:end_index]
|
||||
data = cls.add_extra_field(user, data, managed)
|
||||
|
||||
return data, total
|
||||
|
||||
@classmethod
|
||||
def run_once(cls, login_user: UserPayload, node_input: Dict[str, any], node_data: Dict[any, any], workflow_id: str):
|
||||
workflow_info = FlowDao.get_flow_by_id(workflow_id)
|
||||
if not workflow_info:
|
||||
raise NotFoundError()
|
||||
|
||||
node_data = BaseNodeData(**node_data.get('data', {}))
|
||||
base_callback = BaseCallback()
|
||||
graph_state = GraphState()
|
||||
graph_state.history_memory = ConversationBufferWindowMemory(k=10)
|
||||
node = NodeFactory.instance_node(node_type=node_data.type,
|
||||
node_data=node_data,
|
||||
user_id=login_user.user_id,
|
||||
workflow_id=workflow_info.id,
|
||||
workflow_name=workflow_info.name,
|
||||
graph_state=graph_state,
|
||||
target_edges=None,
|
||||
max_steps=233,
|
||||
callback=base_callback)
|
||||
if node_data.type == NodeType.CODE.value:
|
||||
node.handle_input({
|
||||
'code_input': [
|
||||
{
|
||||
'key': k,
|
||||
'value': v,
|
||||
'type': 'input'
|
||||
} for k, v in node_input.items()
|
||||
]
|
||||
})
|
||||
elif node_data.type == NodeType.TOOL.value:
|
||||
user_input = {}
|
||||
for k, v in node_input.items():
|
||||
user_input[k] = v
|
||||
node.handle_input(user_input)
|
||||
else:
|
||||
for key, val in node_input.items():
|
||||
graph_state.set_variable_by_str(key, val)
|
||||
|
||||
exec_id = generate_uuid()
|
||||
result = node._run(exec_id)
|
||||
log_data = node.parse_log(exec_id, result)
|
||||
res = []
|
||||
for one_batch in log_data:
|
||||
ret = []
|
||||
for one in one_batch:
|
||||
if node_data.type == NodeType.QA_RETRIEVER.value and one['key'] != 'retrieved_result':
|
||||
continue
|
||||
if node_data.type == NodeType.RAG.value and one['key'] != 'retrieved_result' and one[
|
||||
'type'] != 'variable':
|
||||
continue
|
||||
if node_data.type == NodeType.LLM.value and one['type'] != 'variable':
|
||||
continue
|
||||
if node_data.type == NodeType.AGENT.value and one['type'] not in ['tool', 'variable']:
|
||||
continue
|
||||
if node_data.type == NodeType.CODE.value and one['key'] != 'code_output':
|
||||
continue
|
||||
if node_data.type == NodeType.TOOL.value and one['key'] != 'output':
|
||||
continue
|
||||
ret.append({
|
||||
'key': one['key'],
|
||||
'value': one['value'],
|
||||
'type': one['type']
|
||||
})
|
||||
res.append(ret)
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
async def update_flow_status(cls, login_user: UserPayload, flow_id: str, version_id: int, status: int):
|
||||
"""
|
||||
Modify workflow status, Also modify the current version of the workflow
|
||||
"""
|
||||
db_flow = await FlowDao.aget_flow_by_id(flow_id)
|
||||
if not db_flow:
|
||||
raise NotFoundError()
|
||||
if not await login_user.async_access_check(db_flow.user_id, flow_id, AccessType.WORKFLOW_WRITE):
|
||||
raise UnAuthorizedError()
|
||||
|
||||
version_info = await FlowVersionDao.aget_version_by_id(version_id)
|
||||
if not version_info or version_info.flow_id != flow_id:
|
||||
raise NotFoundError()
|
||||
if status == FlowStatus.ONLINE.value:
|
||||
# workflowInitialization check for
|
||||
try:
|
||||
_ = Workflow(flow_id, db_flow.name, login_user.user_id, version_info.data, False,
|
||||
10,
|
||||
10,
|
||||
None)
|
||||
except Exception as e:
|
||||
raise WorkFlowInitError(msg=str(e))
|
||||
|
||||
await FlowVersionDao.change_current_version(flow_id, version_info)
|
||||
db_flow.status = status
|
||||
await FlowDao.aupdate_flow(db_flow)
|
||||
await telemetry_service.log_event(
|
||||
user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def convert_chat_response_to_workflow_event(cls, chat_response: ChatResponse) -> WorkflowEvent:
|
||||
workflow_event = WorkflowEvent(
|
||||
event=chat_response.category,
|
||||
message_id=chat_response.message_id,
|
||||
status='end',
|
||||
node_id=chat_response.message.get('node_id'),
|
||||
node_name=chat_response.message.get('name'),
|
||||
node_execution_id=chat_response.message.get('unique_id'),
|
||||
)
|
||||
match workflow_event.event:
|
||||
case WorkflowEventType.UserInput.value:
|
||||
return cls.convert_user_input_event(chat_response, workflow_event)
|
||||
case WorkflowEventType.GuideWord.value:
|
||||
workflow_event.output_schema = WorkflowOutputSchema(
|
||||
message=chat_response.message.get('guide_word')
|
||||
)
|
||||
case WorkflowEventType.GuideQuestion.value:
|
||||
workflow_event.output_schema = WorkflowOutputSchema(
|
||||
message=chat_response.message.get('guide_question')
|
||||
)
|
||||
case WorkflowEventType.OutputMsg.value:
|
||||
return cls.convert_output_event(chat_response, workflow_event)
|
||||
case WorkflowEventType.OutputWithChoose.value:
|
||||
return cls.convert_output_choose_event(chat_response, workflow_event)
|
||||
case WorkflowEventType.OutputWithInput.value:
|
||||
return cls.convert_output_input_event(chat_response, workflow_event)
|
||||
case WorkflowEventType.StreamMsg.value:
|
||||
workflow_event.status = chat_response.type
|
||||
workflow_event.output_schema = WorkflowOutputSchema(
|
||||
message=chat_response.message.get('msg'),
|
||||
reasoning_content=chat_response.message.get('reasoning_content'),
|
||||
output_key=chat_response.message.get('output_key'),
|
||||
)
|
||||
cls.handle_source(chat_response, workflow_event)
|
||||
case WorkflowEventType.Error.value:
|
||||
workflow_event.event = WorkflowEventType.Close.value
|
||||
workflow_event.output_schema = WorkflowOutputSchema(
|
||||
message=chat_response.message
|
||||
)
|
||||
|
||||
return workflow_event
|
||||
|
||||
@classmethod
|
||||
def handle_source(cls, chat_response: ChatResponse, workflow_event: WorkflowEvent):
|
||||
if chat_response.source == SourceType.FILE.value:
|
||||
workflow_event.output_schema.source_url = f'resouce/{chat_response.chat_id}/{chat_response.message_id}'
|
||||
elif chat_response.source in [SourceType.LINK.value, SourceType.QA.value]:
|
||||
workflow_event.output_schema.extra = chat_response.extra
|
||||
|
||||
@classmethod
|
||||
def convert_user_input_event(cls, chat_response: ChatResponse, workflow_event: WorkflowEvent) -> WorkflowEvent:
|
||||
event_input_schema = chat_response.message.get('input_schema')
|
||||
input_schema = WorkflowInputSchema(
|
||||
input_type=event_input_schema.get('tab'),
|
||||
)
|
||||
if input_schema.input_type == 'form_input':
|
||||
# Front-end form definitions go to back-end form definitions
|
||||
input_schema.value = [WorkflowInputItem(**one) for one in event_input_schema.get('value', [])]
|
||||
for one in input_schema.value:
|
||||
one.label = one.value
|
||||
one.value = ''
|
||||
else:
|
||||
# Description is input box input
|
||||
input_schema.value = [
|
||||
WorkflowInputItem(
|
||||
key=event_input_schema.get('key'),
|
||||
type='text',
|
||||
required=True,
|
||||
value=''
|
||||
)
|
||||
]
|
||||
for one in event_input_schema.get('value', []):
|
||||
if not one:
|
||||
continue
|
||||
tmp = WorkflowInputItem(**one)
|
||||
if tmp.key == 'dialog_files_content':
|
||||
tmp.type = 'dialog_file'
|
||||
tmp.value = []
|
||||
elif tmp.key == 'dialog_file_accept':
|
||||
tmp.type = 'dialog_file_accept'
|
||||
input_schema.value.append(tmp)
|
||||
workflow_event.input_schema = input_schema
|
||||
return workflow_event
|
||||
|
||||
@classmethod
|
||||
def convert_output_event(cls, chat_response: ChatResponse, workflow_event: WorkflowEvent) -> WorkflowEvent:
|
||||
workflow_event.output_schema = WorkflowOutputSchema(
|
||||
message=chat_response.message.get('msg'),
|
||||
files=chat_response.files,
|
||||
output_key=chat_response.message.get('output_key')
|
||||
)
|
||||
cls.handle_source(chat_response, workflow_event)
|
||||
return workflow_event
|
||||
|
||||
@classmethod
|
||||
def convert_output_input_event(cls, chat_response: ChatResponse, workflow_event: WorkflowEvent) -> WorkflowEvent:
|
||||
workflow_event = cls.convert_output_event(chat_response, workflow_event)
|
||||
workflow_event.input_schema = WorkflowInputSchema(
|
||||
input_type='message_inline_input',
|
||||
value=[WorkflowInputItem(
|
||||
key=chat_response.message.get('key'),
|
||||
type='text',
|
||||
required=True,
|
||||
value=chat_response.message.get('input_msg', '')
|
||||
)]
|
||||
)
|
||||
return workflow_event
|
||||
|
||||
@classmethod
|
||||
def convert_output_choose_event(cls, chat_response: ChatResponse, workflow_event: WorkflowEvent) -> WorkflowEvent:
|
||||
workflow_event = cls.convert_output_event(chat_response, workflow_event)
|
||||
workflow_event.input_schema = WorkflowInputSchema(
|
||||
input_type='message_inline_option',
|
||||
value=[WorkflowInputItem(
|
||||
key=chat_response.message.get('key'),
|
||||
type='select',
|
||||
required=True,
|
||||
value='',
|
||||
options=chat_response.message.get('options', [])
|
||||
)]
|
||||
)
|
||||
return workflow_event
|
||||
|
||||
@classmethod
|
||||
def get_frequently_used_flows(cls, user: UserPayload, user_link_type: str,
|
||||
page: int = 1,
|
||||
page_size: int = 8) -> (list[dict], int):
|
||||
"""
|
||||
Get common skills
|
||||
"""
|
||||
# Setujuuser_idAndtagDapatkanidlist and keep pressingcreate_timeAscending order
|
||||
flow_ids = []
|
||||
user_link_order = {} # Record the order of each app in the common list of users
|
||||
|
||||
ret = UserLinkDao.get_user_link(user.user_id, [app_type.value for app_type in UserLinkType.app.value])
|
||||
if not ret:
|
||||
return [], 0
|
||||
|
||||
# Save original order andflow_ids
|
||||
for index, user_link in enumerate(ret):
|
||||
flow_ids.append(user_link.type_detail)
|
||||
user_link_order[user_link.type_detail] = index
|
||||
|
||||
# Get a list of skills visible to the user (no pagination as we need to sort manually)
|
||||
if user.is_admin():
|
||||
data, _ = FlowDao.get_all_apps(status=FlowStatus.ONLINE.value, id_list=flow_ids, page=0, limit=0)
|
||||
else:
|
||||
flow_id_extra = user.get_user_access_resource_ids(
|
||||
[AccessType.WORKFLOW, AccessType.ASSISTANT_READ])
|
||||
data, _ = FlowDao.get_all_apps(status=FlowStatus.ONLINE.value, id_list=flow_ids, user_id=user.user_id,
|
||||
id_extra=flow_id_extra, page=0, limit=0)
|
||||
data = cls.filter_supported_apps(data)
|
||||
|
||||
# Reorder users in the order they are added to the stock
|
||||
data.sort(key=lambda x: user_link_order.get(x['id'], float('inf')))
|
||||
|
||||
# Manual pagination
|
||||
total = len(data)
|
||||
start_index = (page - 1) * page_size
|
||||
end_index = start_index + page_size
|
||||
data = data[start_index:end_index]
|
||||
|
||||
data = cls.add_extra_field(user, data)
|
||||
|
||||
return data, total
|
||||
|
||||
@classmethod
|
||||
def delete_frequently_used_flows(cls, user: UserPayload, user_link_type: str, type_detail: str):
|
||||
UserLinkDao.delete_user_link(user.user_id, user_link_type, type_detail)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def add_frequently_used_flows(cls, user: UserPayload, user_link_type: str, type_detail: str):
|
||||
user_link, is_new = UserLinkDao.add_user_link(user.user_id, user_link_type, type_detail)
|
||||
return is_new
|
||||
|
||||
@classmethod
|
||||
def get_uncategorized_flows(
|
||||
cls,
|
||||
user: UserPayload,
|
||||
page: int = 1,
|
||||
page_size: int = 8,
|
||||
keyword: Optional[str] = None,
|
||||
) -> tuple[list, int]:
|
||||
"""
|
||||
Get a list of unsorted skills
|
||||
"""
|
||||
# SetujutagDapatkanidVertical
|
||||
all_tags = TagDao.search_tags(None, 0, 0, business_type=TagBusinessTypeEnum.APPLICATION,
|
||||
business_id=TagBusinessTypeEnum.APPLICATION.value)
|
||||
tag_id = [tag.id for tag in all_tags]
|
||||
flow_ids_not_in = []
|
||||
if tag_id:
|
||||
ret = TagDao.get_resources_by_tags_batch(tag_id, [ResourceTypeEnum.WORK_FLOW, ResourceTypeEnum.ASSISTANT])
|
||||
if not ret:
|
||||
return [], 0
|
||||
flow_ids_not_in = [one.resource_id for one in ret]
|
||||
|
||||
# Get a list of skills visible to the user
|
||||
if user.is_admin():
|
||||
data, _ = FlowDao.get_all_apps(
|
||||
keyword,
|
||||
FlowStatus.ONLINE.value,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
flow_ids_not_in,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
else:
|
||||
user_role = UserRoleDao.get_user_roles(user.user_id)
|
||||
role_ids = [role.role_id for role in user_role]
|
||||
role_access = RoleAccessDao.get_role_access_batch(role_ids,
|
||||
[AccessType.WORKFLOW, AccessType.ASSISTANT_READ])
|
||||
flow_id_extra = []
|
||||
if role_access:
|
||||
flow_id_extra = [access.third_id for access in role_access]
|
||||
data, _ = FlowDao.get_all_apps(keyword, FlowStatus.ONLINE.value, None, None, user.user_id, flow_id_extra,
|
||||
flow_ids_not_in, 0, 0)
|
||||
data = cls.filter_supported_apps(data)
|
||||
total = len(data)
|
||||
start_index = (page - 1) * page_size
|
||||
end_index = start_index + page_size
|
||||
data = data[start_index:end_index]
|
||||
|
||||
# <g id="Bold">Medical Treatment:</g>logo URL, convert relative paths to full accessible links
|
||||
for one in data:
|
||||
one['logo'] = cls.get_logo_share_link(one['logo'])
|
||||
|
||||
return data, total
|
||||
|
||||
@classmethod
|
||||
async def get_one_workflow_simple_info(cls, workflow_id: str) -> Flow | None:
|
||||
"""
|
||||
Get individual workflow details
|
||||
"""
|
||||
return await FlowDao.get_one_flow_simple(workflow_id)
|
||||
|
||||
@classmethod
|
||||
def get_one_workflow_simple_info_sync(cls, workflow_id: str) -> Optional[Flow]:
|
||||
"""
|
||||
Get individual workflow details (Sync)
|
||||
"""
|
||||
return FlowDao.get_one_flow_simple_sync(workflow_id)
|
||||
|
||||
@classmethod
|
||||
def get_all_apps_by_time_range_sync(cls, start_time: datetime, end_time: datetime, page: int = 1,
|
||||
page_size: int = 100) -> list[dict]:
|
||||
"""
|
||||
Get all apps based on timeframe
|
||||
"""
|
||||
return FlowDao.get_all_app_by_time_range_sync(start_time, end_time, page, page_size)
|
||||
|
||||
@classmethod
|
||||
def get_first_app(cls) -> Dict | None:
|
||||
return FlowDao.get_first_app()
|
||||
@@ -0,0 +1,13 @@
|
||||
from bisheng.workstation.domain.schemas import (
|
||||
SSECallbackClient,
|
||||
WorkstationConversation,
|
||||
WorkstationMessage,
|
||||
)
|
||||
from bisheng.workstation.domain.services import WorkStationService
|
||||
|
||||
__all__ = [
|
||||
'WorkStationService',
|
||||
'WorkstationMessage',
|
||||
'WorkstationConversation',
|
||||
'SSECallbackClient',
|
||||
]
|
||||
@@ -0,0 +1,11 @@
|
||||
import aiohttp
|
||||
|
||||
|
||||
async def get_url_content(url: str) -> str:
|
||||
""" Get the returned of the interfacebodyContents """
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url) as response:
|
||||
if response.status != 200:
|
||||
raise Exception(f'Failed to download content, HTTP status code: {response.status}')
|
||||
res = await response.read()
|
||||
return res.decode('utf-8')
|
||||
@@ -0,0 +1,37 @@
|
||||
from bisheng.api.v1.assistant import router as assistant_router
|
||||
from bisheng.api.v1.audit import router as audit_router
|
||||
from bisheng.api.v1.chat import router as chat_router
|
||||
from bisheng.api.v1.endpoints import router as endpoints_router
|
||||
from bisheng.api.v1.evaluation import router as evaluation_router
|
||||
from bisheng.api.v1.flows import router as flows_router
|
||||
from bisheng.api.v1.invite_code import router as invite_code_router
|
||||
from bisheng.api.v1.mark_task import router as mark_router
|
||||
from bisheng.api.v1.report import router as report_router
|
||||
from bisheng.api.v1.skillcenter import router as skillcenter_router
|
||||
from bisheng.api.v1.tag import router as tag_router
|
||||
from bisheng.api.v1.usergroup import router as group_router
|
||||
from bisheng.api.v1.variable import router as variable_router
|
||||
from bisheng.api.v1.workflow import router as workflow_router
|
||||
from bisheng.workstation.api import router as workstation_router
|
||||
from bisheng.tool.api.tool import router as tool_router
|
||||
from bisheng.user.api.user import router as user_router
|
||||
|
||||
__all__ = [
|
||||
'chat_router',
|
||||
'endpoints_router',
|
||||
'flows_router',
|
||||
'skillcenter_router',
|
||||
'user_router',
|
||||
'variable_router',
|
||||
'report_router',
|
||||
'assistant_router',
|
||||
'evaluation_router',
|
||||
'group_router',
|
||||
'audit_router',
|
||||
'tag_router',
|
||||
'workflow_router',
|
||||
'mark_router',
|
||||
"tool_router",
|
||||
"invite_code_router",
|
||||
"workstation_router",
|
||||
]
|
||||
@@ -0,0 +1,176 @@
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from fastapi import (APIRouter, Body, Depends, HTTPException, Query, Request, WebSocket,
|
||||
WebSocketException)
|
||||
from fastapi import status as http_status
|
||||
from fastapi.responses import StreamingResponse
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.assistant import AssistantService
|
||||
from bisheng.api.v1.schemas import (AssistantCreateReq, AssistantUpdateReq,
|
||||
StreamData, resp_200)
|
||||
from bisheng.common.chat.manager import ChatManager
|
||||
from bisheng.common.chat.types import WorkType
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import NotFoundError
|
||||
from bisheng.common.schemas.api import PageData
|
||||
from bisheng.core.cache.redis_manager import get_redis_client
|
||||
from bisheng.database.models.assistant import Assistant
|
||||
from bisheng.share_link.api.dependencies import header_share_token_parser
|
||||
from bisheng.share_link.domain.models.share_link import ShareLink
|
||||
from bisheng.utils import generate_uuid
|
||||
|
||||
router = APIRouter(prefix='/assistant', tags=['Assistant'])
|
||||
chat_manager = ChatManager()
|
||||
|
||||
|
||||
@router.get('')
|
||||
def get_assistant(*,
|
||||
name: str = Query(default=None, description='assistant name, fuzzy matching, Fuzzy matches with description'),
|
||||
tag_id: int = Query(default=None, description='labelID'),
|
||||
page: Optional[int] = Query(default=1, gt=0, description='Page'),
|
||||
limit: Optional[int] = Query(default=10, gt=0, description='Listings Per Page'),
|
||||
status: Optional[int] = Query(default=None, description='Is online status'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
data, total = AssistantService.get_assistant(login_user, name, status, tag_id, page, limit)
|
||||
return resp_200(PageData(data=data, total=total))
|
||||
|
||||
|
||||
# Get the details of an assistant
|
||||
@router.get('/info/{assistant_id}')
|
||||
async def get_assistant_info(*, assistant_id: str, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
share_link: Union['ShareLink', None] = Depends(header_share_token_parser)):
|
||||
"""Getting Helper Information"""
|
||||
res = await AssistantService.get_assistant_info(assistant_id, login_user, share_link)
|
||||
return resp_200(data=res)
|
||||
|
||||
|
||||
@router.post('/delete')
|
||||
def delete_assistant(*,
|
||||
request: Request,
|
||||
assistant_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""Delete Assistant"""
|
||||
AssistantService.delete_assistant(request, login_user, assistant_id)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.post('')
|
||||
async def create_assistant(*,
|
||||
request: Request,
|
||||
req: AssistantCreateReq,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
# get login user
|
||||
assistant = Assistant(**req.model_dump(), user_id=login_user.user_id)
|
||||
res = await AssistantService.create_assistant(request, login_user, assistant)
|
||||
return resp_200(data=res)
|
||||
|
||||
|
||||
@router.put('')
|
||||
async def update_assistant(*,
|
||||
request: Request,
|
||||
req: AssistantUpdateReq,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
# get login user
|
||||
assistant_model = await AssistantService.update_assistant(request, login_user, req)
|
||||
return resp_200(data=assistant_model)
|
||||
|
||||
|
||||
@router.post('/status')
|
||||
async def update_status(*,
|
||||
request: Request,
|
||||
assistant_id: str = Body(description='Assistant UniqueID', alias='id'),
|
||||
status: int = Body(description='whether to go online: 0 offline, 1 online'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
await AssistantService.update_status(request, login_user, assistant_id, status)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.post('/auto/task')
|
||||
async def auto_update_assistant_task(*, request: Request, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
assistant_id: str = Body(description='Assistant UniqueID'),
|
||||
prompt: str = Body(description='User-filled prompts')):
|
||||
# Deposit Cache
|
||||
task_id = generate_uuid()
|
||||
redis_client = await get_redis_client()
|
||||
await redis_client.aset(f'auto_update_task:{task_id}', {
|
||||
'assistant_id': assistant_id,
|
||||
'prompt': prompt,
|
||||
})
|
||||
return resp_200(data={
|
||||
'task_id': task_id
|
||||
})
|
||||
|
||||
|
||||
# Nicepromptand tool selection
|
||||
@router.get('/auto', response_class=StreamingResponse)
|
||||
async def auto_update_assistant(*, task_id: str = Query(description='Optimization Task UniqueID'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
redis_client = await get_redis_client()
|
||||
task = await redis_client.aget(f'auto_update_task:{task_id}')
|
||||
if not task:
|
||||
raise NotFoundError()
|
||||
assistant_id = task['assistant_id']
|
||||
prompt = task['prompt']
|
||||
|
||||
async def event_stream():
|
||||
try:
|
||||
async for message in AssistantService.auto_update_stream(assistant_id, prompt, login_user):
|
||||
yield message
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'data': ''}))
|
||||
except Exception as e:
|
||||
logger.exception('assistant auto update error')
|
||||
yield str(StreamData(event='message', data={'type': 'end', 'message': str(e)}))
|
||||
|
||||
return StreamingResponse(event_stream(), media_type='text/event-stream')
|
||||
|
||||
|
||||
# Update assistant prompts
|
||||
@router.post('/prompt')
|
||||
async def update_prompt(*,
|
||||
assistant_id: str = Body(description='Assistant UniqueID', alias='id'),
|
||||
prompt: str = Body(description='Used by Usersprompt'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
AssistantService.update_prompt(assistant_id, prompt, login_user)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.post('/flow')
|
||||
async def update_flow_list(*,
|
||||
assistant_id: str = Body(description='Assistant UniqueID', alias='id'),
|
||||
flow_list: List[str] = Body(description='List of user-selected skills'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
AssistantService.update_flow_list(assistant_id, flow_list, login_user)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.post('/tool')
|
||||
async def update_tool_list(*,
|
||||
assistant_id: str = Body(description='Assistant UniqueID', alias='id'),
|
||||
tool_list: List[int] = Body(description='List of tools selected by the user'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Update the list of tools selected by the assistant """
|
||||
AssistantService.update_tool_list(assistant_id, tool_list, login_user)
|
||||
return resp_200()
|
||||
|
||||
|
||||
# Assistant Dialogue'swebsocketCONNECT
|
||||
@router.websocket('/chat/{assistant_id}')
|
||||
async def chat(*,
|
||||
assistant_id: str,
|
||||
websocket: WebSocket,
|
||||
chat_id: Optional[str] = None,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user_from_ws)):
|
||||
try:
|
||||
await chat_manager.dispatch_client(websocket, assistant_id, chat_id, login_user,
|
||||
WorkType.GPTS, websocket)
|
||||
except WebSocketException as exc:
|
||||
logger.error(f'Websocket exception: {str(exc)}')
|
||||
await websocket.close(code=http_status.WS_1011_INTERNAL_ERROR, reason=str(exc))
|
||||
except Exception as exc:
|
||||
logger.exception(f'Error in chat websocket: {str(exc)}')
|
||||
message = exc.detail if isinstance(exc, HTTPException) else str(exc)
|
||||
if 'Could not validate credentials' in str(exc):
|
||||
await websocket.close(code=http_status.WS_1008_POLICY_VIOLATION, reason='Unauthorized')
|
||||
else:
|
||||
await websocket.close(code=http_status.WS_1011_INTERNAL_ERROR, reason=message)
|
||||
@@ -0,0 +1,77 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import APIRouter, Query, Depends
|
||||
|
||||
from bisheng.api.services.audit_log import AuditLogService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
|
||||
router = APIRouter(prefix='/audit', tags=['AuditLog'])
|
||||
|
||||
|
||||
@router.get('')
|
||||
async def get_audit_logs(*,
|
||||
group_ids: Optional[List[str]] = Query(default=[], description='GroupingidVertical'),
|
||||
operator_ids: Optional[List[int]] = Query(default=[], description='WhoidVertical'),
|
||||
start_time: Optional[datetime] = Query(default=None, description='Start when'),
|
||||
end_time: Optional[datetime] = Query(default=None, description='End time'),
|
||||
system_id: Optional[str] = Query(default=None, description='Module Item'),
|
||||
event_type: Optional[str] = Query(default=None, description='Operation behaviors'),
|
||||
page: Optional[int] = Query(default=0, description='Page'),
|
||||
limit: Optional[int] = Query(default=0, description='Listings Per Page'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
group_ids = [one for one in group_ids if one]
|
||||
operator_ids = [one for one in operator_ids if one]
|
||||
return await AuditLogService.get_audit_log(login_user, group_ids, operator_ids,
|
||||
start_time, end_time, system_id, event_type, page, limit)
|
||||
|
||||
|
||||
@router.get('/operators')
|
||||
def get_all_operators(*, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get all users who have acted on a resource under a group
|
||||
"""
|
||||
return resp_200(data=AuditLogService.get_all_operators(login_user))
|
||||
|
||||
|
||||
@router.get('/session')
|
||||
async def get_session_list(login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
flow_ids: Optional[List[str]] = Query(default=[], description='ApplicationsidVertical'),
|
||||
user_ids: Optional[List[int]] = Query(default=[], description='UsersidVertical'),
|
||||
group_ids: Optional[List[int]] = Query(default=[], description='User GroupsidVertical'),
|
||||
start_date: Optional[datetime] = Query(default=None, description='Start when'),
|
||||
end_date: Optional[datetime] = Query(default=None, description='End time'),
|
||||
feedback: Optional[str] = Query(default=None,
|
||||
description='like LikedislikeUnlikecopiedCopy:'),
|
||||
sensitive_status: Optional[int] = Query(default=None,
|
||||
description='Sensitive word review status'),
|
||||
page: Optional[int] = Query(default=1, description='Page'),
|
||||
page_size: Optional[int] = Query(default=10, description='Listings Per Page')):
|
||||
""" Filter all session lists """
|
||||
data, total = await AuditLogService.get_session_list(login_user, flow_ids, user_ids, group_ids, start_date,
|
||||
end_date,
|
||||
feedback, sensitive_status, page, page_size)
|
||||
return resp_200(data={
|
||||
'data': data,
|
||||
'total': total
|
||||
})
|
||||
|
||||
|
||||
@router.get('/session/export/data')
|
||||
async def get_session_messages(login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
flow_ids: Optional[List[str]] = Query(default=[], description='ApplicationsidVertical'),
|
||||
user_ids: Optional[List[int]] = Query(default=[], description='UsersidVertical'),
|
||||
group_ids: Optional[List[int]] = Query(default=[], description='User GroupsidVertical'),
|
||||
start_date: Optional[datetime] = Query(default=None, description='Start when'),
|
||||
end_date: Optional[datetime] = Query(default=None, description='End time'),
|
||||
feedback: Optional[str] = Query(default=None,
|
||||
description='like LikedislikeUnlikecopiedCopy:'),
|
||||
sensitive_status: Optional[int] = Query(default=None,
|
||||
description='Sensitive word review status')):
|
||||
""" Export data for a list of session details """
|
||||
result = await AuditLogService.get_session_messages(login_user, flow_ids, user_ids, group_ids, start_date, end_date,
|
||||
feedback, sensitive_status)
|
||||
return resp_200(data={
|
||||
'data': result
|
||||
})
|
||||
@@ -0,0 +1,577 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
from queue import Queue
|
||||
from typing import Any, Dict, List, Union
|
||||
|
||||
from fastapi import WebSocket
|
||||
from langchain.callbacks.base import AsyncCallbackHandler, BaseCallbackHandler
|
||||
from langchain.schema import AgentFinish, LLMResult
|
||||
from langchain.schema.agent import AgentAction
|
||||
from langchain.schema.document import Document
|
||||
from langchain.schema.messages import BaseMessage
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from bisheng.api.v1.schemas import ChatResponse
|
||||
from bisheng.database.models.message import ChatMessage as ChatMessageModel
|
||||
from bisheng.database.models.message import ChatMessageDao
|
||||
from loguru import logger
|
||||
|
||||
|
||||
# https://github.com/hwchase17/chat-langchain/blob/master/callback.py
|
||||
class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler):
|
||||
"""Callback handler for streaming LLM responses."""
|
||||
|
||||
def __init__(self,
|
||||
websocket: WebSocket,
|
||||
flow_id: str,
|
||||
chat_id: str,
|
||||
user_id: int = None,
|
||||
**kwargs: Any):
|
||||
self.websocket = websocket
|
||||
self.flow_id = flow_id
|
||||
self.chat_id = chat_id
|
||||
self.user_id = user_id
|
||||
|
||||
# Cache for tool calls intool_endWhen stitching the start and end together, store it in the database
|
||||
self.tool_cache = {}
|
||||
# self.tool_cache = {
|
||||
# 'run_id': {
|
||||
# 'input': {},
|
||||
# 'category': "",
|
||||
# }, # Storage tool callinputMessage
|
||||
# }
|
||||
|
||||
# Queue for Streaming Output
|
||||
self.stream_queue: Queue = kwargs.get('stream_queue')
|
||||
|
||||
async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
|
||||
chunk = kwargs.get('chunk')
|
||||
# azureOccasionally returns aNone
|
||||
if token is None and chunk is None:
|
||||
return
|
||||
reasoning_content = getattr(chunk.message, 'additional_kwargs',
|
||||
{}).get('reasoning_content')
|
||||
if token is None:
|
||||
token = ''
|
||||
resp = ChatResponse(message={
|
||||
'content': token,
|
||||
'reasoning_content': reasoning_content
|
||||
},
|
||||
type='stream',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
# Streaming output is placed in a queue to facilitate recording of content to a database after interrupting the streaming output
|
||||
await self.websocket.send_json(resp.dict())
|
||||
if self.stream_queue:
|
||||
if reasoning_content:
|
||||
self.stream_queue.put({'type': 'reasoning', 'content': reasoning_content})
|
||||
if token:
|
||||
self.stream_queue.put({'type': 'answer', 'content': token})
|
||||
|
||||
async def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when LLM starts running."""
|
||||
logger.debug(f'llm_start prompts={prompts}')
|
||||
|
||||
async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> Any:
|
||||
"""Run when LLM ends running."""
|
||||
logger.debug(f'llm_end response={response}')
|
||||
|
||||
async def on_llm_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> Any:
|
||||
"""Run when LLM errors."""
|
||||
logger.debug(f'on_llm_error error={error} kwargs={kwargs}')
|
||||
|
||||
async def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when chain starts running."""
|
||||
logger.debug(f'on_chain_start inputs={inputs} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_chain_start flow_id={} input_dict={}', self.flow_id, inputs)
|
||||
|
||||
async def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> Any:
|
||||
"""Run when chain ends running."""
|
||||
logger.debug(f'on_chain_end outputs={outputs} kwargs={kwargs}')
|
||||
tmp_output = copy.deepcopy(outputs)
|
||||
if isinstance(tmp_output, dict):
|
||||
tmp_output.pop('source_documents', '')
|
||||
logger.info('k=s act=on_chain_end flow_id={} output_dict={}', self.flow_id, tmp_output)
|
||||
|
||||
async def on_chain_error(self, error: Union[Exception, KeyboardInterrupt],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when chain errors."""
|
||||
logger.debug(f'on_chain_error error={error} kwargs={kwargs}')
|
||||
|
||||
async def on_tool_start(self, serialized: Dict[str, Any], input_str: str,
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when tool starts running."""
|
||||
logger.debug(
|
||||
f'on_tool_start serialized={serialized} input_str={input_str} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_tool_start flow_id={} tool_name={} input_str={}', self.flow_id,
|
||||
serialized.get('name'), input_str)
|
||||
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=f'Tool input: {input_str}',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_tool_end(self, output: str, **kwargs: Any) -> Any:
|
||||
"""Run when tool ends running."""
|
||||
logger.debug(f'on_tool_end output={output} kwargs={kwargs}')
|
||||
logger.info("k=s act=on_tool_end flow_id={} output='{}'", self.flow_id, output)
|
||||
observation_prefix = kwargs.get('observation_prefix', 'Tool output: ')
|
||||
# from langchain.docstore.document import Document # noqa
|
||||
# result = eval(output).get('result')
|
||||
result = output if isinstance(output, str) else getattr(output, 'content', output)
|
||||
|
||||
# Create a formatted message.
|
||||
intermediate_steps = f'{observation_prefix}{result[:100]}'
|
||||
|
||||
# Create a ChatResponse instance.
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=intermediate_steps,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
|
||||
try:
|
||||
# This is to emulate the stream of tokens
|
||||
await self.websocket.send_json(resp.dict())
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
|
||||
async def on_tool_error(self, error: Union[Exception, KeyboardInterrupt],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when tool errors."""
|
||||
logger.debug(f'on_tool_error error={error} kwargs={kwargs}')
|
||||
|
||||
async def on_text(self, text: str, **kwargs: Any) -> Any:
|
||||
"""Run on arbitrary text."""
|
||||
# This runs when first sending the prompt
|
||||
# to the LLM, adding it will send the final prompt
|
||||
# to the frontend
|
||||
logger.debug(f'on_text text={text} kwargs={kwargs}')
|
||||
if 'Prompt after formatting:' in text:
|
||||
prompt_str = text[24:]
|
||||
logger.info(
|
||||
"k=s act=on_text prompt='{}'",
|
||||
prompt_str,
|
||||
)
|
||||
sender = kwargs.get('sender')
|
||||
receiver = kwargs.get('receiver')
|
||||
if kwargs.get('sender'):
|
||||
log = ChatResponse(message=text,
|
||||
type='end',
|
||||
sender=sender,
|
||||
receiver=receiver,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
start = ChatResponse(type='start',
|
||||
sender=sender,
|
||||
receiver=receiver,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
|
||||
if receiver and receiver.get('is_self'):
|
||||
await self.websocket.send_json(log.dict())
|
||||
else:
|
||||
await self.websocket.send_json(log.dict())
|
||||
await self.websocket.send_json(start.dict())
|
||||
elif 'category' in kwargs:
|
||||
if 'autogen' == kwargs['category']:
|
||||
log = ChatResponse(message=text,
|
||||
type='stream',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(log.dict())
|
||||
if kwargs.get('type'):
|
||||
# Under compatibility
|
||||
start = ChatResponse(type='start',
|
||||
category=kwargs.get('type'),
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
end = ChatResponse(type='end',
|
||||
intermediate_steps=text,
|
||||
category=kwargs.get('type'),
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(start.dict())
|
||||
await self.websocket.send_json(end.dict())
|
||||
else:
|
||||
log = ChatResponse(message=text,
|
||||
intermediate_steps=kwargs['log'],
|
||||
type=kwargs['type'],
|
||||
category=kwargs['category'],
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(log.dict())
|
||||
|
||||
async def on_agent_action(self, action: AgentAction, **kwargs: Any):
|
||||
logger.debug(f'on_agent_action action={action} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_agent_action {}', action)
|
||||
log = f'\nThought: {action.log}'
|
||||
# if there are line breaks, split them and send them
|
||||
# as separate messages
|
||||
log = log.replace('\n', '\n\n')
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=log,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> Any:
|
||||
"""Run on agent end."""
|
||||
logger.debug(f'on_agent_finish finish={finish} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_agent_finish {}', finish)
|
||||
resp = ChatResponse(flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
type='stream',
|
||||
intermediate_steps=finish.log)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_retriever_start(self, serialized: Dict[str, Any], query: str,
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when retriever start running."""
|
||||
logger.debug(f'on_retriever_start serialized={serialized} query={query} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_retriever_start flow_id={} query={} meta={}', self.flow_id, query,
|
||||
serialized.get('repr'))
|
||||
|
||||
async def on_retriever_end(self, result: List[Document], **kwargs: Any) -> Any:
|
||||
"""Run when retriever end running."""
|
||||
# todo Determine skill permissions
|
||||
logger.debug(f'on_retriever_end result={result} kwargs={kwargs}')
|
||||
if result:
|
||||
tmp_result = copy.deepcopy(result)
|
||||
[doc.metadata.pop('bbox', '') for doc in tmp_result]
|
||||
logger.info('k=s act=on_retriever_end flow_id={} result_without_bbox={}', self.flow_id,
|
||||
tmp_result)
|
||||
|
||||
async def on_chat_model_start(self, serialized: Dict[str, Any],
|
||||
messages: List[List[BaseMessage]], **kwargs: Any) -> Any:
|
||||
# """Run when retriever end running."""
|
||||
# content = messages[0][0] if isinstance(messages[0][0], str) else messages[0][0].get('content')
|
||||
# stream = ChatResponse(message=f'{content}', type='stream')
|
||||
# await self.websocket.send_json(stream.dict())
|
||||
logger.debug(
|
||||
f'on_chat_model_start serialized={serialized} messages={messages} kwargs={kwargs}')
|
||||
logger.info('k=s act=on_chat_model_start messages={}', messages)
|
||||
|
||||
|
||||
class StreamingLLMCallbackHandler(BaseCallbackHandler):
|
||||
"""Callback handler for streaming LLM responses."""
|
||||
|
||||
def __init__(self,
|
||||
websocket: WebSocket,
|
||||
flow_id: str,
|
||||
chat_id: str,
|
||||
user_id: int = None,
|
||||
**kwargs: Any):
|
||||
self.websocket = websocket
|
||||
self.flow_id = flow_id
|
||||
self.chat_id = chat_id
|
||||
self.user_id = user_id
|
||||
|
||||
self.stream_queue: Queue = kwargs.get('stream_queue')
|
||||
|
||||
def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
|
||||
# azureOccasionally returns aNone
|
||||
if token is None:
|
||||
return
|
||||
resp = ChatResponse(message=token,
|
||||
type='stream',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
if self.websocket:
|
||||
loop = asyncio.get_event_loop()
|
||||
coroutine = self.websocket.send_json(resp.dict())
|
||||
asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
|
||||
if self.stream_queue:
|
||||
self.stream_queue.put(token)
|
||||
|
||||
def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
|
||||
log = f'\nThought: {action.log}'
|
||||
# if there are line breaks, split them and send them
|
||||
# as separate messages
|
||||
log = log.replace('\n', '\n\n')
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=log,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
if self.websocket:
|
||||
loop = asyncio.get_event_loop()
|
||||
coroutine = self.websocket.send_json(resp.dict())
|
||||
asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
logger.info('k=s act=on_agent_action {}', action)
|
||||
|
||||
def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> Any:
|
||||
"""Run on agent end."""
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=finish.log,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
if self.websocket:
|
||||
loop = asyncio.get_event_loop()
|
||||
coroutine = self.websocket.send_json(resp.dict())
|
||||
asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
logger.info('k=s act=on_agent_finish {}', finish)
|
||||
|
||||
def on_tool_start(self, serialized: Dict[str, Any], input_str: str, **kwargs: Any) -> Any:
|
||||
"""Run when tool starts running."""
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=f'Tool input: {input_str}',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
if self.websocket:
|
||||
loop = asyncio.get_event_loop()
|
||||
coroutine = self.websocket.send_json(resp.dict())
|
||||
asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
logger.info('k=s act=on_tool_start flow_id={} tool_name={} input_str={}', self.flow_id,
|
||||
serialized.get('name'), input_str)
|
||||
|
||||
def on_tool_end(self, output: str, **kwargs: Any) -> Any:
|
||||
"""Run when tool ends running."""
|
||||
observation_prefix = kwargs.get('observation_prefix', 'Tool output: ')
|
||||
|
||||
# from langchain.docstore.document import Document # noqa
|
||||
# result = eval(output).get('result')
|
||||
result = output if isinstance(output, str) else getattr(output, 'content', output)
|
||||
# Create a formatted message.
|
||||
intermediate_steps = f'{observation_prefix}{result}'
|
||||
|
||||
# Create a ChatResponse instance.
|
||||
resp = ChatResponse(type='stream',
|
||||
intermediate_steps=intermediate_steps,
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
|
||||
# Try to send the response, handle potential errors.
|
||||
try:
|
||||
if self.websocket:
|
||||
loop = asyncio.get_event_loop()
|
||||
coroutine = self.websocket.send_json(resp.dict())
|
||||
asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
logger.info("k=s act=on_tool_end flow_id={} output='{}'", self.flow_id, output)
|
||||
|
||||
def on_retriever_start(self, serialized: Dict[str, Any], query: str, **kwargs: Any) -> Any:
|
||||
"""Run when retriever start running."""
|
||||
logger.info('k=s act=on_retriever_start flow_id={} query={} meta={}', self.flow_id, query,
|
||||
serialized.get('repr'))
|
||||
|
||||
def on_retriever_end(self, result: List[Document], **kwargs: Any) -> Any:
|
||||
"""Run when retriever end running."""
|
||||
# todo Determine skill permissions
|
||||
logger.debug(f'retriver_result result={result}')
|
||||
if result:
|
||||
tmp_result = copy.deepcopy(result)
|
||||
[doc.metadata.pop('bbox', '') for doc in tmp_result]
|
||||
logger.info('k=s act=on_retriever_end flow_id={} result_without_bbox={}', self.flow_id,
|
||||
tmp_result)
|
||||
|
||||
def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when chain starts running."""
|
||||
logger.debug(f'on_chain_start inputs={inputs}')
|
||||
logger.info('k=s act=on_chain_start flow_id={} input_dict={}', self.flow_id, inputs)
|
||||
|
||||
def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> Any:
|
||||
"""Run when chain ends running."""
|
||||
logger.debug(f'on_chain_end outputs={outputs}')
|
||||
tmp_output = copy.deepcopy(outputs)
|
||||
if isinstance(tmp_output, dict):
|
||||
tmp_output.pop('source_documents', '')
|
||||
logger.info('k=s act=on_chain_end flow_id={} output_dict={}', self.flow_id, tmp_output)
|
||||
|
||||
def on_chat_model_start(self, serialized: Dict[str, Any], messages: List[List[BaseMessage]],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when retriever end running."""
|
||||
# sender = kwargs['sender']
|
||||
# receiver = kwargs['receiver']
|
||||
# content = messages[0][0] if isinstance(messages[0][0], str) else messages[0][0].get('content')
|
||||
# end = ChatResponse(message=f'{content}', type='end', sender=sender, recevier=receiver)
|
||||
# start = ChatResponse(type='start', sender=sender, recevier=receiver)
|
||||
# loop = asyncio.get_event_loop()
|
||||
# coroutine2 = self.websocket.send_json(end.dict())
|
||||
# coroutine3 = self.websocket.send_json(start.dict())
|
||||
# asyncio.run_coroutine_threadsafe(coroutine2, loop)
|
||||
# asyncio.run_coroutine_threadsafe(coroutine3, loop)
|
||||
logger.debug(f'on_chat result={messages}')
|
||||
logger.info('k=s act=on_chat_model_start messages={}', messages)
|
||||
|
||||
def on_text(self, text: str, **kwargs) -> Any:
|
||||
logger.info(text)
|
||||
if 'Prompt after formatting:' in text:
|
||||
prompt_str = text[24:]
|
||||
logger.info(
|
||||
"k=s act=on_text prompt='{}'",
|
||||
prompt_str,
|
||||
)
|
||||
|
||||
|
||||
class AsyncGptsLLMCallbackHandler(AsyncStreamingLLMCallbackHandler):
|
||||
|
||||
async def on_tool_start(self, serialized: Dict[str, Any], input_str: str,
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when tool starts running."""
|
||||
logger.debug(
|
||||
f'on_tool_start serialized={serialized} input_str={input_str} kwargs={kwargs}')
|
||||
pass
|
||||
|
||||
async def on_tool_end(self, output: str, **kwargs: Any) -> Any:
|
||||
"""Run when tool ends running."""
|
||||
logger.debug(f'on_tool_end output={output} kwargs={kwargs}')
|
||||
pass
|
||||
|
||||
|
||||
class AsyncGptsDebugCallbackHandler(AsyncGptsLLMCallbackHandler):
|
||||
|
||||
@staticmethod
|
||||
def parse_tool_category(tool_name) -> (str, str):
|
||||
"""
|
||||
will betool_nameResolve totool_categoryand the realtool_name
|
||||
"""
|
||||
tool_category = 'tool'
|
||||
if tool_name.startswith('flow_'):
|
||||
# Description is a skill call
|
||||
tool_category = 'flow'
|
||||
tool_name = tool_name.replace('flow_', '')
|
||||
elif tool_name.startswith('knowledge_'):
|
||||
# Description is a knowledge base call
|
||||
tool_category = 'knowledge'
|
||||
tool_name = tool_name.replace('knowledge_', '')
|
||||
return tool_name, tool_category
|
||||
|
||||
async def on_chat_model_start(self, serialized: Dict[str, Any],
|
||||
messages: List[List[BaseMessage]], **kwargs: Any) -> Any:
|
||||
# """Run when retriever end running."""
|
||||
# content = messages[0][0] if isinstance(messages[0][0], str) else messages[0][0].get('content')
|
||||
# stream = ChatResponse(message=f'{content}', type='stream')
|
||||
# await self.websocket.send_json(stream.dict())
|
||||
logger.debug(
|
||||
f'on_chat_model_start serialized={serialized} messages={messages} kwargs={kwargs}')
|
||||
resp = ChatResponse(type='start',
|
||||
category='processing',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> Any:
|
||||
"""Run when LLM ends running."""
|
||||
logger.debug(f'llm_end response={response}')
|
||||
resp = ChatResponse(type='end',
|
||||
category='processing',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_llm_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> Any:
|
||||
"""Run when LLM errors."""
|
||||
logger.debug(f'on_llm_error error={error} kwargs={kwargs}')
|
||||
resp = ChatResponse(type='end',
|
||||
category='processing',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id)
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_tool_start(self, serialized: Dict[str, Any], input_str: str,
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when tool starts running."""
|
||||
logger.debug(
|
||||
f'on_tool_start serialized={serialized} input_str={input_str} kwargs={kwargs}')
|
||||
|
||||
input_str = input_str
|
||||
tool_name, tool_category = self.parse_tool_category(serialized['name'])
|
||||
input_info = {'tool_key': tool_name, 'serialized': serialized, 'input_str': input_str}
|
||||
self.tool_cache[kwargs.get('run_id').hex] = {
|
||||
'input': input_info,
|
||||
'category': tool_category,
|
||||
'steps': f'Tool input: \n\n{input_str}\n\n',
|
||||
}
|
||||
resp = ChatResponse(type='start',
|
||||
category=tool_category,
|
||||
intermediate_steps=self.tool_cache[kwargs.get('run_id').hex]['steps'],
|
||||
message=json.dumps(input_info, ensure_ascii=False),
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
extra=json.dumps({'run_id': kwargs.get('run_id').hex}))
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
async def on_tool_end(self, output: ToolMessage, **kwargs: Any) -> Any:
|
||||
"""Run when tool ends running."""
|
||||
logger.debug(f'on_tool_end output={output} kwargs={kwargs}')
|
||||
observation_prefix = kwargs.get('observation_prefix', 'Tool output: ')
|
||||
|
||||
result = output if isinstance(output, str) else getattr(output, 'content', output)
|
||||
# Create a formatted message.
|
||||
intermediate_steps = f'{observation_prefix}\n\n{result}'
|
||||
tool_name, tool_category = self.parse_tool_category(kwargs.get('name'))
|
||||
|
||||
# Create a ChatResponse instance.
|
||||
output_info = {'tool_key': tool_name, 'output': result}
|
||||
resp = ChatResponse(type='end',
|
||||
category=tool_category,
|
||||
intermediate_steps=intermediate_steps,
|
||||
message=json.dumps(output_info, ensure_ascii=False),
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
extra=json.dumps({'run_id': kwargs.get('run_id').hex}))
|
||||
|
||||
await self.websocket.send_json(resp.dict())
|
||||
# FROMtool cacheGet ininputMessage
|
||||
input_info = self.tool_cache.get(kwargs.get('run_id').hex)
|
||||
if input_info:
|
||||
if not self.chat_id:
|
||||
# Explain that it is a debugging interface and does not need to persist data
|
||||
self.tool_cache.pop(kwargs.get('run_id').hex)
|
||||
return
|
||||
output_info.update(input_info['input'])
|
||||
intermediate_steps = f'{input_info["steps"]}\n\n{intermediate_steps}'
|
||||
ChatMessageDao.insert_one(
|
||||
ChatMessageModel(is_bot=1,
|
||||
message=json.dumps(output_info),
|
||||
intermediate_steps=intermediate_steps,
|
||||
category=tool_category,
|
||||
type='end',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
user_id=self.user_id,
|
||||
extra=json.dumps({'run_id': kwargs.get('run_id').hex})))
|
||||
self.tool_cache.pop(kwargs.get('run_id').hex)
|
||||
|
||||
async def on_tool_error(self, error: Union[Exception, KeyboardInterrupt],
|
||||
**kwargs: Any) -> Any:
|
||||
"""Run when tool errors."""
|
||||
logger.debug(f'on_tool_error error={error} kwargs={kwargs}')
|
||||
input_info = self.tool_cache.get(kwargs.get('run_id').hex)
|
||||
if input_info:
|
||||
output_info = {'output': 'Error: ' + str(error)}
|
||||
output_info.update(input_info['input'])
|
||||
resp = ChatResponse(type='end',
|
||||
category=input_info['category'],
|
||||
intermediate_steps='\n\nTool output:\n\n Error: ' + str(error),
|
||||
message=json.dumps(output_info, ensure_ascii=False),
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
extra=json.dumps({'run_id': kwargs.get('run_id').hex}))
|
||||
await self.websocket.send_json(resp.dict())
|
||||
|
||||
# Save tool call history
|
||||
if not self.chat_id:
|
||||
# Explain that it is a debugging interface and does not need to persist data
|
||||
self.tool_cache.pop(kwargs.get('run_id').hex)
|
||||
return
|
||||
tool_name, tool_category = self.parse_tool_category(kwargs.get('name'))
|
||||
self.tool_cache.pop(kwargs.get('run_id').hex)
|
||||
ChatMessageDao.insert_one(
|
||||
ChatMessageModel(
|
||||
is_bot=1,
|
||||
message=json.dumps(output_info),
|
||||
intermediate_steps=f'{input_info["steps"]}\n\nTool output:\n\n Error: ' +
|
||||
str(error),
|
||||
category=tool_category,
|
||||
type='end',
|
||||
flow_id=self.flow_id,
|
||||
chat_id=self.chat_id,
|
||||
user_id=self.user_id,
|
||||
extra=json.dumps({'run_id': kwargs.get('run_id').hex})))
|
||||
@@ -0,0 +1,51 @@
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.params import Depends
|
||||
|
||||
from bisheng.api.services.workflow import WorkFlowService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.chat.manager import ChatManager
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.database.models.flow import FlowStatus
|
||||
from bisheng.database.models.session import MessageSessionDao
|
||||
|
||||
router = APIRouter(tags=['Chat'])
|
||||
chat_manager = ChatManager()
|
||||
|
||||
|
||||
@router.get('/chat/online')
|
||||
async def get_online_chat(*,
|
||||
keyword: Optional[str] = None,
|
||||
tag_id: Optional[int] = None,
|
||||
page: Optional[int] = 1,
|
||||
limit: Optional[int] = 10,
|
||||
user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""Access to online workflows and assistants."""
|
||||
data, total = await asyncio.to_thread(
|
||||
WorkFlowService.get_all_flows,
|
||||
user, keyword, FlowStatus.ONLINE.value, tag_id, None, page, limit,
|
||||
skip_pagination=True)
|
||||
|
||||
# Get user's last conversation time per app
|
||||
used_apps = await MessageSessionDao.get_user_used_apps(use_create_time=True)
|
||||
used_map = {app[0]: app[1] for app in used_apps}
|
||||
|
||||
# Sort: apps with conversations first (by last used time DESC),
|
||||
# then apps without (by update_time DESC)
|
||||
def sort_key(app):
|
||||
last_chat = used_map.get(app['id'])
|
||||
if last_chat:
|
||||
return (0, -last_chat.timestamp())
|
||||
return (1, -app['update_time'].timestamp() if app.get('update_time') else 0)
|
||||
|
||||
data.sort(key=sort_key)
|
||||
|
||||
# Manual pagination
|
||||
total = len(data)
|
||||
start_index = (page - 1) * limit
|
||||
end_index = start_index + limit
|
||||
data = data[start_index:end_index]
|
||||
|
||||
return resp_200(data=data)
|
||||
@@ -0,0 +1,52 @@
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from bisheng.api.services.dataset_service import DatasetService
|
||||
from bisheng.api.v1.schema.dataset_param import CreateDatasetParam
|
||||
from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.database.models.dataset import DatasetRead
|
||||
|
||||
# build router
|
||||
router = APIRouter(prefix='/dataset', tags=['FineTune'])
|
||||
|
||||
|
||||
@router.get('/list', summary='Get dataset list')
|
||||
def list_dataset(*,
|
||||
keyword: str = None,
|
||||
page: int = 1,
|
||||
limit: int = 10) -> UnifiedResponseModel[List[DatasetRead]]:
|
||||
"""
|
||||
Get dataset list
|
||||
"""
|
||||
res, count = DatasetService.build_dataset_list(page, limit, keyword)
|
||||
return resp_200(data={'list': res, 'total': count})
|
||||
|
||||
|
||||
@router.post('/create', summary='Create Dataset')
|
||||
def create_dataset(
|
||||
*,
|
||||
request: Request,
|
||||
data: CreateDatasetParam,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
) -> UnifiedResponseModel:
|
||||
"""
|
||||
Create Dataset
|
||||
"""
|
||||
dataset = DatasetService.create_dataset(login_user.user_id, data)
|
||||
return resp_200(data=dataset)
|
||||
|
||||
|
||||
@router.delete('/del', summary='Delete Dataset')
|
||||
def delete_dataset(
|
||||
*,
|
||||
request: Request,
|
||||
dataset_id: int,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
) -> UnifiedResponseModel:
|
||||
"""
|
||||
Create Dataset
|
||||
"""
|
||||
DatasetService.delete_dataset(dataset_id)
|
||||
return resp_200()
|
||||
@@ -0,0 +1,202 @@
|
||||
import copy
|
||||
import time
|
||||
from typing import List
|
||||
|
||||
import yaml
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Path, Request, UploadFile
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.v1.schemas import (UploadFileResponse,
|
||||
resp_200)
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.server import SystemConfigEmptyError, SystemConfigInvalidError, UploadFileEmptyError, \
|
||||
UploadFileExtError
|
||||
from bisheng.common.models.config import Config, ConfigDao, ConfigKeyEnum
|
||||
from bisheng.common.services.config_service import settings as bisheng_settings
|
||||
from bisheng.core.cache.redis_manager import get_redis_client_sync
|
||||
from bisheng.core.cache.utils import save_uploaded_file, upload_file_to_minio
|
||||
from bisheng.utils import generate_uuid
|
||||
from bisheng.utils import get_request_ip
|
||||
|
||||
# build router
|
||||
router = APIRouter(tags=['Base'])
|
||||
|
||||
if bisheng_settings.debug:
|
||||
import tracemalloc
|
||||
import os
|
||||
import threading
|
||||
|
||||
|
||||
@router.get("/tracemalloc")
|
||||
def tracemalloc_point():
|
||||
snapshot = tracemalloc.take_snapshot()
|
||||
process_id = os.getpid()
|
||||
thread_id = threading.get_ident()
|
||||
snapshot.dump(f"/app/data/snapshot_{process_id}_{thread_id}_{time.time()}.prof")
|
||||
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.get('/env')
|
||||
def get_env():
|
||||
from bisheng import __version__
|
||||
"""Get environment variable parameters"""
|
||||
uns_support = ['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf', 'csv']
|
||||
|
||||
etl_for_lm_url = bisheng_settings.get_knowledge().etl4lm.url
|
||||
if etl_for_lm_url:
|
||||
uns_support.extend(['png', 'jpg', 'jpeg', 'bmp'])
|
||||
|
||||
env = {}
|
||||
if isinstance(bisheng_settings.environment, str):
|
||||
env['env'] = bisheng_settings.environment
|
||||
else:
|
||||
env = copy.deepcopy(bisheng_settings.environment)
|
||||
|
||||
env['uns_support'] = uns_support
|
||||
if bisheng_settings.get_from_db('office_url'):
|
||||
env['office_url'] = bisheng_settings.get_from_db('office_url')
|
||||
# add tips from settings
|
||||
env['dialog_tips'] = bisheng_settings.get_from_db('dialog_tips')
|
||||
# add env dict from settings
|
||||
env.update(bisheng_settings.get_from_db('env') or {})
|
||||
env['pro'] = bisheng_settings.get_system_login_method().bisheng_pro
|
||||
env['dashboard_pro'] = bisheng_settings.get_system_login_method().dashboard_pro
|
||||
env['version'] = __version__
|
||||
env['enable_etl4lm'] = bool(etl_for_lm_url)
|
||||
|
||||
return resp_200(env)
|
||||
|
||||
|
||||
@router.get('/config')
|
||||
def get_config(admin_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
db_config = ConfigDao.get_config(ConfigKeyEnum.INIT_DB)
|
||||
config_str = db_config.value if db_config else ''
|
||||
return resp_200(config_str)
|
||||
|
||||
|
||||
@router.post('/config/save')
|
||||
def save_config(data: dict, admin_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
if not data.get('data', '').strip():
|
||||
raise SystemConfigEmptyError()
|
||||
try:
|
||||
# Check for complianceyamlFormat
|
||||
config = yaml.safe_load(data.get('data'))
|
||||
|
||||
# Judging linsight_invitation_code Right?boolean
|
||||
if isinstance(config, dict) and 'linsight_invitation_code' in config.keys():
|
||||
if config['linsight_invitation_code'] is not None and bool(config['linsight_invitation_code']) not in [True,
|
||||
False]:
|
||||
raise ValueError('linsight_invitation_code must be a boolean value')
|
||||
|
||||
db_config = ConfigDao.get_config(ConfigKeyEnum.INIT_DB)
|
||||
db_config.value = data.get('data')
|
||||
ConfigDao.insert_config(db_config)
|
||||
get_redis_client_sync().delete('config:initdb_config')
|
||||
except Exception as e:
|
||||
raise SystemConfigInvalidError()
|
||||
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.get('/web/config')
|
||||
async def get_web_config():
|
||||
""" Get some configuration items required by the front-end, the content is determined by the front-end """
|
||||
web_conf = ConfigDao.get_config(ConfigKeyEnum.WEB_CONFIG)
|
||||
if not web_conf:
|
||||
return resp_200(data='')
|
||||
return resp_200(data={'value': web_conf.value})
|
||||
|
||||
|
||||
@router.post('/web/config')
|
||||
async def update_web_config(request: Request,
|
||||
admin_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
value: str = Body(embed=True)):
|
||||
""" Update some configuration items required by the front-end, the content is determined by the front-end """
|
||||
logger.info(
|
||||
f'update_web_config user_name={admin_user.user_name}, ip={get_request_ip(request)}')
|
||||
web_conf = ConfigDao.get_config(ConfigKeyEnum.WEB_CONFIG)
|
||||
if not web_conf:
|
||||
web_conf = Config(key=ConfigKeyEnum.WEB_CONFIG.value, value=value)
|
||||
else:
|
||||
web_conf.value = value
|
||||
ConfigDao.insert_config(web_conf)
|
||||
return resp_200(data={'value': web_conf.value})
|
||||
|
||||
|
||||
async def _upload_file(file: UploadFile, object_name_prefix: str, file_supports: List[str] = None,
|
||||
bucket_name: str = None) \
|
||||
-> UploadFileResponse:
|
||||
if file.size == 0:
|
||||
raise UploadFileEmptyError()
|
||||
file_ext = file.filename.split('.')[-1].lower()
|
||||
if file_supports and file_ext not in file_supports:
|
||||
raise UploadFileExtError()
|
||||
object_name = f'{object_name_prefix}/{generate_uuid()}.png'
|
||||
file_path = await upload_file_to_minio(file, object_name=object_name, bucket_name=bucket_name)
|
||||
if not isinstance(file_path, str):
|
||||
file_path = str(file_path)
|
||||
|
||||
return UploadFileResponse(
|
||||
file_path=file_path, # minioAccessible links
|
||||
relative_path=object_name, # miniohitting the nail on the headobject_name
|
||||
)
|
||||
|
||||
|
||||
@router.post('/upload/icon')
|
||||
async def upload_icon(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
file: UploadFile = None):
|
||||
try:
|
||||
bucket = bisheng_settings.object_storage.minio.public_bucket
|
||||
resp = await _upload_file(file,
|
||||
object_name_prefix='icon',
|
||||
file_supports=['jpeg', 'jpg', 'png'],
|
||||
bucket_name=bucket)
|
||||
return resp_200(data=resp)
|
||||
except Exception as e:
|
||||
raise e
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
@router.post('/upload/workflow/{workflow_id}')
|
||||
async def upload_icon_workflow(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
file: UploadFile = None,
|
||||
workflow_id: str = Path(..., description='workflow id')):
|
||||
try:
|
||||
bucket = bisheng_settings.object_storage.minio.public_bucket
|
||||
resp = await _upload_file(file, object_name_prefix=f'workflow/{workflow_id}', bucket_name=bucket)
|
||||
return resp_200(data=resp)
|
||||
except Exception as e:
|
||||
raise e
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
@router.post('/upload/{flow_id}')
|
||||
async def create_upload_file(file: UploadFile, flow_id: str):
|
||||
# Cache file
|
||||
try:
|
||||
if len(file.filename) > 80:
|
||||
file.filename = file.filename[-80:]
|
||||
file_path = await save_uploaded_file(file, folder_name=flow_id, file_name=file.filename)
|
||||
if not isinstance(file_path, str):
|
||||
file_path = str(file_path)
|
||||
return resp_200(UploadFileResponse(
|
||||
flowId=flow_id,
|
||||
file_path=file_path,
|
||||
))
|
||||
except Exception as exc:
|
||||
logger.error(f'Error saving file: {exc}')
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
# get endpoint to return version of bisheng
|
||||
@router.get('/version')
|
||||
def get_version():
|
||||
from bisheng import __version__
|
||||
return resp_200({'version': __version__})
|
||||
@@ -0,0 +1,103 @@
|
||||
import io
|
||||
from typing import Optional
|
||||
|
||||
from datasets import Dataset
|
||||
from fastapi import APIRouter, Depends, Query, UploadFile, Form, BackgroundTasks
|
||||
|
||||
from bisheng.api.services.evaluation import EvaluationService, add_evaluation_task
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.server import UploadFileExtError
|
||||
from bisheng.core.cache.utils import convert_encoding_cchardet
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage
|
||||
from bisheng.database.models.evaluation import EvaluationCreate, Evaluation
|
||||
|
||||
router = APIRouter(prefix='/evaluation', tags=['Evaluation'], dependencies=[Depends(UserPayload.get_login_user)])
|
||||
|
||||
|
||||
@router.get('')
|
||||
def get_evaluation(*,
|
||||
page: Optional[int] = Query(default=1, gt=0, description='Page'),
|
||||
limit: Optional[int] = Query(default=10, gt=0, description='Listings Per Page'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Get a list of assessment tasks. """
|
||||
return EvaluationService.get_evaluation(login_user, page, limit)
|
||||
|
||||
|
||||
@router.post('')
|
||||
def create_evaluation(*,
|
||||
file: UploadFile,
|
||||
prompt: str = Form(),
|
||||
exec_type: str = Form(),
|
||||
unique_id: str = Form(),
|
||||
version: Optional[int | str] = Form(default=None),
|
||||
background_tasks: BackgroundTasks,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Create Assessment Task. """
|
||||
try:
|
||||
user_id = login_user.user_id
|
||||
if not version:
|
||||
version = 0
|
||||
|
||||
try:
|
||||
# Try transcoding
|
||||
output_file = convert_encoding_cchardet(file.file.read(), 'utf-8')
|
||||
csv_data = EvaluationService.parse_csv(file_data=output_file)
|
||||
data_samples = {
|
||||
"question": [one.get('question') for one in csv_data],
|
||||
"answer": [one.get('answer') for one in csv_data],
|
||||
"ground_truths": [[one.get('ground_truth')] for one in csv_data]
|
||||
}
|
||||
dataset = Dataset.from_dict(data_samples)
|
||||
except Exception:
|
||||
raise UploadFileExtError()
|
||||
finally:
|
||||
file.file.seek(0)
|
||||
|
||||
file_name, file_path = EvaluationService.upload_file(file=file)
|
||||
db_evaluation = Evaluation.model_validate(EvaluationCreate(unique_id=unique_id,
|
||||
exec_type=exec_type,
|
||||
version=version,
|
||||
prompt=prompt,
|
||||
user_id=user_id,
|
||||
file_name=file_name,
|
||||
file_path=file_path))
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_evaluation)
|
||||
session.commit()
|
||||
session.refresh(db_evaluation)
|
||||
|
||||
background_tasks.add_task(add_evaluation_task, evaluation_id=db_evaluation.id)
|
||||
|
||||
return resp_200(db_evaluation.copy())
|
||||
except Exception as e:
|
||||
raise e
|
||||
finally:
|
||||
file.file.close()
|
||||
|
||||
|
||||
@router.delete('/{evaluation_id}', status_code=200)
|
||||
def delete_evaluation(*, evaluation_id: int, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Delete Assessment Task (Logical Delete). """
|
||||
return EvaluationService.delete_evaluation(evaluation_id, user_payload=login_user)
|
||||
|
||||
|
||||
@router.get('/result/file/download')
|
||||
async def get_download_url(*,
|
||||
file_url: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Get file download address. """
|
||||
minio_client = await get_minio_storage()
|
||||
download_url = await minio_client.get_share_link(file_url)
|
||||
return resp_200(data={
|
||||
'url': download_url
|
||||
})
|
||||
|
||||
|
||||
@router.post('/{evaluation_id}/process', status_code=200)
|
||||
def process_evaluation(*, evaluation_id: int, background_tasks: BackgroundTasks,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
""" Perform assessment tasks manually. """
|
||||
background_tasks.add_task(add_evaluation_task, evaluation_id=evaluation_id)
|
||||
return resp_200()
|
||||
@@ -0,0 +1,48 @@
|
||||
from typing import Union
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from bisheng.api.services.flow import FlowService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import NotFoundError, UnAuthorizedError
|
||||
from bisheng.common.services import telemetry_service
|
||||
from bisheng.core.logger import trace_id_var
|
||||
from bisheng.database.models.flow import FlowDao
|
||||
from bisheng.database.models.role_access import AccessType
|
||||
from bisheng.share_link.api.dependencies import header_share_token_parser
|
||||
from bisheng.share_link.domain.models.share_link import ShareLink
|
||||
|
||||
# build router
|
||||
router = APIRouter(prefix='/flows', tags=['Flows'], dependencies=[Depends(UserPayload.get_login_user)])
|
||||
|
||||
|
||||
@router.get('/{flow_id}')
|
||||
async def read_flow(*, flow_id: str, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
share_link: Union['ShareLink', None] = Depends(header_share_token_parser)):
|
||||
"""Read a flow."""
|
||||
return await FlowService.get_one_flow(login_user, flow_id, share_link)
|
||||
|
||||
|
||||
@router.delete('/{flow_id}', status_code=200)
|
||||
def delete_flow(*,
|
||||
request: Request,
|
||||
flow_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""Delete a flow."""
|
||||
|
||||
db_flow = FlowDao.get_flow_by_id(flow_id)
|
||||
if not db_flow:
|
||||
raise NotFoundError()
|
||||
access_type = AccessType.WORKFLOW_WRITE
|
||||
if not login_user.access_check(db_flow.user_id, flow_id, access_type):
|
||||
return UnAuthorizedError.return_resp()
|
||||
FlowDao.delete_flow(db_flow)
|
||||
telemetry_service.log_event_sync(
|
||||
user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.DELETE_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
FlowService.delete_flow_hook(request, login_user, db_flow)
|
||||
return resp_200()
|
||||
@@ -0,0 +1,47 @@
|
||||
from fastapi import APIRouter, Depends, Body, Request
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.services.invite_code.invite_code import InviteCodeService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.utils import get_request_ip
|
||||
|
||||
router = APIRouter(prefix='/invite', tags=['InviteCode'])
|
||||
|
||||
|
||||
@router.post('/code')
|
||||
async def create_invite_code(request: Request, login_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
name: str = Body(..., description='Batch'),
|
||||
num: int = Body(..., description='Number of invitation codes in the current batch'),
|
||||
limit: int = Body(..., description='Current batch invite code usage limit')):
|
||||
"""
|
||||
Create an invite code
|
||||
"""
|
||||
logger.debug(
|
||||
f"create invite code user_id: {login_user.user_id}, ip: {get_request_ip(request)}, name: {name}, num: {num}, limit: {limit}")
|
||||
codes = await InviteCodeService.create_batch_invite_codes(login_user, name, num, limit)
|
||||
return resp_200(data={
|
||||
"name": name,
|
||||
"limit": limit,
|
||||
"codes": codes
|
||||
})
|
||||
|
||||
|
||||
@router.post('/bind')
|
||||
async def bind_invite_code(request: Request, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
code: str = Body(..., embed=True, description='Invitation Code')):
|
||||
"""
|
||||
Binding Invitation Code
|
||||
"""
|
||||
result = await InviteCodeService.bind_invite_code(login_user, code)
|
||||
logger.debug(f"bind_invite_code user_id:{login_user.user_id}, code:{code}, flag:{result}")
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.get('/code')
|
||||
async def get_bind_code_num(request: Request, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get the number of times a valid invitation code bound by a user can be used
|
||||
"""
|
||||
num = await InviteCodeService.get_invite_code_num(login_user)
|
||||
return resp_200(data=num)
|
||||
@@ -0,0 +1,260 @@
|
||||
from collections import deque
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.api.v1.schema.mark_schema import MarkData, MarkTaskCreate
|
||||
from bisheng.api.v1.schemas import resp_200, resp_500
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.database.models.mark_app_user import MarkAppUser, MarkAppUserDao
|
||||
from bisheng.database.models.mark_record import MarkRecord, MarkRecordDao
|
||||
from bisheng.database.models.mark_task import MarkTask, MarkTaskDao, MarkTaskRead, MarkTaskStatus
|
||||
from bisheng.database.models.message import ChatMessageDao
|
||||
from bisheng.database.models.session import MessageSessionDao
|
||||
from bisheng.database.models.user_group import UserGroupDao
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
from bisheng.utils.linked_list import DoubleLinkList
|
||||
|
||||
router = APIRouter(prefix='/mark', tags=['Mark'])
|
||||
|
||||
|
||||
@router.get('/list')
|
||||
def list(request: Request,
|
||||
status: Optional[int] = None,
|
||||
page_size: int = 10,
|
||||
page_num: int = 1,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Nonadmin Can only see their own marked and unlabeled
|
||||
"""
|
||||
groups = UserGroupDao.get_user_admin_group(login_user.user_id)
|
||||
if login_user.is_admin():
|
||||
task_list, count = MarkTaskDao.get_task_list(page_size=page_size, page_num=page_num, status=status,
|
||||
create_id=None, user_id=None)
|
||||
else:
|
||||
task_list, count = MarkTaskDao.get_task_list(page_size=page_size, page_num=page_num, status=status,
|
||||
create_id=login_user.user_id if groups else None,
|
||||
user_id=login_user.user_id)
|
||||
|
||||
result_list = []
|
||||
for task in task_list:
|
||||
record = MarkRecordDao.get_count(task.id)
|
||||
process_list = []
|
||||
user_count = {}
|
||||
|
||||
for c in task.process_users.split(","):
|
||||
user = UserDao.get_user(int(c))
|
||||
process_count = "{}:{}".format(user.user_name, 0)
|
||||
user_count[int(c)] = process_count
|
||||
|
||||
for c in record:
|
||||
process_count = "{}:{}".format(c.create_user, c.user_count)
|
||||
user_count[c.create_id] = process_count
|
||||
|
||||
for c in user_count:
|
||||
process_list.append(user_count[c])
|
||||
|
||||
result_list.append(MarkTaskRead(**task.model_dump(), mark_process=process_list))
|
||||
|
||||
result = {"list": result_list, "total": count}
|
||||
return resp_200(data=result)
|
||||
|
||||
|
||||
@router.get('/get_status')
|
||||
async def get_status(task_id: int, chat_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
record = MarkRecordDao.get_record(task_id, chat_id)
|
||||
if not record:
|
||||
return resp_200(data={"status": ""})
|
||||
|
||||
if login_user.user_id == record.create_id:
|
||||
is_self = True
|
||||
else:
|
||||
is_self = False
|
||||
result = {"status": record.status, "is_self": is_self}
|
||||
return resp_200(result)
|
||||
|
||||
|
||||
@router.post('/create_task')
|
||||
async def create(task_create: MarkTaskCreate, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Apps and users are in a many-to-many relationship, relying on one director record
|
||||
"""
|
||||
|
||||
task = MarkTask(create_id=login_user.user_id,
|
||||
create_user=login_user.user_name,
|
||||
app_id=",".join(task_create.app_list),
|
||||
process_users=",".join(task_create.user_list)
|
||||
)
|
||||
MarkTaskDao.create_task(task)
|
||||
|
||||
user_app = [MarkAppUser(task_id=task.id, create_id=login_user.user_id, app_id=app, user_id=user) for app in
|
||||
task_create.app_list for user in task_create.user_list]
|
||||
|
||||
MarkAppUserDao.create_task(user_app)
|
||||
return resp_200(data="ok")
|
||||
|
||||
|
||||
@router.get('/get_user')
|
||||
async def get_user(task_id: int):
|
||||
"""
|
||||
Query under this app All Users
|
||||
"""
|
||||
|
||||
# accordingtype Query different sessions
|
||||
task = MarkTaskDao.get_task_byid(task_id)
|
||||
user_list = []
|
||||
|
||||
for u in task.process_users.split(","):
|
||||
user = UserDao.get_user(int(u))
|
||||
user_list.append(user)
|
||||
|
||||
return resp_200(data=user_list)
|
||||
|
||||
|
||||
@router.post('/mark')
|
||||
async def mark(data: MarkData,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Flag task as current user and cannot be overwritten by others
|
||||
flow_type flow assistant
|
||||
"""
|
||||
|
||||
# record = MarkRecordDao.get_record(data.task_id,data.session_id)
|
||||
# if record:
|
||||
# return resp_500(data="Already flagged")
|
||||
|
||||
session_info = MessageSessionDao.get_one(data.session_id)
|
||||
if session_info:
|
||||
data.flow_type = session_info.flow_type
|
||||
|
||||
db_r = MarkRecordDao.get_record(data.task_id, data.session_id)
|
||||
if db_r:
|
||||
if data.status == MarkTaskStatus.DEFAULT.value:
|
||||
MarkRecordDao.del_task_chat(task_id=db_r.task_id, session_id=db_r.session_id)
|
||||
return resp_200(data="ok")
|
||||
db_r.status = data.status
|
||||
MarkRecordDao.update_record(db_r)
|
||||
else:
|
||||
# Not marked No data recorded
|
||||
if data.status == MarkTaskStatus.DEFAULT.value:
|
||||
return resp_200(data="ok")
|
||||
record_info = MarkRecord(create_user=login_user.user_name, create_id=login_user.user_id,
|
||||
session_id=data.session_id, task_id=data.task_id, status=data.status,
|
||||
flow_type=data.flow_type)
|
||||
# Create an article User callout record
|
||||
MarkRecordDao.create_record(record_info)
|
||||
|
||||
task = MarkTaskDao.get_task_byid(task_id=data.task_id)
|
||||
msg_list = ChatMessageDao.get_msg_by_flows(task.app_id.split(","))
|
||||
# m_list = [msg.chat_id for msg in msg_list]
|
||||
m_list = msg_list
|
||||
r_list = MarkRecordDao.get_list_by_taskid(data.task_id)
|
||||
app_record = [r.session_id for r in r_list]
|
||||
|
||||
m_list = [s.strip() for s in m_list if s.strip()]
|
||||
app_record = [s.strip() for s in app_record if s.strip()]
|
||||
|
||||
m_list.sort()
|
||||
app_record.sort()
|
||||
|
||||
logger.info("m_list={} app_record={}", m_list, app_record)
|
||||
|
||||
if m_list == app_record:
|
||||
MarkTaskDao.update_task(data.task_id, MarkTaskStatus.DONE.value)
|
||||
else:
|
||||
MarkTaskDao.update_task(data.task_id, MarkTaskStatus.ING.value)
|
||||
|
||||
return resp_200(data="ok")
|
||||
|
||||
|
||||
@router.get('/get_record')
|
||||
async def get_record(chat_id: str, task_id: int):
|
||||
record = MarkRecordDao.get_record(task_id, chat_id)
|
||||
return resp_200(data=record)
|
||||
|
||||
|
||||
@router.get("/next")
|
||||
async def pre_or_next(chat_id: str, action: str, task_id: int,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
prev or next
|
||||
"""
|
||||
|
||||
if action not in ["prev", "next"]:
|
||||
return resp_500(data="actionParameter salah")
|
||||
|
||||
result = {"task_id": task_id}
|
||||
|
||||
if action == "prev":
|
||||
record = MarkRecordDao.get_prev_task(login_user.user_id, task_id)
|
||||
top_queue = deque()
|
||||
bottom_queue = deque()
|
||||
if record:
|
||||
queue = top_queue
|
||||
for r in record:
|
||||
if r.session_id == chat_id:
|
||||
queue = bottom_queue
|
||||
continue
|
||||
queue.append(r)
|
||||
|
||||
logger.info("top_queue={} bottom_queue={}", top_queue, bottom_queue)
|
||||
if len(top_queue) == 0 and len(bottom_queue) == 0:
|
||||
return resp_200()
|
||||
record = bottom_queue.popleft() if len(bottom_queue) else top_queue.popleft()
|
||||
chat = MessageSessionDao.get_one(record.session_id)
|
||||
result["chat_id"] = chat.chat_id
|
||||
result["flow_type"] = chat.flow_type
|
||||
result["flow_id"] = chat.flow_id
|
||||
return resp_200(data=result)
|
||||
else:
|
||||
task = MarkTaskDao.get_task_byid(task_id)
|
||||
record = MarkRecordDao.get_list_by_taskid(task_id)
|
||||
chat_list = [r.session_id for r in record]
|
||||
|
||||
msg = MessageSessionDao.filter_session(flow_ids=task.app_id.split(","), exclude_chats=chat_list)
|
||||
linked = DoubleLinkList()
|
||||
k_list = {}
|
||||
for m in msg:
|
||||
k_list[m.chat_id] = m
|
||||
linked.append(m.chat_id)
|
||||
|
||||
cur = linked.find(chat_id)
|
||||
if not k_list:
|
||||
return resp_200()
|
||||
|
||||
logger.info("k_list={} cur={}", k_list, cur)
|
||||
|
||||
if cur:
|
||||
if cur.next is None:
|
||||
if linked.length() == 1 and linked.head().data == chat_id:
|
||||
return resp_200()
|
||||
cur = k_list[linked.head().data]
|
||||
else:
|
||||
cur = k_list[cur.next.data]
|
||||
|
||||
result["chat_id"] = cur.chat_id
|
||||
result["flow_id"] = cur.flow_id
|
||||
result['flow_type'] = cur.flow_type
|
||||
return resp_200(data=result)
|
||||
else:
|
||||
cur = k_list[linked.head().data]
|
||||
result['flow_type'] = cur.flow_type
|
||||
result["chat_id"] = cur.chat_id
|
||||
result["flow_id"] = cur.flow_id
|
||||
return resp_200(data=result)
|
||||
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.delete('/del')
|
||||
def del_task(request: Request, task_id: int, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Nonadmin Can only see their own marked and unlabeled
|
||||
"""
|
||||
|
||||
MarkTaskDao.delete_task(task_id)
|
||||
MarkRecordDao.del_record(task_id)
|
||||
|
||||
return resp_200(data="ok")
|
||||
@@ -0,0 +1,91 @@
|
||||
import jwt
|
||||
from fastapi import APIRouter, Body, HTTPException
|
||||
from loguru import logger
|
||||
from sqlalchemy import or_
|
||||
from sqlmodel import select
|
||||
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.services.config_service import settings as bisheng_settings
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage
|
||||
from bisheng.database.models.report import Report
|
||||
from bisheng.utils import generate_uuid
|
||||
from bisheng_langchain.utils.requests import Requests
|
||||
|
||||
# build router
|
||||
router = APIRouter(prefix='/report', tags=['report'])
|
||||
mino_prefix = 'report/'
|
||||
|
||||
|
||||
@router.post('/office_token')
|
||||
async def get_office_token(payload: dict = Body(...)):
|
||||
"""Sign the OnlyOffice editorConfig with JWT secret and return the token."""
|
||||
secret = bisheng_settings.get_from_db('office_jwt_secret') or ''
|
||||
if not secret:
|
||||
return resp_200({'token': ''})
|
||||
token = jwt.encode(payload, secret, algorithm='HS256')
|
||||
return resp_200({'token': token})
|
||||
|
||||
|
||||
@router.post('/callback')
|
||||
async def callback(data: dict):
|
||||
status = data.get('status')
|
||||
file_url = data.get('url')
|
||||
key = data.get('key')
|
||||
logger.debug(f'calback={data}')
|
||||
if status in {2, 6}:
|
||||
# Save Back
|
||||
logger.info(f'office_callback url={file_url}')
|
||||
file = Requests().get(url=file_url)
|
||||
object_name = mino_prefix + key + '.docx'
|
||||
minio_client = await get_minio_storage()
|
||||
await minio_client.put_object(bucket_name=minio_client.bucket,
|
||||
object_name=object_name, file=file._content,
|
||||
content_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document') # noqa
|
||||
# Duplicate save,key Data Update Error
|
||||
with get_sync_db_session() as session:
|
||||
db_report = session.exec(
|
||||
select(Report).where(or_(Report.version_key == key,
|
||||
Report.newversion_key == key))).first()
|
||||
if not db_report:
|
||||
logger.error(f'report_callback cannot find the flow_id flow_id={key}')
|
||||
raise HTTPException(status_code=500, detail='cannot find the flow_id')
|
||||
db_report.object_name = object_name
|
||||
db_report.version_key = key
|
||||
db_report.newversion_key = None
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_report)
|
||||
session.commit()
|
||||
return {'error': 0}
|
||||
|
||||
|
||||
@router.get('/report_temp')
|
||||
async def get_template(*, flow_id: str):
|
||||
with get_sync_db_session() as session:
|
||||
db_report = session.exec(
|
||||
select(Report).where(Report.flow_id == flow_id,
|
||||
Report.del_yn == 0).order_by(Report.update_time.desc())).first()
|
||||
file_url = ''
|
||||
if not db_report:
|
||||
db_report = Report(flow_id=flow_id)
|
||||
elif db_report.object_name:
|
||||
minio_client = await get_minio_storage()
|
||||
file_url = await minio_client.get_share_link(db_report.object_name, clear_host=False)
|
||||
|
||||
if not db_report.newversion_key or not db_report.object_name:
|
||||
version_key = generate_uuid()
|
||||
db_report.newversion_key = version_key
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_report)
|
||||
session.commit()
|
||||
session.refresh(db_report)
|
||||
else:
|
||||
version_key = db_report.newversion_key
|
||||
res = {
|
||||
'flow_id': flow_id,
|
||||
'temp_url': file_url,
|
||||
'original_version': db_report.version_key,
|
||||
'version_key': version_key,
|
||||
}
|
||||
|
||||
return resp_200(res)
|
||||
@@ -0,0 +1,11 @@
|
||||
from typing import Generic, List, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Create generic variables
|
||||
DataT = TypeVar('DataT')
|
||||
|
||||
|
||||
class PageList(BaseModel, Generic[DataT]):
|
||||
list: List[DataT]
|
||||
total: int
|
||||
@@ -0,0 +1,107 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, field_validator
|
||||
|
||||
from bisheng.database.models.message import ChatMessage, ChatMessageQuery
|
||||
from bisheng.database.models.session import MessageSession
|
||||
from bisheng.user.domain.models.user import User
|
||||
|
||||
|
||||
class AppChatList(BaseModel):
|
||||
flow_name: str
|
||||
user_name: str
|
||||
user_id: int
|
||||
chat_id: str
|
||||
flow_id: str
|
||||
flow_type: int
|
||||
create_time: datetime
|
||||
like_count: Optional[int] = None
|
||||
dislike_count: Optional[int] = None
|
||||
copied_count: Optional[int] = None
|
||||
sensitive_status: Optional[int] = None # Sensitive word review status
|
||||
user_groups: Optional[List[Any]] = None # Groups to which the user belongs
|
||||
mark_user: Optional[str] = None
|
||||
mark_status: Optional[int] = None
|
||||
mark_id: Optional[int] = None
|
||||
messages: Optional[List[dict]] = None # All message list data for the session
|
||||
|
||||
@field_validator('user_name', mode='before')
|
||||
@classmethod
|
||||
def convert_user_name(cls, v: Any):
|
||||
if not isinstance(v, str):
|
||||
return str(v)
|
||||
return v
|
||||
|
||||
|
||||
class APIAddQAParam(BaseModel):
|
||||
question: str
|
||||
answer: List[str]
|
||||
relative_questions: Optional[List[str]] = []
|
||||
|
||||
|
||||
class UseKnowledgeBaseParam(BaseModel):
|
||||
personal_knowledge_enabled: Optional[bool] = False
|
||||
organization_knowledge_ids: Optional[List[int]] = []
|
||||
knowledge_space_ids: Optional[List[int]] = []
|
||||
|
||||
@field_validator('organization_knowledge_ids', mode='before')
|
||||
@classmethod
|
||||
def convert_organization_knowledge_ids(cls, v: Any):
|
||||
if len(v) > 50:
|
||||
raise ValueError('Can only be used up to 50 organization knowledge base')
|
||||
|
||||
return v
|
||||
|
||||
@field_validator('knowledge_space_ids', mode='before')
|
||||
@classmethod
|
||||
def convert_knowledge_space_ids(cls, v: Any):
|
||||
if len(v) > 50:
|
||||
raise ValueError('Can only be used up to 50 knowledge space')
|
||||
|
||||
return v
|
||||
|
||||
|
||||
class APIChatCompletion(BaseModel):
|
||||
clientTimestamp: str
|
||||
conversationId: Optional[str] = None
|
||||
error: Optional[bool] = False
|
||||
generation: Optional[str] = ''
|
||||
isCreatedByUser: Optional[bool] = False
|
||||
isContinued: Optional[bool] = False
|
||||
model: str
|
||||
text: Optional[str] = ''
|
||||
search_enabled: Optional[bool] = False
|
||||
use_knowledge_base: Optional[UseKnowledgeBaseParam] = None
|
||||
files: Optional[List[Dict]] = None
|
||||
parentMessageId: Optional[str] = None
|
||||
overrideParentMessageId: Optional[str] = None
|
||||
responseMessageId: Optional[str] = None
|
||||
|
||||
|
||||
class delta(BaseModel):
|
||||
id: Optional[str]
|
||||
delta: Dict
|
||||
|
||||
|
||||
class SSEResponse(BaseModel):
|
||||
event: str
|
||||
data: delta
|
||||
|
||||
def toString(self) -> str:
|
||||
return f'event: message\ndata: {json.dumps(self.dict())}\n\n'
|
||||
|
||||
|
||||
class ChatMessageHistoryResponse(ChatMessageQuery):
|
||||
user_name: Optional[str] = None
|
||||
flow_name: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
def from_chat_message_objs(cls, chat_messages: List[ChatMessage], user_model: User,
|
||||
message_session: MessageSession):
|
||||
return [
|
||||
cls.model_validate(obj).model_copy(
|
||||
update={"user_name": user_model.user_name, "flow_name": message_session.flow_name,
|
||||
"name": message_session.name}) for obj in chat_messages
|
||||
]
|
||||
@@ -0,0 +1,11 @@
|
||||
from ast import List
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class CreateDatasetParam(BaseModel):
|
||||
name: str
|
||||
description: str
|
||||
file_url: Optional[str]
|
||||
qa_list: Optional[List[str]]
|
||||
@@ -0,0 +1,9 @@
|
||||
from typing import Optional
|
||||
|
||||
from bisheng.knowledge.domain.models.knowledge_file import KnowledgeFileBase
|
||||
from pydantic import Field
|
||||
|
||||
|
||||
class KnowledgeFileResp(KnowledgeFileBase):
|
||||
id: Optional[int] = Field(default=None)
|
||||
title: Optional[str] = Field(default=None, description="Document Summary")
|
||||
@@ -0,0 +1,26 @@
|
||||
from typing import List, Optional, Any
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class MarkTaskCreate(BaseModel):
|
||||
app_list: List[str] = Field(max_length=30)
|
||||
user_list: List[str]
|
||||
|
||||
@field_validator('user_list', mode='before')
|
||||
@classmethod
|
||||
def convert_user_list(cls, v: Any):
|
||||
ret = []
|
||||
for one in v:
|
||||
if isinstance(one, str):
|
||||
ret.append(one)
|
||||
else:
|
||||
ret.append(str(one))
|
||||
return ret
|
||||
|
||||
|
||||
class MarkData(BaseModel):
|
||||
session_id: str
|
||||
task_id: int
|
||||
status: int
|
||||
flow_type: Optional[int] = None
|
||||
@@ -0,0 +1,72 @@
|
||||
from enum import Enum
|
||||
from typing import Optional, Any, List
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class WorkflowEventType(Enum):
|
||||
NodeRun = 'node_run'
|
||||
# Ice Breaker
|
||||
GuideWord = 'guide_word'
|
||||
# Facilitation Questions
|
||||
GuideQuestion = 'guide_question'
|
||||
# Inform the user that user input is now required
|
||||
UserInput = 'input'
|
||||
# Output events that return predefined content to the user
|
||||
OutputMsg = 'output_msg'
|
||||
# Output requires user input at the same time
|
||||
OutputWithInput = 'output_with_input_msg'
|
||||
# Output requires user selection at the same time
|
||||
OutputWithChoose = 'output_with_choose_msg'
|
||||
# Streaming output events, including streaming process, streaming end two states
|
||||
StreamMsg = 'stream_msg'
|
||||
Close = 'close'
|
||||
Error = 'error'
|
||||
|
||||
|
||||
class WorkflowOutputSchema(BaseModel):
|
||||
message: Any = Field(default=None, description='The message content')
|
||||
reasoning_content: Optional[str] = Field(default=None, description='The reasoning content')
|
||||
output_key: Optional[str] = Field(default=None, description='output message key')
|
||||
files: Optional[List[Any]] = Field(default=None, description='The files list')
|
||||
source_url: Optional[str] = Field(default=None, description='The document source url, is web url')
|
||||
extra: Optional[str] = Field(default=None, description='The extra data')
|
||||
|
||||
|
||||
class WorkflowInputItem(BaseModel):
|
||||
key: str = Field(default=None, description='Unique key corresponding to user input')
|
||||
type: str = Field(default=None, description='The input type, select or dialog or file')
|
||||
value: Any = Field(default=None, description='The input default value')
|
||||
label: str = Field(default=None, description='The key label')
|
||||
multiple: bool = Field(default=False, description='The input is multi select')
|
||||
required: bool = Field(default=False, description='The input is required')
|
||||
options: Optional[Any] = Field(default=None, description='The select type options')
|
||||
file_type: Optional[str] = Field(default=None, description='The allow upload file type')
|
||||
|
||||
|
||||
class WorkflowInputSchema(BaseModel):
|
||||
input_type: str = Field(default=None, description='The judge user input is dialog or form')
|
||||
value: List[WorkflowInputItem] = Field(default=None, description='The input schema items')
|
||||
|
||||
|
||||
class WorkflowEvent(BaseModel):
|
||||
event: str = Field(default=None, description='The event type')
|
||||
message_id: Optional[str] = Field(default=None, description='message id for save into mysql')
|
||||
status: Optional[str] = Field(default='end', description='The event status')
|
||||
node_id: Optional[str] = Field(default=None, description='The node id')
|
||||
node_name: Optional[str] = Field(default=None, description='The node name')
|
||||
node_execution_id: Optional[str] = Field(default=None, description='The node exec unique id')
|
||||
output_schema: Optional[WorkflowOutputSchema] = Field(default=None, description='The output schema')
|
||||
input_schema: Optional[WorkflowInputSchema] = Field(default=None, description='The input schema')
|
||||
|
||||
@field_validator('message_id', mode='before')
|
||||
@classmethod
|
||||
def validate_message_id(cls, v: Any) -> Optional[str]:
|
||||
if isinstance(v, str) or v is None:
|
||||
return v
|
||||
return str(v)
|
||||
|
||||
|
||||
class WorkflowStream(BaseModel):
|
||||
session_id: str = Field(default=None, description='The session id')
|
||||
data: WorkflowEvent | list[WorkflowEvent] = Field(default=None, description='The event data or event data list')
|
||||
@@ -0,0 +1,551 @@
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Generic, List, Optional, TypeVar, Union
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from orjson import orjson
|
||||
from pydantic import BaseModel, Field, model_validator, field_validator, ConfigDict
|
||||
|
||||
from bisheng.database.models.assistant import AssistantBase
|
||||
from bisheng.database.models.flow import FlowCreate, FlowRead, FlowType
|
||||
from bisheng.database.models.message import ChatMessageRead
|
||||
from bisheng.database.models.tag import Tag
|
||||
from bisheng.knowledge.domain.models.knowledge import KnowledgeRead
|
||||
from bisheng.knowledge.domain.schemas.knowledge_rag_schema import Metadata
|
||||
from bisheng.tool.domain.models.gpts_tools import GptsToolsRead
|
||||
|
||||
|
||||
class CaptchaInput(BaseModel):
|
||||
captcha_key: str
|
||||
captcha: str
|
||||
|
||||
|
||||
class ChunkInput(BaseModel):
|
||||
knowledge_id: int
|
||||
documents: List[Document]
|
||||
|
||||
|
||||
class BuildStatus(Enum):
|
||||
"""Status of the build."""
|
||||
|
||||
SUCCESS = 'success'
|
||||
FAILURE = 'failure'
|
||||
STARTED = 'started'
|
||||
IN_PROGRESS = 'in_progress'
|
||||
|
||||
|
||||
class GraphData(BaseModel):
|
||||
"""Data inside the exported flow."""
|
||||
|
||||
nodes: List[Dict[str, Any]]
|
||||
edges: List[Dict[str, Any]]
|
||||
|
||||
|
||||
class ExportedFlow(BaseModel):
|
||||
"""Exported flow from bisheng."""
|
||||
|
||||
description: str
|
||||
name: str
|
||||
id: str
|
||||
data: GraphData
|
||||
|
||||
|
||||
class InputRequest(BaseModel):
|
||||
input: str = Field(description='question or command asked LLM to do')
|
||||
|
||||
|
||||
class TweaksRequest(BaseModel):
|
||||
tweaks: Optional[Dict[str, Dict[str, str]]] = Field(default_factory=dict, description='List of dictionaries')
|
||||
|
||||
|
||||
class UpdateTemplateRequest(BaseModel):
|
||||
template: dict
|
||||
|
||||
|
||||
# Create generic variables
|
||||
DataT = TypeVar('DataT')
|
||||
|
||||
|
||||
class UnifiedResponseModel(BaseModel, Generic[DataT]):
|
||||
"""Unified Response Model"""
|
||||
status_code: int
|
||||
status_message: str
|
||||
data: DataT = None
|
||||
|
||||
|
||||
def resp_200(data: Union[list, dict, str, Any] = None,
|
||||
message: str = 'SUCCESS') -> UnifiedResponseModel:
|
||||
"""Success code"""
|
||||
return UnifiedResponseModel(status_code=200, status_message=message, data=data)
|
||||
# return data
|
||||
|
||||
|
||||
def resp_500(code: int = 500,
|
||||
data: Union[list, dict, str, Any] = None,
|
||||
message: str = 'BAD REQUEST') -> UnifiedResponseModel:
|
||||
"""Wrong logical response"""
|
||||
return UnifiedResponseModel(status_code=code, status_message=message, data=data)
|
||||
|
||||
|
||||
class ProcessResponse(BaseModel):
|
||||
"""Process response schema."""
|
||||
|
||||
result: Any = None
|
||||
# task: Optional[TaskResponse] = None
|
||||
session_id: Optional[str] = None
|
||||
backend: Optional[str] = None
|
||||
|
||||
|
||||
class ChatInput(BaseModel):
|
||||
message_id: int
|
||||
comment: str = None
|
||||
liked: int = 0
|
||||
|
||||
|
||||
class AddChatMessages(BaseModel):
|
||||
"""Add a pair of chat messages."""
|
||||
|
||||
flow_id: str # Skills or assistantsID
|
||||
chat_id: str # SessionsID
|
||||
human_message: str = None # User Questions
|
||||
answer_message: str = None # Execution Status
|
||||
|
||||
|
||||
class ChatList(BaseModel):
|
||||
"""Chat message list."""
|
||||
name: str = None
|
||||
flow_name: str = None
|
||||
flow_description: str = None
|
||||
flow_id: str = None
|
||||
chat_id: str = None
|
||||
create_time: datetime = None
|
||||
update_time: datetime = None
|
||||
flow_type: int = None
|
||||
latest_message: Optional[ChatMessageRead] = None
|
||||
logo: Optional[str] = None
|
||||
|
||||
|
||||
class ChatListGroup(BaseModel):
|
||||
"""Chat list grouped by time dimension."""
|
||||
|
||||
group_name: str = Field(description='Group display name, e.g. "今天", "昨天", "2025"')
|
||||
group_key: str = Field(description='Group identifier, e.g. "today", "yesterday", "year_2025"')
|
||||
sessions: List[ChatList] = Field(default_factory=list, description='List of chat sessions in this group')
|
||||
|
||||
|
||||
class FlowGptsOnlineList(BaseModel):
|
||||
id: str = Field('Uniqueness quantificationID')
|
||||
name: str = None
|
||||
desc: str = None
|
||||
logo: str = None
|
||||
create_time: datetime = None
|
||||
update_time: datetime = None
|
||||
flow_type: str = None # flow: Skill assistant：gptsassistant
|
||||
count: int = 0
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
"""Chat message schema."""
|
||||
|
||||
is_bot: bool = False
|
||||
message: Union[str, None, dict, list] = ''
|
||||
type: str = 'human'
|
||||
category: str = 'processing' # system processing answer tool
|
||||
intermediate_steps: Optional[str] = None
|
||||
files: Optional[list] = []
|
||||
user_id: Optional[int] = None
|
||||
message_id: Optional[int | str] = None
|
||||
source: Optional[int] = 0
|
||||
sender: Optional[str] = None
|
||||
receiver: Optional[dict] = None
|
||||
liked: int = 0
|
||||
extra: Optional[str | dict] = '{}'
|
||||
flow_id: Optional[str] = None
|
||||
chat_id: Optional[str] = None
|
||||
|
||||
|
||||
class ChatResponse(ChatMessage):
|
||||
"""Chat response schema."""
|
||||
|
||||
intermediate_steps: Optional[str] = ''
|
||||
is_bot: bool | int = True
|
||||
category: str = 'processing'
|
||||
|
||||
@field_validator('type')
|
||||
@classmethod
|
||||
def validate_message_type(cls, v):
|
||||
"""
|
||||
end_cover: End & Overwrite Previousmessage
|
||||
"""
|
||||
if v not in [
|
||||
'start', 'stream', 'end', 'error', 'info', 'file', 'begin', 'close', 'end_cover',
|
||||
'over'
|
||||
]:
|
||||
raise ValueError('type must be start, stream, end, error, info, or file')
|
||||
return v
|
||||
|
||||
|
||||
class FileResponse(ChatMessage):
|
||||
"""File response schema."""
|
||||
|
||||
data: Any = None
|
||||
data_type: str
|
||||
type: str = 'file'
|
||||
is_bot: bool = True
|
||||
|
||||
@field_validator('data_type')
|
||||
@classmethod
|
||||
def validate_data_type(cls, v):
|
||||
if v not in ['image', 'csv']:
|
||||
raise ValueError('data_type must be image or csv')
|
||||
return v
|
||||
|
||||
|
||||
class FlowListCreate(BaseModel):
|
||||
flows: List[FlowCreate]
|
||||
|
||||
|
||||
class FlowListRead(BaseModel):
|
||||
flows: List[FlowRead]
|
||||
|
||||
|
||||
class InitResponse(BaseModel):
|
||||
flowId: str
|
||||
|
||||
|
||||
class BuiltResponse(BaseModel):
|
||||
built: bool
|
||||
|
||||
|
||||
class UploadFileResponse(BaseModel):
|
||||
"""Upload file response schema."""
|
||||
|
||||
flowId: Optional[str] = None
|
||||
file_path: str
|
||||
relative_path: Optional[str] = None # minioRelative path, i.e.object_name
|
||||
file_name: Optional[str] = None
|
||||
repeat: bool = False # Duplicate in Knowledge Base
|
||||
repeat_file_name: Optional[str] = None # Returns the file name of a duplicate file if it is a duplicate
|
||||
repeat_update_time: Optional[datetime] = None # Returns the update time of a duplicate file if it is a duplicate
|
||||
|
||||
|
||||
class StreamData(BaseModel):
|
||||
event: str
|
||||
data: dict | str
|
||||
|
||||
def __str__(self) -> str:
|
||||
if isinstance(self.data, dict):
|
||||
return f'event: {self.event}\ndata: {orjson.dumps(self.data).decode()}\n\n'
|
||||
return f'event: {self.event}\ndata: {self.data}\n\n'
|
||||
|
||||
|
||||
class CreateComponentReq(BaseModel):
|
||||
name: str = Field(max_length=50, description='Component Name')
|
||||
data: Any = Field(default='', description='Component Data')
|
||||
description: Optional[str] = Field(default='', description='DESCRIPTION')
|
||||
|
||||
|
||||
class CustomComponentCode(BaseModel):
|
||||
code: str
|
||||
field: Optional[str] = None
|
||||
frontend_node: Optional[dict] = None
|
||||
|
||||
|
||||
class AssistantCreateReq(BaseModel):
|
||||
name: str = Field(max_length=50, description='The assistant name.')
|
||||
prompt: str = Field(min_length=20, max_length=1000, description='Helper Prompt')
|
||||
logo: str = Field(description='logoRelative address of the file')
|
||||
|
||||
|
||||
class AssistantUpdateReq(BaseModel):
|
||||
id: str = Field(description='assistantID')
|
||||
name: Optional[str] = Field('', description='The assistant name. Leave empty to not update')
|
||||
desc: Optional[str] = Field('', description='Assistant description Leave empty to not update')
|
||||
logo: Optional[str] = Field('', description='logoRelative address of the file, empty to not update')
|
||||
prompt: Optional[str] = Field('', description='Visible to Userprompt， Leave empty to not update')
|
||||
guide_word: Optional[str] = Field('', description='Ice Breaker Leave empty to not update')
|
||||
guide_question: Optional[List] = Field([], description='Guided Question List, Leave empty to not update')
|
||||
model_name: Optional[str] = Field('', description='Selected model name, Leave empty to not update')
|
||||
temperature: Optional[float] = Field(None, description='Model Temperature, Do not pass or do not update')
|
||||
max_token: Optional[int] = Field(32000, description='MaxtokenQuantity Do not pass or do not update')
|
||||
|
||||
tool_list: List[int] | None = Field(default=None,
|
||||
description='Tools for assistantsIDVertical,An empty list empties the bound tool forNonethen do not update')
|
||||
flow_list: List[str] | None = Field(default=None,
|
||||
description="Assistant's SkillsIDVertical,An empty list clears the bound skills forNonethen do not update")
|
||||
knowledge_list: List[int] | None = Field(default=None,
|
||||
description='The knowledge base uponIDlist, forNonethen do not update')
|
||||
|
||||
@field_validator('model_name', mode='before')
|
||||
@classmethod
|
||||
def convert_model_name(cls, v):
|
||||
return str(v)
|
||||
|
||||
|
||||
class AssistantSimpleInfo(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
desc: str
|
||||
logo: str
|
||||
user_id: int
|
||||
user_name: str
|
||||
status: int
|
||||
flow_type: Optional[int] = None
|
||||
write: Optional[bool] = Field(default=False)
|
||||
group_ids: Optional[List[int]] = None
|
||||
tags: Optional[List[Tag]] = None
|
||||
create_time: datetime
|
||||
update_time: datetime
|
||||
|
||||
|
||||
class AssistantInfo(AssistantBase):
|
||||
tool_list: List[GptsToolsRead] = Field(default_factory=list, description='Tools for assistantsIDVertical')
|
||||
flow_list: List[FlowRead] = Field(default_factory=list, description='Skills for assistantsIDVertical')
|
||||
knowledge_list: List[KnowledgeRead] = Field(default_factory=list, description='The knowledge base uponIDVertical')
|
||||
|
||||
|
||||
class FlowVersionCreate(BaseModel):
|
||||
name: Optional[str] = Field(default=None, description='Version Name')
|
||||
description: Optional[str] = Field(default=None, description='Version description')
|
||||
data: Optional[Dict] = Field(default=None, description='Skill Version Node Data Data')
|
||||
original_version_id: Optional[int] = Field(default=None, description='Version Source VersionID')
|
||||
flow_type: Optional[int] = Field(default=FlowType.WORKFLOW.value,
|
||||
description='Type of version') # 10:new Version
|
||||
|
||||
|
||||
class FlowCompareReq(BaseModel):
|
||||
inputs: Any = Field(default=None, description='Inputs Required for Skill Run')
|
||||
question_list: List[str] = Field(default_factory=list, description='TestcaseVertical')
|
||||
version_list: List[int] = Field(default_factory=list, description='Compare VersionsIDVertical')
|
||||
node_id: str = Field(default=None, description='The nodes that need to be compared are uniqueID')
|
||||
thread_num: Optional[int] = Field(default=1, description='Compare Threads')
|
||||
|
||||
|
||||
class DeleteToolTypeReq(BaseModel):
|
||||
tool_type_id: int = Field(description='Tool category to deleteID')
|
||||
|
||||
|
||||
class GroupAndRoles(BaseModel):
|
||||
group_id: int
|
||||
role_ids: List[int]
|
||||
|
||||
|
||||
class CreateUserReq(BaseModel):
|
||||
user_name: str = Field(max_length=30, description='Username')
|
||||
password: str = Field(description='Passwords')
|
||||
group_roles: List[GroupAndRoles] = Field(description='List of user groups and roles to join')
|
||||
|
||||
|
||||
class OpenAIChatCompletionReq(BaseModel):
|
||||
messages: List[dict] = Field(...,
|
||||
description='Chat message list, only supporteduser、assistant。systemUse data from within the database')
|
||||
model: str = Field(..., description='The only assistantID')
|
||||
n: int = Field(default=1,
|
||||
description='Number of answers returned, The assistant side defaults to1, multiple answers are not supported at this time')
|
||||
stream: bool = Field(default=False, description='Whether to turn on streaming replies')
|
||||
temperature: float = Field(default=0.0,
|
||||
description="Model Temperature, Incoming0or don't post means don't overwrite")
|
||||
tools: List[dict] = Field(default_factory=list,
|
||||
description='Tools List, The assistant is temporarily unsupported, use the configuration of the assistant')
|
||||
|
||||
|
||||
class OpenAIChoice(BaseModel):
|
||||
index: int = Field(..., description='Index of options')
|
||||
message: dict = Field(default=None, description='The corresponding message content matches the format of the input')
|
||||
finish_reason: str = Field(default='stop', description='End Reason, Assistants onlystop')
|
||||
delta: dict = Field(default=None, description='counterpart'sopenaiStreaming Return Message Content')
|
||||
|
||||
|
||||
class OpenAIChatCompletionResp(BaseModel):
|
||||
id: str = Field(..., description='The only one requestedID')
|
||||
object: str = Field(default='chat.completion', description='Type of posts to return.')
|
||||
created: int = Field(default=..., description='Returned creation timestamp')
|
||||
model: str = Field(..., description="returned model, corresponding to the assistant'sid")
|
||||
choices: List[OpenAIChoice] = Field(..., description='Back to answers list')
|
||||
usage: dict = Field(default=None, description='Various of concerntokenQuantity, Assistant This value is empty')
|
||||
system_fingerprint: Optional[str] = Field(default=None, description='System Fingerprint')
|
||||
|
||||
|
||||
class Icon(BaseModel):
|
||||
enabled: bool
|
||||
image: Optional[str] = None
|
||||
relative_path: Optional[str] = None
|
||||
|
||||
|
||||
class WSModel(BaseModel):
|
||||
key: Optional[str] = None
|
||||
id: str
|
||||
name: Optional[str] = None
|
||||
displayName: Optional[str] = None
|
||||
visual: Optional[bool] = False
|
||||
|
||||
|
||||
class WSPrompt(BaseModel):
|
||||
enabled: bool
|
||||
prompt: Optional[str] = None
|
||||
|
||||
|
||||
# linsight Configuration
|
||||
class LinsightConfig(BaseModel):
|
||||
"""
|
||||
Ideas Management Configuration
|
||||
"""
|
||||
model_config = ConfigDict(validate_by_alias=True, validate_by_name=True)
|
||||
|
||||
linsight_entry: bool = Field(default=True, description='Whether to open the Ideas entrance')
|
||||
input_placeholder: str = Field(..., description='Input Box Prompt')
|
||||
tools: Optional[List[Dict]] = Field(default=None, description='List of optional tools for Ideas')
|
||||
tab_display_name: Optional[str] = Field(default='Linsight', description='Tab Display Name')
|
||||
|
||||
|
||||
# Daily Chat Configuration
|
||||
class WorkstationConfig(BaseModel):
|
||||
model_config = ConfigDict(validate_by_alias=True, validate_by_name=True)
|
||||
|
||||
tabDisplayName: Optional[str] = Field(default='', alias='tabDisplayName', description='Tab Display Name')
|
||||
maxTokens: Optional[int] = Field(default=15000, description='Max chunk size for knowledge rag or web search')
|
||||
sidebarIcon: Optional[Icon] = None
|
||||
assistantIcon: Optional[Icon] = None
|
||||
sidebarSlogan: Optional[str] = Field(default='', description='Sidebarslogan')
|
||||
welcomeMessage: Optional[str] = Field(default='')
|
||||
functionDescription: Optional[str] = Field(default='')
|
||||
inputPlaceholder: Optional[str] = ''
|
||||
models: Optional[Union[List[WSModel], str]] = None
|
||||
webSearch: Optional[WSPrompt] = None
|
||||
knowledgeBase: Optional[WSPrompt] = None
|
||||
fileUpload: Optional[WSPrompt] = None
|
||||
systemPrompt: Optional[str] = None
|
||||
applicationCenterWelcomeMessage: Optional[str] = Field(default='', max_length=1000,
|
||||
pattern=r'^[\u4e00-\u9fff\w\s\.,;:!@#$%^&*()\-_=+\[\]{}|\\\'"<>/?`~·！￥（）【】、《》，。；：“”‘’？]+$',
|
||||
description='App Center Welcome Message')
|
||||
applicationCenterDescription: Optional[str] = Field(default='', max_length=1000,
|
||||
pattern=r'^[\u4e00-\u9fff\w\s\.,;:!@#$%^&*()\-_=+\[\]{}|\\\'"<>/?`~·！￥（）【】、《》，。；：“”‘’？]+$',
|
||||
description='App Center Description')
|
||||
|
||||
|
||||
class SubscriptionConfig(BaseModel):
|
||||
system_prompt: Optional[str] = Field(default='', description='System Prompt')
|
||||
user_prompt: Optional[str] = Field(default='', description='User Prompt')
|
||||
max_chunk_size: Optional[int] = Field(default=15000, description='Max chunk size for file chunks')
|
||||
feedback_tips: Optional[str] = Field(default='', description='Feedback Tips')
|
||||
|
||||
|
||||
class KnowledgeSpaceConfig(BaseModel):
|
||||
system_prompt: Optional[str] = Field(default='', description='System Prompt')
|
||||
user_prompt: Optional[str] = Field(default='', description='User Prompt')
|
||||
max_chunk_size: Optional[int] = Field(default=15000, description='Max chunk size for file chunks')
|
||||
|
||||
|
||||
class ExcelRule(BaseModel):
|
||||
slice_length: Optional[int] = Field(default=10, description='Data Line')
|
||||
header_start_row: Optional[int] = Field(default=1, description='Table header start')
|
||||
header_end_row: Optional[int] = Field(default=1, description='End of header')
|
||||
append_header: Optional[int] = Field(default=1, description='Whether to add a header')
|
||||
|
||||
|
||||
# File Split Request Base Parameters
|
||||
class FileProcessBase(BaseModel):
|
||||
knowledge_id: int = Field(..., description='The knowledge base uponID')
|
||||
separator: Optional[List[str]] = Field(default=None,
|
||||
description='Split text rule, If not passed on, it is the default')
|
||||
separator_rule: Optional[List[str]] = Field(default=None,
|
||||
description='Segmentation before or after the segmentation rule;before/after')
|
||||
chunk_size: Optional[int] = Field(default=1000, description='Split text length, default if not passed')
|
||||
chunk_overlap: Optional[int] = Field(default=100, description='Split text overlap length, default if not passed')
|
||||
retain_images: Optional[int] = Field(default=1, description='Keep document image')
|
||||
force_ocr: Optional[int] = Field(default=0, description='EnableOCR')
|
||||
enable_formula: Optional[int] = Field(default=1, description='latexFormula Recognition')
|
||||
filter_page_header_footer: Optional[int] = Field(default=0, description='Filter Header Footer')
|
||||
excel_rule: Optional[ExcelRule] = Field(default=None, description="excel rule")
|
||||
cache: Optional[bool] = Field(default=True,
|
||||
description='Whether to fetch data from the cache when previewing the document')
|
||||
|
||||
@model_validator(mode='before')
|
||||
@classmethod
|
||||
def check_separator_rule(cls, values: Any):
|
||||
if not values.get('separator', None):
|
||||
values['separator'] = ['\n\n', '\n']
|
||||
if not values.get('separator_rule', None):
|
||||
values['separator_rule'] = ['after' for _ in values['separator']]
|
||||
if values.get('chunk_size', None) is None:
|
||||
values['chunk_size'] = 1000
|
||||
if values.get('chunk_overlap') is None:
|
||||
values['chunk_overlap'] = 100
|
||||
if values.get('filter_page_header_footer') is None:
|
||||
values['filter_page_header_footer'] = 0
|
||||
if values.get('force_ocr') is None:
|
||||
values['force_ocr'] = 1
|
||||
if values.get('enable_formula') is None:
|
||||
values['enable_formula'] = 1
|
||||
if values.get("retain_images") is None:
|
||||
values['retain_images'] = 1
|
||||
if values.get("excel_rule") is None:
|
||||
values['excel_rule'] = ExcelRule()
|
||||
if values.get("knowledge_id") is None:
|
||||
raise ValueError('knowledge_id is required')
|
||||
|
||||
return values
|
||||
|
||||
|
||||
# File chunked data format
|
||||
class FileChunk(BaseModel):
|
||||
text: str = Field(..., description='Text block Content')
|
||||
parse_type: Optional[str] = Field(default=None, description='File parsing type to which the text belongs')
|
||||
metadata: Metadata = Field(..., description='Text block metadata')
|
||||
|
||||
|
||||
# Preview File Chunked Content Request Parameters
|
||||
class PreviewFileChunk(FileProcessBase):
|
||||
file_path: str = Field(..., description='FilePath')
|
||||
cache: bool = Field(default=True, description='Whether to fetch from cache')
|
||||
excel_rule: Optional[ExcelRule] = Field(default=None, description="excel rule")
|
||||
|
||||
|
||||
class UpdatePreviewFileChunk(BaseModel):
|
||||
knowledge_id: int = Field(..., description='The knowledge base uponID')
|
||||
file_path: str = Field(..., description='FilePath')
|
||||
text: str = Field(..., description='Text block Content')
|
||||
chunk_index: int = Field(..., description='Text block index, Insidemetadatamile')
|
||||
bbox: Optional[str] = Field(default='', description='Text blocksbboxMessage')
|
||||
|
||||
|
||||
class KnowledgeFileOne(BaseModel):
|
||||
file_path: str = Field(..., description='FilePath')
|
||||
excel_rule: Optional[ExcelRule] = Field(default=None, description="Excel rules")
|
||||
|
||||
|
||||
# Knowledge Base File Processing
|
||||
class KnowledgeFileProcess(FileProcessBase):
|
||||
file_list: List[KnowledgeFileOne] = Field(..., description='List of files')
|
||||
callback_url: Optional[str] = Field(default=None, description='Asynchronous Task Callback Address')
|
||||
extra: Optional[str] = Field(default=None, description='Additional Information')
|
||||
|
||||
|
||||
# Knowledge Base Re-Segment Adjustment
|
||||
class KnowledgeFileReProcess(FileProcessBase):
|
||||
kb_file_id: int = Field(..., description='Knowledge Base FilesID')
|
||||
file_path: str = Field(default="", description='FilePath')
|
||||
excel_rule: Optional[ExcelRule] = Field(default=None, description="Excel rules")
|
||||
callback_url: Optional[str] = Field(default=None, description='Asynchronous Task Callback Address')
|
||||
extra: Optional[Dict] = Field(default=None, description='Additional Information')
|
||||
|
||||
|
||||
class FrequentlyUsedChat(BaseModel):
|
||||
user_link_type: str = Field(..., description='User-associatedtype')
|
||||
type_detail: str = Field(..., description='User-associatedtype_id')
|
||||
|
||||
|
||||
class UsedAppPin(BaseModel):
|
||||
"""Schema for pinning/unpinning used apps"""
|
||||
flow_id: str = Field(..., description='Application ID to pin/unpin')
|
||||
|
||||
|
||||
class UpdateKnowledgeReq(BaseModel):
|
||||
"""Update Knowledge Base Model Request"""
|
||||
model_id: int = Field(..., description='embeddingModelsID')
|
||||
model_type: Optional[str] = Field(default=None,
|
||||
description='Model type, when not passed on, it will be based onmodel_idAuto Query')
|
||||
knowledge_id: Optional[int] = Field(default=None,
|
||||
description='The knowledge base uponID, if empty, update all private repositories')
|
||||
knowledge_name: Optional[str] = Field(default=None, description='Library Name')
|
||||
description: Optional[str] = Field(default=None, description='KB Description')
|
||||
@@ -0,0 +1,105 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from sqlmodel import select
|
||||
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.flow import FlowTemplateNameError
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.database.models.flow import Flow
|
||||
from bisheng.database.models.template import Template, TemplateCreate, TemplateUpdate
|
||||
|
||||
# build router
|
||||
router = APIRouter(prefix='/skill', tags=['Skills'], dependencies=[Depends(UserPayload.get_login_user)])
|
||||
ORDER_GAP = 65535
|
||||
|
||||
|
||||
@router.post('/template/create')
|
||||
def create_template(*, template: TemplateCreate):
|
||||
"""Create a new flow."""
|
||||
db_template = Template.model_validate(template)
|
||||
if not db_template.data:
|
||||
with get_sync_db_session() as session:
|
||||
db_flow = session.get(Flow, template.flow_id)
|
||||
db_template.data = db_flow.data
|
||||
# Correctionname
|
||||
with get_sync_db_session() as session:
|
||||
name_repeat = session.exec(
|
||||
select(Template).where(Template.name == db_template.name)).first()
|
||||
if name_repeat:
|
||||
raise FlowTemplateNameError.http_exception()
|
||||
# Boost order_num x,x+65535
|
||||
with get_sync_db_session() as session:
|
||||
max_order = session.exec(select(Template).order_by(
|
||||
Template.order_num.desc()).limit(1)).first()
|
||||
# If no data is available, proceed from 65535 Getting Started
|
||||
db_template.order_num = max_order.order_num + ORDER_GAP if max_order else ORDER_GAP
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_template)
|
||||
session.commit()
|
||||
session.refresh(db_template)
|
||||
return resp_200(db_template)
|
||||
|
||||
|
||||
@router.get('/template')
|
||||
def read_template(page_size: Optional[int] = None,
|
||||
page_name: Optional[int] = None,
|
||||
flow_type: Optional[int] = None,
|
||||
id: Optional[int] = None,
|
||||
name: Optional[str] = None):
|
||||
"""Read all flows."""
|
||||
sql = select(Template.id, Template.name, Template.description, Template.update_time, Template.order_num)
|
||||
if id:
|
||||
with get_sync_db_session() as session:
|
||||
template = session.get(Template, id)
|
||||
return resp_200([template])
|
||||
if name:
|
||||
sql = sql.where(Template.name == name)
|
||||
if flow_type:
|
||||
sql = sql.where(Template.flow_type == flow_type)
|
||||
|
||||
sql = sql.order_by(Template.order_num.desc())
|
||||
if page_size and page_name:
|
||||
sql = sql.offset(page_size * (page_name - 1)).limit(page_size)
|
||||
try:
|
||||
with get_sync_db_session() as session:
|
||||
template_session = session.exec(sql)
|
||||
templates = template_session.mappings().all()
|
||||
res = []
|
||||
for one in templates:
|
||||
res.append(Template.model_validate(one))
|
||||
return resp_200(res)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
|
||||
|
||||
@router.post('/template/{id}')
|
||||
def update_template(*, id: int, template: TemplateUpdate):
|
||||
"""Update a flow."""
|
||||
with get_sync_db_session() as session:
|
||||
db_template = session.get(Template, id)
|
||||
if not db_template:
|
||||
raise HTTPException(status_code=404, detail='Template not found')
|
||||
template_data = template.model_dump(exclude_unset=True)
|
||||
for key, value in template_data.items():
|
||||
setattr(db_template, key, value)
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_template)
|
||||
session.commit()
|
||||
session.refresh(db_template)
|
||||
return resp_200(db_template)
|
||||
|
||||
|
||||
@router.delete('/template/{id}', status_code=200)
|
||||
def delete_template(*, id: int):
|
||||
"""Delete a flow."""
|
||||
with get_sync_db_session() as session:
|
||||
db_template = session.get(Template, id)
|
||||
if not db_template:
|
||||
raise HTTPException(status_code=404, detail='Template not found')
|
||||
with get_sync_db_session() as session:
|
||||
session.delete(db_template)
|
||||
session.commit()
|
||||
return resp_200()
|
||||
@@ -0,0 +1,92 @@
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Request, Depends, Query, Body
|
||||
|
||||
from bisheng.api.services.tag import TagService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.database.models.group_resource import ResourceTypeEnum
|
||||
|
||||
router = APIRouter(prefix='/tag', tags=['Tag'])
|
||||
|
||||
|
||||
@router.get('')
|
||||
def get_all_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
keyword: str = Query(default=None, description='Search keyword ...'),
|
||||
page: int = Query(default=0, description='Page'),
|
||||
limit: int = Query(default=10, description='Listings Per Page')):
|
||||
result, total = TagService.get_all_tag(request, login_user, keyword, page, limit)
|
||||
return resp_200(data={
|
||||
'data': result,
|
||||
'total': total
|
||||
})
|
||||
|
||||
|
||||
@router.post('')
|
||||
def create_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
name: str = Body(..., embed=True, description='Label Name')):
|
||||
result = TagService.create_tag(request, login_user, name)
|
||||
return resp_200(result)
|
||||
|
||||
|
||||
@router.put('')
|
||||
def update_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
tag_id: int = Body(..., embed=True, description='labelID'),
|
||||
name: str = Body(..., embed=True, description='Label Name')):
|
||||
result = TagService.update_tag(request, login_user, tag_id, name)
|
||||
return resp_200(result)
|
||||
|
||||
|
||||
@router.delete('')
|
||||
def delete_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
tag_id: int = Body(..., embed=True, description='labelID')):
|
||||
TagService.delete_tag(request, login_user, tag_id)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.post('/link')
|
||||
def create_tag_link(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
tag_id: int = Body(..., embed=True, description='labelID'),
|
||||
resource_id: str = Body(..., embed=True, description='reasourseID'),
|
||||
resource_type: ResourceTypeEnum = Body(..., embed=True, description='Resource Type')):
|
||||
result = TagService.create_tag_link(request, login_user, tag_id, resource_id, resource_type)
|
||||
return resp_200(result)
|
||||
|
||||
|
||||
@router.delete('/link')
|
||||
def delete_tag_link(
|
||||
request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
tag_id: int = Body(..., embed=True, description='labelID'),
|
||||
resource_id: str = Body(..., embed=True, description='reasourseID'),
|
||||
resource_type: ResourceTypeEnum = Body(..., embed=True, description='Resource Type')):
|
||||
TagService.delete_tag_link(request, login_user, tag_id, resource_id, resource_type)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.get('/home')
|
||||
def get_home_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get a list of tags to show on the homepage
|
||||
"""
|
||||
|
||||
result = TagService.get_home_tag(request, login_user)
|
||||
return resp_200(result)
|
||||
|
||||
|
||||
@router.post('/home')
|
||||
def update_home_tag(request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user),
|
||||
tag_ids: List[int] = Body(..., embed=True, description='labelIDVertical')):
|
||||
"""
|
||||
Update the list of tags displayed on the homepage
|
||||
"""
|
||||
|
||||
result = TagService.update_home_tag(request, login_user, tag_ids)
|
||||
return resp_200(result)
|
||||
@@ -0,0 +1,185 @@
|
||||
# build router
|
||||
from typing import Annotated, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Query, Request
|
||||
|
||||
from bisheng.api.services.role_group_service import RoleGroupService
|
||||
from bisheng.api.v1.schemas import resp_200
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError
|
||||
from bisheng.common.errcode.user import UserGroupEmptyError
|
||||
from bisheng.database.models.group import Group, GroupCreate
|
||||
from bisheng.database.models.group_resource import ResourceTypeEnum
|
||||
from bisheng.database.models.role import RoleDao
|
||||
from bisheng.database.models.user_group import UserGroupDao
|
||||
|
||||
router = APIRouter(prefix='/group', tags=['User'], dependencies=[Depends(UserPayload.get_login_user)])
|
||||
|
||||
|
||||
@router.get('/list')
|
||||
async def get_all_group(login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get all groups
|
||||
"""
|
||||
if login_user.is_admin():
|
||||
groups = []
|
||||
else:
|
||||
# Query if you are an administrator of another user group under
|
||||
user_groups = UserGroupDao.get_user_admin_group(login_user.user_id)
|
||||
groups = []
|
||||
for one in user_groups:
|
||||
if one.is_group_admin:
|
||||
groups.append(one.group_id)
|
||||
# Not an administrator of any user group does not have permission to view
|
||||
if not groups:
|
||||
raise UnAuthorizedError()
|
||||
|
||||
groups_res = RoleGroupService().get_group_list(groups)
|
||||
return resp_200({'records': groups_res})
|
||||
|
||||
|
||||
@router.post('/create')
|
||||
async def create_group(request: Request, group: GroupCreate,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
"""
|
||||
Add Usergroup
|
||||
"""
|
||||
return resp_200(RoleGroupService().create_group(request, login_user, group))
|
||||
|
||||
|
||||
@router.put('/create')
|
||||
async def update_group(request: Request,
|
||||
group: Group,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
"""
|
||||
Can edit existing usergroups
|
||||
"""
|
||||
return resp_200(RoleGroupService().update_group(request, login_user, group))
|
||||
|
||||
|
||||
@router.delete('/create', status_code=200)
|
||||
async def delete_group(request: Request,
|
||||
group_id: int,
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
"""
|
||||
Can delete existing usergroups
|
||||
"""
|
||||
|
||||
return RoleGroupService().delete_group(request, login_user, group_id)
|
||||
|
||||
|
||||
@router.post('/set_user_group')
|
||||
async def set_user_group(request: Request,
|
||||
user_id: Annotated[int, Body(embed=True)],
|
||||
group_id: Annotated[List[int], Body(embed=True)],
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Set up user groups, Batch Replacement, Replace different user groups according to different operation permissions
|
||||
User group management replaces only the user groups for which he has permissions. Super Admin Full Replacement
|
||||
"""
|
||||
if not group_id:
|
||||
raise UserGroupEmptyError()
|
||||
return resp_200(RoleGroupService().replace_user_groups(request, login_user, user_id, group_id))
|
||||
|
||||
|
||||
@router.get('/get_user_group')
|
||||
async def get_user_group(user_id: int, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get the group to which the user belongs
|
||||
"""
|
||||
return resp_200(RoleGroupService().get_user_groups_list(user_id))
|
||||
|
||||
|
||||
@router.get('/get_group_user')
|
||||
async def get_group_user(group_id: int,
|
||||
page_size: int = None,
|
||||
page_num: int = None,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get grouped users
|
||||
"""
|
||||
return RoleGroupService().get_group_user_list(group_id, page_size, page_num)
|
||||
|
||||
|
||||
@router.post('/set_group_admin')
|
||||
async def set_group_admin(
|
||||
request: Request,
|
||||
user_ids: Annotated[List[int], Body(embed=True)],
|
||||
group_id: Annotated[int, Body(embed=True)],
|
||||
login_user: UserPayload = Depends(UserPayload.get_admin_user)):
|
||||
"""
|
||||
Get groupingadmin, batch setting interface, overriding the historicaladmin
|
||||
"""
|
||||
|
||||
return resp_200(RoleGroupService().set_group_admin(request, login_user, user_ids, group_id))
|
||||
|
||||
|
||||
@router.post('/set_update_user', status_code=200)
|
||||
async def set_update_user(group_id: Annotated[int, Body(embed=True)],
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Update user group last modified by
|
||||
"""
|
||||
return resp_200(RoleGroupService().set_group_update_user(login_user, group_id))
|
||||
|
||||
|
||||
@router.get('/get_group_resources')
|
||||
async def get_group_resources(*,
|
||||
group_id: int,
|
||||
resource_type: int,
|
||||
name: Optional[str] = None,
|
||||
page_size: Optional[int] = 10,
|
||||
page_num: Optional[int] = 1,
|
||||
user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get a list of resources under a user group
|
||||
"""
|
||||
# Determine if you are an administrator of a user group
|
||||
if not user.check_group_admin(group_id):
|
||||
return UnAuthorizedError.return_resp()
|
||||
res, total = await RoleGroupService().get_group_resources(
|
||||
group_id,
|
||||
resource_type=ResourceTypeEnum(resource_type),
|
||||
name=name,
|
||||
page_size=page_size,
|
||||
page_num=page_num)
|
||||
return resp_200(data={
|
||||
"data": res,
|
||||
"total": total
|
||||
})
|
||||
|
||||
|
||||
@router.get("/roles")
|
||||
async def get_group_roles(*,
|
||||
group_id: List[int] = Query(..., description="User GroupsIDVertical"),
|
||||
keyword: str = Query(None, description="Search keyword ..."),
|
||||
page: int = 0,
|
||||
limit: int = 0,
|
||||
user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get a list of roles within a user group
|
||||
"""
|
||||
# Determine if you are an administrator of a user group
|
||||
if not user.check_groups_admin(group_id):
|
||||
return UnAuthorizedError.return_resp()
|
||||
# List of roles under query group
|
||||
role_list = RoleDao.get_role_by_groups(group_id, keyword, page, limit)
|
||||
total = RoleDao.count_role_by_groups(group_id, keyword)
|
||||
|
||||
return resp_200(data={
|
||||
"data": role_list,
|
||||
"total": total
|
||||
})
|
||||
|
||||
|
||||
@router.get("/manage/resources")
|
||||
async def get_manage_resources(login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
keyword: str = Query(None, description="Search keyword ..."),
|
||||
page: int = 1,
|
||||
page_size: int = 10):
|
||||
""" Get a list of apps under a managed user group """
|
||||
res, total = await RoleGroupService().get_manage_resources(login_user, keyword, page, page_size)
|
||||
return resp_200(data={
|
||||
"data": res,
|
||||
"total": total
|
||||
})
|
||||
@@ -0,0 +1,94 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from sqlmodel import delete, select
|
||||
|
||||
from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.database.models.flow_version import FlowVersionDao
|
||||
from bisheng.database.models.variable_value import Variable, VariableCreate, VariableRead, VariableDao
|
||||
|
||||
# build router
|
||||
router = APIRouter(prefix='/variable', tags=['variable'])
|
||||
|
||||
|
||||
@router.post('/', status_code=200)
|
||||
def post_variable(variable: Variable):
|
||||
try:
|
||||
if not variable.version_id:
|
||||
raise HTTPException(status_code=500, detail='version_id is required')
|
||||
if variable.id:
|
||||
# Update with full replacement
|
||||
with get_sync_db_session() as session:
|
||||
db_variable = session.get(Variable, variable.id)
|
||||
db_variable.variable_name = variable.variable_name[:50]
|
||||
db_variable.value = variable.value
|
||||
db_variable.value_type = variable.value_type
|
||||
else:
|
||||
# if name exist
|
||||
with get_sync_db_session() as session:
|
||||
db_variable = session.exec(
|
||||
select(Variable).where(
|
||||
Variable.node_id == variable.node_id,
|
||||
Variable.variable_name == variable.variable_name,
|
||||
Variable.version_id == variable.version_id)).all()
|
||||
if db_variable:
|
||||
raise HTTPException(status_code=500, detail='name repeat, please choose another')
|
||||
db_variable = Variable.from_orm(variable)
|
||||
|
||||
with get_sync_db_session() as session:
|
||||
session.add(db_variable)
|
||||
session.commit()
|
||||
session.refresh(db_variable)
|
||||
return resp_200(db_variable)
|
||||
except Exception as e:
|
||||
logger.exception("post variable error: ")
|
||||
return HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get('/list')
|
||||
def get_variables(*,
|
||||
flow_id: str,
|
||||
node_id: Optional[str] = None,
|
||||
variable_name: Optional[str] = None,
|
||||
version_id: Optional[int] = None):
|
||||
try:
|
||||
# No passingIDGet data for the current version by default
|
||||
if version_id is None:
|
||||
version_id = FlowVersionDao.get_version_by_flow(flow_id).id
|
||||
res = VariableDao.get_variables(flow_id, node_id, variable_name, version_id)
|
||||
return resp_200(res)
|
||||
|
||||
except Exception as e:
|
||||
return HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.delete('/del', status_code=200)
|
||||
def del_variables(*, id: int):
|
||||
try:
|
||||
statment = delete(Variable).where(Variable.id == id)
|
||||
with get_sync_db_session() as session:
|
||||
session.exec(statment)
|
||||
session.commit()
|
||||
return resp_200()
|
||||
|
||||
except Exception as e:
|
||||
return HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post('/save_all', status_code=200)
|
||||
def save_all_variables(*, data: List[VariableCreate]):
|
||||
try:
|
||||
# delete first
|
||||
flow_id = data[0].flow_id
|
||||
with get_sync_db_session() as session:
|
||||
session.exec(delete(Variable).where(Variable.flow_id == flow_id))
|
||||
session.commit()
|
||||
for var in data:
|
||||
db_var = Variable.model_validate(var)
|
||||
session.add(db_var)
|
||||
session.commit()
|
||||
return resp_200()
|
||||
except Exception as e:
|
||||
return HTTPException(status_code=500, detail=str(e))
|
||||
@@ -0,0 +1,315 @@
|
||||
import time
|
||||
from typing import Optional, Union
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Query, WebSocket, WebSocketException, Request, \
|
||||
status as http_status
|
||||
from loguru import logger
|
||||
from sqlmodel import select
|
||||
|
||||
from bisheng.api.services.flow import FlowService
|
||||
from bisheng.api.services.workflow import WorkFlowService
|
||||
from bisheng.api.v1.chat import chat_manager
|
||||
from bisheng.api.v1.schemas import FlowVersionCreate, resp_200
|
||||
from bisheng.common.chat.types import WorkType
|
||||
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.flow import WorkflowNameExistsError, WorkFlowOnlineEditError, AppWriteAuthError
|
||||
from bisheng.common.errcode.http_error import UnAuthorizedError, NotFoundError
|
||||
from bisheng.common.services import telemetry_service
|
||||
from bisheng.core.database import get_sync_db_session
|
||||
from bisheng.core.logger import trace_id_var
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage
|
||||
from bisheng.database.models.assistant import AssistantDao
|
||||
from bisheng.database.models.flow import Flow, FlowCreate, FlowDao, FlowRead, FlowType, FlowUpdate, \
|
||||
FlowStatus
|
||||
from bisheng.database.models.flow_version import FlowVersionDao
|
||||
from bisheng.database.models.role_access import AccessType
|
||||
from bisheng.share_link.api.dependencies import header_share_token_parser
|
||||
from bisheng.share_link.domain.models.share_link import ShareLink
|
||||
from bisheng.utils import generate_uuid
|
||||
from bisheng_langchain.utils.requests import Requests
|
||||
|
||||
router = APIRouter(prefix='/workflow', tags=['Workflow'])
|
||||
|
||||
|
||||
@router.get("/write/auth")
|
||||
async def check_app_write_auth(
|
||||
request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
flow_id: str = Query(..., description="ApplicationsID"),
|
||||
flow_type: int = Query(..., description="Apply type")
|
||||
):
|
||||
""" Check if the user has administrative rights to the app """
|
||||
if flow_type == FlowType.ASSISTANT.value:
|
||||
flow_info = await AssistantDao.aget_one_assistant(flow_id)
|
||||
check_auth_type = AccessType.ASSISTANT_WRITE
|
||||
elif flow_type == FlowType.WORKFLOW.value:
|
||||
flow_info = await FlowDao.aget_flow_by_id(flow_id)
|
||||
check_auth_type = AccessType.WORKFLOW_WRITE
|
||||
if flow_info and flow_info.flow_type != FlowType.WORKFLOW.value:
|
||||
flow_info = None
|
||||
else:
|
||||
raise NotFoundError.http_exception()
|
||||
if not flow_info:
|
||||
raise NotFoundError.http_exception()
|
||||
owner_id = flow_info.user_id
|
||||
if await login_user.async_access_check(owner_id, flow_id, check_auth_type):
|
||||
return resp_200()
|
||||
return AppWriteAuthError.return_resp()
|
||||
|
||||
|
||||
@router.get("/report/file")
|
||||
async def get_report_file(
|
||||
request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
version_key: str = Query("", description="minioright of privacyobject_name"),
|
||||
workflow_id: str = Query(..., description="The WorkflowID")
|
||||
):
|
||||
""" DapatkanreportTemplate file for the node """
|
||||
|
||||
# Check if the user has read access to the app
|
||||
flow_info = await FlowDao.aget_flow_by_id(workflow_id)
|
||||
if not flow_info:
|
||||
raise NotFoundError.http_exception()
|
||||
if not await login_user.async_access_check(flow_info.user_id, workflow_id, AccessType.WORKFLOW):
|
||||
return UnAuthorizedError.return_resp()
|
||||
|
||||
if not version_key:
|
||||
# Regenerate aversion_key
|
||||
version_key = generate_uuid()
|
||||
else:
|
||||
version_key = version_key.split('_', 1)[0]
|
||||
file_url = ""
|
||||
object_name = f"workflow/report/{version_key}.docx"
|
||||
minio_client = await get_minio_storage()
|
||||
if await minio_client.object_exists(minio_client.bucket, object_name):
|
||||
file_url = await minio_client.get_share_link(object_name, clear_host=False)
|
||||
|
||||
return resp_200(data={
|
||||
'url': file_url,
|
||||
'version_key': f'{version_key}_{int(time.time() * 1000)}',
|
||||
})
|
||||
|
||||
|
||||
@router.post('/report/copy', status_code=200)
|
||||
async def copy_report_file(
|
||||
request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
version_key: str = Body(..., embed=True, description="minioright of privacyobject_name")):
|
||||
""" SalinreportTemplate file for the node """
|
||||
version_key = version_key.split('_', 1)[0]
|
||||
new_version_key = generate_uuid()
|
||||
object_name = f"workflow/report/{version_key}.docx"
|
||||
new_object_name = f"workflow/report/{new_version_key}.docx"
|
||||
minio_client = await get_minio_storage()
|
||||
if await minio_client.object_exists(minio_client.bucket, object_name):
|
||||
await minio_client.copy_object(source_object=object_name, dest_object=new_object_name,
|
||||
source_bucket=minio_client.bucket, dest_bucket=minio_client.bucket)
|
||||
return resp_200(data={
|
||||
'version_key': f'{new_version_key}',
|
||||
})
|
||||
|
||||
|
||||
@router.post('/report/callback', status_code=200)
|
||||
async def upload_report_file(
|
||||
request: Request,
|
||||
data: dict = Body(...)):
|
||||
""" office Callback interface save reportTemplate file for the node """
|
||||
status = data.get('status')
|
||||
file_url = data.get('url')
|
||||
key = data.get('key')
|
||||
logger.debug(f'callback={data}')
|
||||
if status not in {2, 6}:
|
||||
# Non-saved callbacks are not processed
|
||||
return {'error': 0}
|
||||
logger.info(f'office_callback url={file_url}')
|
||||
file = Requests().get(url=file_url)
|
||||
version_key = key.split('_', 1)[0]
|
||||
|
||||
minio_client = await get_minio_storage()
|
||||
object_name = f"workflow/report/{version_key}.docx"
|
||||
await minio_client.put_object(
|
||||
object_name=object_name, file=file._content, bucket_name=minio_client.bucket,
|
||||
content_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document')
|
||||
return {'error': 0}
|
||||
|
||||
|
||||
@router.post('/run_once', status_code=200)
|
||||
def run_once(request: Request, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
node_input: Optional[dict] = None, # Input parameters of the node
|
||||
node_data: dict = None,
|
||||
workflow_id: str = Body(..., description='The WorkflowID')):
|
||||
""" Single node operation """
|
||||
result = WorkFlowService.run_once(login_user, node_input, node_data, workflow_id)
|
||||
|
||||
return resp_200(data=result)
|
||||
|
||||
|
||||
@router.websocket('/chat/{workflow_id}')
|
||||
async def workflow_ws(*,
|
||||
workflow_id: str,
|
||||
websocket: WebSocket,
|
||||
chat_id: Optional[str] = None,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user_from_ws)):
|
||||
try:
|
||||
await chat_manager.dispatch_client(websocket, workflow_id, chat_id, login_user, WorkType.WORKFLOW, websocket)
|
||||
except WebSocketException as exc:
|
||||
logger.error(f'Websocket exception: {str(exc)}')
|
||||
await websocket.close(code=http_status.WS_1011_INTERNAL_ERROR, reason=str(exc))
|
||||
|
||||
|
||||
@router.post('/create', status_code=201)
|
||||
def create_flow(*, request: Request, flow: FlowCreate, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""Create a new flow."""
|
||||
# Determine if the user repeats the skill name
|
||||
with get_sync_db_session() as session:
|
||||
if session.exec(
|
||||
select(Flow).where(Flow.name == flow.name, Flow.flow_type == FlowType.WORKFLOW.value,
|
||||
Flow.user_id == login_user.user_id)).first():
|
||||
raise WorkflowNameExistsError.http_exception()
|
||||
flow.user_id = login_user.user_id
|
||||
db_flow = Flow.model_validate(flow)
|
||||
db_flow.create_time = None
|
||||
db_flow.update_time = None
|
||||
db_flow.flow_type = FlowType.WORKFLOW.value
|
||||
# Create New Skill
|
||||
db_flow = FlowDao.create_flow(db_flow, FlowType.WORKFLOW.value)
|
||||
|
||||
current_version = FlowVersionDao.get_version_by_flow(db_flow.id)
|
||||
ret = FlowRead.model_validate(db_flow)
|
||||
ret.version_id = current_version.id
|
||||
FlowService.create_flow_hook(request, login_user, db_flow)
|
||||
return resp_200(data=ret)
|
||||
|
||||
|
||||
@router.get('/versions', status_code=200)
|
||||
def get_versions(*, flow_id: str, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get a list of versions for your skill
|
||||
"""
|
||||
return FlowService.get_version_list_by_flow(login_user, flow_id)
|
||||
|
||||
|
||||
@router.post('/versions', status_code=200)
|
||||
async def create_versions(*,
|
||||
flow_id: str,
|
||||
flow_version: FlowVersionCreate,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Create New Skill Version
|
||||
"""
|
||||
flow_version.flow_type = FlowType.WORKFLOW.value
|
||||
return await FlowService.create_new_version(login_user, flow_id, flow_version)
|
||||
|
||||
|
||||
@router.put('/versions/{version_id}', status_code=200)
|
||||
async def update_versions(*,
|
||||
request: Request,
|
||||
version_id: int,
|
||||
flow_version: FlowVersionCreate,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Update to version
|
||||
"""
|
||||
return await FlowService.update_version_info(request, login_user, version_id, flow_version)
|
||||
|
||||
|
||||
@router.delete('/versions/{version_id}', status_code=200)
|
||||
def delete_versions(*, version_id: int, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Remove Version
|
||||
"""
|
||||
return FlowService.delete_version(login_user, version_id)
|
||||
|
||||
|
||||
@router.get('/versions/{version_id}', status_code=200)
|
||||
def get_version_info(*, version_id: int, login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Get Version Info
|
||||
"""
|
||||
return FlowService.get_version_info(login_user, version_id)
|
||||
|
||||
|
||||
@router.post('/change_version', status_code=200)
|
||||
def change_version(*,
|
||||
request: Request,
|
||||
flow_id: str = Query(default=None, description='Skill UniqueID'),
|
||||
version_id: int = Query(default=None, description='Current version that needs to be setID'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""
|
||||
Modify Current Version
|
||||
"""
|
||||
return FlowService.change_current_version(request, login_user, flow_id, version_id)
|
||||
|
||||
|
||||
@router.get('/get_one_flow/{flow_id}')
|
||||
async def read_flow(*, flow_id: str, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
share_link: Union['ShareLink', None] = Depends(header_share_token_parser)):
|
||||
"""Read a flow."""
|
||||
return await FlowService.get_one_flow(login_user, flow_id, share_link)
|
||||
|
||||
|
||||
@router.patch('/update/{flow_id}')
|
||||
async def update_flow(*,
|
||||
request: Request,
|
||||
flow_id: str,
|
||||
flow: FlowUpdate,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)):
|
||||
"""online offline"""
|
||||
db_flow = await FlowDao.aget_flow_by_id(flow_id)
|
||||
if not db_flow:
|
||||
raise NotFoundError()
|
||||
|
||||
if not await login_user.async_access_check(db_flow.user_id, flow_id, AccessType.WORKFLOW_WRITE):
|
||||
return UnAuthorizedError.return_resp()
|
||||
|
||||
flow_data = flow.model_dump(exclude_unset=True)
|
||||
|
||||
if db_flow.status == FlowStatus.ONLINE.value and (
|
||||
'status' not in flow_data or flow_data['status'] != FlowStatus.OFFLINE.value):
|
||||
raise WorkFlowOnlineEditError.http_exception()
|
||||
|
||||
for key, value in flow_data.items():
|
||||
if key in ['data', 'create_time', 'update_time']:
|
||||
continue
|
||||
if key == "logo" and not value:
|
||||
continue
|
||||
setattr(db_flow, key, value)
|
||||
db_flow = await FlowDao.aupdate_flow(db_flow)
|
||||
await telemetry_service.log_event(
|
||||
user_id=login_user.user_id,
|
||||
event_type=BaseTelemetryTypeEnum.EDIT_APPLICATION,
|
||||
trace_id=trace_id_var.get()
|
||||
)
|
||||
await FlowService.update_flow_hook(request, login_user, db_flow)
|
||||
return resp_200(db_flow)
|
||||
|
||||
|
||||
@router.patch('/status')
|
||||
async def update_flow_status(request: Request, login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
flow_id: str = Body(..., description='SkillID'),
|
||||
version_id: int = Body(..., description='VersionID'),
|
||||
status: int = Body(..., description='Status')):
|
||||
await WorkFlowService.update_flow_status(login_user, flow_id, version_id, status)
|
||||
return resp_200()
|
||||
|
||||
|
||||
@router.get('/list', status_code=200)
|
||||
def read_flows(*,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
name: str = Query(default=None,
|
||||
description='accordingnameFind databases with fuzzy searches for descriptions'),
|
||||
tag_id: int = Query(default=None, description='labelID'),
|
||||
flow_type: int = Query(default=None, description='Type 5 assistant 10 workflow'),
|
||||
page_size: int = Query(default=10, description='Items per page'),
|
||||
page_num: int = Query(default=1, description='Page'),
|
||||
status: int = None,
|
||||
managed: bool = Query(default=False,
|
||||
description='Whether to query the list of apps with administrative permissions')):
|
||||
"""Read all flows."""
|
||||
data, total = WorkFlowService.get_all_flows(login_user, name, status, tag_id, flow_type, page_num, page_size,
|
||||
managed)
|
||||
return resp_200(data={
|
||||
'data': data,
|
||||
'total': total
|
||||
})
|
||||
@@ -0,0 +1,72 @@
|
||||
from fastapi import Depends
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from bisheng.channel.domain.repositories.implementations.channel_info_source_repository_impl import \
|
||||
ChannelInfoSourceRepositoryImpl
|
||||
from bisheng.channel.domain.repositories.implementations.channel_repository_impl import ChannelRepositoryImpl
|
||||
from bisheng.channel.domain.repositories.interfaces.channel_info_source_repository import ChannelInfoSourceRepository
|
||||
from bisheng.channel.domain.repositories.interfaces.channel_repository import ChannelRepository
|
||||
from bisheng.channel.domain.services.article_es_service import ArticleEsService
|
||||
from bisheng.channel.domain.services.channel_service import ChannelService
|
||||
from bisheng.common.dependencies.core_deps import get_db_session
|
||||
from bisheng.channel.domain.repositories.implementations.article_read_repository_impl import ArticleReadRepositoryImpl
|
||||
from bisheng.channel.domain.repositories.interfaces.article_read_repository import ArticleReadRepository
|
||||
from bisheng.common.repositories.implementations.space_channel_member_repository_impl import \
|
||||
SpaceChannelMemberRepositoryImpl
|
||||
from bisheng.common.repositories.interfaces.space_channel_member_repository import SpaceChannelMemberRepository
|
||||
from bisheng.message.api.dependencies import get_message_service as _get_message_service
|
||||
|
||||
|
||||
async def get_channel_repository(
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> ChannelRepository:
|
||||
"""Adaptation ChannelRepositoryInstance Dependencies"""
|
||||
return ChannelRepositoryImpl(session)
|
||||
|
||||
|
||||
async def get_space_channel_member_repository(
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> 'SpaceChannelMemberRepository':
|
||||
"""Adaptation SpaceChannelMemberRepositoryInstance Dependencies"""
|
||||
return SpaceChannelMemberRepositoryImpl(session)
|
||||
|
||||
|
||||
async def get_channel_info_source_repository(
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> 'ChannelInfoSourceRepository':
|
||||
"""Adaptation ChannelInfoSourceRepository Dependencies"""
|
||||
return ChannelInfoSourceRepositoryImpl(session)
|
||||
|
||||
|
||||
def get_article_es_service() -> ArticleEsService:
|
||||
"""Get ArticleEsService instance"""
|
||||
return ArticleEsService()
|
||||
|
||||
async def get_article_read_repository(
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> ArticleReadRepository:
|
||||
"""Adaptation ArticleReadRepository Dependencies"""
|
||||
return ArticleReadRepositoryImpl(session)
|
||||
|
||||
|
||||
async def get_channel_service(
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> 'ChannelService':
|
||||
"""Adaptation ChannelServiceInstance Dependencies"""
|
||||
|
||||
channel_repository = await get_channel_repository(session)
|
||||
space_channel_member_repository = await get_space_channel_member_repository(session)
|
||||
channel_info_source_repository = await get_channel_info_source_repository(session)
|
||||
article_es_service = get_article_es_service()
|
||||
article_read_repository = await get_article_read_repository(session)
|
||||
message_service = await _get_message_service(session)
|
||||
|
||||
return ChannelService(
|
||||
channel_repository=channel_repository,
|
||||
space_channel_member_repository=space_channel_member_repository,
|
||||
channel_info_source_repository=channel_info_source_repository,
|
||||
article_es_service=article_es_service,
|
||||
article_read_repository=article_read_repository,
|
||||
message_service=message_service,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
"""
|
||||
Channel Article AI Assistant Chat API Endpoints
|
||||
|
||||
Provides the following functionalities:
|
||||
- POST /chat/completions: SSE streaming chat
|
||||
- GET /chat/messages/{article_doc_id}: Query chat history
|
||||
- DELETE /chat/messages/{article_doc_id}: Clear chat content
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from langchain_core.documents import Document
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from sse_starlette import EventSourceResponse
|
||||
|
||||
from bisheng.api.services.workstation import (
|
||||
WorkstationConversation, WorkstationMessage
|
||||
)
|
||||
from bisheng.api.v1.schemas import resp_200, ChatResponse
|
||||
from bisheng.channel.domain.schemas.channel_chat_schema import ChannelArticleChatRequest
|
||||
from bisheng.channel.domain.services.article_es_service import ArticleEsService
|
||||
from bisheng.channel.domain.services.channel_chat_service import ChannelChatService
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode import BaseErrorCode
|
||||
from bisheng.common.errcode.channel import ChannelChatConversationNotFoundError
|
||||
from bisheng.common.errcode.http_error import ServerError, UnAuthorizedError
|
||||
from bisheng.common.schemas.api import resp_500, SSEResponse
|
||||
from bisheng.database.constants import MessageCategory
|
||||
from bisheng.database.models.message import ChatMessage, ChatMessageDao
|
||||
from bisheng.database.models.session import MessageSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix='/chat', tags=['Channel Article Chat'])
|
||||
|
||||
|
||||
def custom_json_serializer(obj):
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f'Type {type(obj)} not serializable')
|
||||
|
||||
|
||||
def user_message(msgId, conversationId, sender, text):
|
||||
msg = json.dumps({
|
||||
'message': {
|
||||
'messageId': msgId,
|
||||
'conversationId': conversationId,
|
||||
'sender': sender,
|
||||
'text': text
|
||||
},
|
||||
'created': True
|
||||
})
|
||||
return f'event: message\ndata: {msg}\n\n'
|
||||
|
||||
|
||||
def step_message(stepId, runId, index, msgId):
|
||||
msg = json.dumps({
|
||||
'event': 'on_run_step',
|
||||
'data': {
|
||||
'id': stepId,
|
||||
'runId': runId,
|
||||
'type': 'message_creation',
|
||||
'index': index,
|
||||
'stepDetails': {
|
||||
'type': 'message_creation',
|
||||
'message_creation': {
|
||||
'message_id': msgId
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return f'event: message\ndata: {msg}\n\n'
|
||||
|
||||
|
||||
def delta(id, delta):
|
||||
return {'id': id, 'delta': delta}
|
||||
|
||||
|
||||
async def final_message(conversation: MessageSession, title: str, requestMessage: ChatMessage,
|
||||
text: str, error: bool, modelName: str,
|
||||
source_document: List[Document] = None):
|
||||
responseMessage = await ChatMessageDao.ainsert_one(
|
||||
ChatMessage(
|
||||
user_id=conversation.user_id,
|
||||
chat_id=conversation.chat_id,
|
||||
flow_id=conversation.flow_id,
|
||||
type='assistant',
|
||||
is_bot=True,
|
||||
message=text,
|
||||
category='answer',
|
||||
sender=modelName,
|
||||
extra=json.dumps({
|
||||
'parentMessageId': requestMessage.id,
|
||||
'error': error
|
||||
}),
|
||||
source=0
|
||||
))
|
||||
|
||||
msg = json.dumps(
|
||||
{
|
||||
'final': True,
|
||||
'conversation': WorkstationConversation.from_chat_session(conversation).model_dump(),
|
||||
'title': title,
|
||||
'requestMessage': (await WorkstationMessage.from_chat_message(requestMessage)).model_dump(),
|
||||
'responseMessage': (await WorkstationMessage.from_chat_message(responseMessage)).model_dump(),
|
||||
},
|
||||
default=custom_json_serializer)
|
||||
return f'event: message\ndata: {msg}\n\n'
|
||||
|
||||
|
||||
@router.post('/completions', summary='Channel Article AI Assistant Chat')
|
||||
async def chat_completions(
|
||||
data: ChannelArticleChatRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""
|
||||
Channel Article AI Assistant Chat API, returns SSE stream.
|
||||
Fetches article content by article ID as conversation context, conducts multi-turn conversation with LLM.
|
||||
"""
|
||||
try:
|
||||
# 1. Fetch article content
|
||||
article_es_service = ArticleEsService()
|
||||
article = await ChannelChatService.get_article_content(article_es_service, data.article_doc_id)
|
||||
article_title = article.title
|
||||
article_content = article.content
|
||||
|
||||
# 2. Initialize session and get configuration
|
||||
conversation, bishengllm, is_new_conv, subscription_config = await ChannelChatService.initialize_chat(
|
||||
data, login_user, article_title
|
||||
)
|
||||
conversationId = conversation.chat_id
|
||||
|
||||
# 3. Truncate article content if needed
|
||||
max_chunk_size = subscription_config.max_chunk_size if subscription_config else 15000
|
||||
article_content = ChannelChatService._truncate_article_content(article_content, max_chunk_size)
|
||||
|
||||
except (BaseErrorCode, ValueError) as e:
|
||||
error_response = e if isinstance(e, BaseErrorCode) else ServerError(msg=str(e))
|
||||
return EventSourceResponse(iter([error_response.to_sse_event_instance()]))
|
||||
except Exception as e:
|
||||
logger.exception(f'Error in channel article chat setup: {e}')
|
||||
return EventSourceResponse(iter([ServerError(exception=e).to_sse_event_instance()]))
|
||||
|
||||
async def event_stream():
|
||||
|
||||
try:
|
||||
# Build system prompt from config or default
|
||||
system_prompt = (
|
||||
subscription_config.system_prompt
|
||||
if subscription_config and subscription_config.system_prompt
|
||||
else "You are a professional AI assistant helping users analyze and discuss articles."
|
||||
)
|
||||
|
||||
# Build user prompt from template or default
|
||||
user_prompt_template = (
|
||||
subscription_config.user_prompt
|
||||
if subscription_config and subscription_config.user_prompt
|
||||
else (
|
||||
"# 参考资料\n```\n{article_content}\n```\n# 用户问题\n{question}"
|
||||
)
|
||||
)
|
||||
user_prompt = user_prompt_template.format(
|
||||
article_content=article_content,
|
||||
question=data.text
|
||||
)
|
||||
await ChatMessageDao.ainsert_one(
|
||||
ChatMessage(
|
||||
user_id=login_user.user_id,
|
||||
chat_id=conversation.chat_id,
|
||||
flow_id=data.article_doc_id,
|
||||
type='human',
|
||||
is_bot=False,
|
||||
sender='User',
|
||||
message=json.dumps({"query": data.text}, ensure_ascii=False),
|
||||
category=MessageCategory.QUESTION,
|
||||
source=0,
|
||||
))
|
||||
# Get chat history (excluding the latest one)
|
||||
history_messages = (await ChannelChatService.get_chat_history(conversationId, 8))[:-1]
|
||||
|
||||
# Build LLM input
|
||||
inputs = [
|
||||
SystemMessage(content=system_prompt),
|
||||
*history_messages,
|
||||
HumanMessage(content=user_prompt)
|
||||
]
|
||||
|
||||
answer = ""
|
||||
reasoning_answer = ""
|
||||
# Streaming call to LLM
|
||||
async for chunk in bishengllm.astream(inputs):
|
||||
content = chunk.content
|
||||
reasoning_content = chunk.additional_kwargs.get('reasoning_content', '')
|
||||
answer += content
|
||||
reasoning_answer += reasoning_content
|
||||
yield SSEResponse(data=ChatResponse(
|
||||
category=MessageCategory.STREAM,
|
||||
message={
|
||||
"content": content,
|
||||
"reasoning_content": reasoning_content,
|
||||
},
|
||||
type="stream"
|
||||
)).to_string()
|
||||
|
||||
yield SSEResponse(data=ChatResponse(
|
||||
category=MessageCategory.STREAM,
|
||||
message={
|
||||
"content": answer,
|
||||
"reasoning_content": reasoning_answer
|
||||
},
|
||||
type="end"
|
||||
)).to_string()
|
||||
|
||||
# Append reasoning process to final result
|
||||
await ChatMessageDao.ainsert_one(
|
||||
ChatMessage(
|
||||
category=MessageCategory.ANSWER,
|
||||
message=json.dumps({
|
||||
"content": answer,
|
||||
"reasoning_content": reasoning_answer
|
||||
}, ensure_ascii=False),
|
||||
user_id=login_user.user_id,
|
||||
chat_id=conversation.chat_id,
|
||||
flow_id=data.article_doc_id,
|
||||
type="end",
|
||||
is_bot=True,
|
||||
)
|
||||
)
|
||||
except BaseErrorCode as e:
|
||||
yield e.to_sse_event_instance_str()
|
||||
except Exception as e:
|
||||
logger.exception(f'Error in channel article chat processing')
|
||||
yield ServerError(exception=e).to_sse_event_instance_str()
|
||||
|
||||
try:
|
||||
return StreamingResponse(event_stream(), media_type='text/event-stream')
|
||||
except Exception as e:
|
||||
logger.exception(f'Error creating channel article chat stream: {e}')
|
||||
return EventSourceResponse(iter([ServerError(exception=e).to_sse_event_instance()]))
|
||||
|
||||
|
||||
@router.get('/messages/{article_doc_id}', summary='Query Channel Article AI Assistant Chat History')
|
||||
async def get_chat_history(
|
||||
article_doc_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""Query Channel Article AI Assistant Chat History Content"""
|
||||
messages = await ChannelChatService.get_chat_messages(article_doc_id, login_user)
|
||||
if messages is None:
|
||||
return UnAuthorizedError.return_resp()
|
||||
return resp_200(data=messages)
|
||||
|
||||
|
||||
@router.delete('/messages/{article_doc_id}', summary='Clear Channel Article AI Assistant Chat Content')
|
||||
async def clear_chat(
|
||||
article_doc_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""Clear Channel Article AI Assistant Chat Content"""
|
||||
try:
|
||||
await ChannelChatService.clear_chat(article_doc_id, login_user)
|
||||
return resp_200(data=True)
|
||||
except ChannelChatConversationNotFoundError as e:
|
||||
return resp_500(message=e.Msg)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear channel article chat: {e}")
|
||||
return resp_500(message="Failed to clear chat")
|
||||
@@ -0,0 +1,316 @@
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
|
||||
from bisheng.channel.api.dependencies import get_channel_service
|
||||
from bisheng.channel.domain.schemas.channel_manager_schema import (
|
||||
AddArticlesToKnowledgeSpaceRequest,
|
||||
CreateChannelRequest,
|
||||
UpdateChannelRequest,
|
||||
AddInformationSourceRequest,
|
||||
CrawlWebsiteRequest,
|
||||
MyChannelQueryRequest,
|
||||
SetPinRequest,
|
||||
UpdateMemberRoleRequest,
|
||||
RemoveMemberRequest,
|
||||
QueryTypeEnum,
|
||||
SortByEnum,
|
||||
SubscribeChannelRequest,
|
||||
)
|
||||
from bisheng.channel.domain.services.channel_service import ChannelService
|
||||
from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.schemas.api import resp_200
|
||||
from bisheng.core.external.bisheng_information_client.bisheng_information_manager import get_bisheng_information_client
|
||||
from bisheng.core.external.bisheng_information_client.client import BusinessType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix='/manager', tags=['Channel Management'])
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def create_channel(
|
||||
request: Request,
|
||||
req_param: CreateChannelRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Endpoint to create a new channel."""
|
||||
channel = await channel_service.create_channel(req_param, login_user, request)
|
||||
|
||||
return resp_200(data=channel)
|
||||
|
||||
|
||||
@router.get("/list_sources")
|
||||
async def list_channel_information_sources(
|
||||
business_type: BusinessType = Query(..., description='Information source type: website / wechat'),
|
||||
page: int = Query(1, ge=1, description='Page number, default 1'),
|
||||
page_size: int = Query(20, ge=1, le=100, description='Page size, default 20'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)
|
||||
):
|
||||
"""Endpoint to list information sources of a channel."""
|
||||
|
||||
client = await get_bisheng_information_client()
|
||||
sources, total = await client.list_information_sources(business_type=business_type, page=page,
|
||||
page_size=page_size)
|
||||
return resp_200(data={
|
||||
"sources": [s.model_dump() for s in sources],
|
||||
"total": total
|
||||
})
|
||||
|
||||
|
||||
@router.get("/search_sources")
|
||||
async def search_channel_information_sources(
|
||||
keyword: str = Query(..., min_length=1, description='Search keyword, fuzzy match name and URL'),
|
||||
business_type: Optional[BusinessType] = Query(None, description='Information source type: website / wechat'),
|
||||
page: int = Query(1, ge=1, description='Page number, default 1'),
|
||||
page_size: int = Query(20, ge=1, le=100, description='Page size, default 20'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user)
|
||||
):
|
||||
"""Endpoint to search information sources of a channel by keyword."""
|
||||
client = await get_bisheng_information_client()
|
||||
sources, total = await client.search_information_sources(
|
||||
query=keyword,
|
||||
business_type=business_type,
|
||||
page=page,
|
||||
page_size=page_size
|
||||
)
|
||||
return resp_200(data={
|
||||
"sources": [s.model_dump() for s in sources],
|
||||
"total": total
|
||||
})
|
||||
|
||||
|
||||
@router.post("/add_website_source")
|
||||
async def add_website_information_source(
|
||||
req_param: AddInformationSourceRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""Endpoint to add a new information source by URL."""
|
||||
client = await get_bisheng_information_client()
|
||||
result = await client.add_website_information_source(req_param.url)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.post("/add_wechat_source")
|
||||
async def add_wechat_information_source(
|
||||
req_param: AddInformationSourceRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""Endpoint to add a new WeChat information source by URL."""
|
||||
client = await get_bisheng_information_client()
|
||||
result = await client.add_wechat_information_source(req_param.url)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.post("/crawl")
|
||||
async def crawl_website(
|
||||
req_param: CrawlWebsiteRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
):
|
||||
"""Endpoint to temporarily crawl a website and return its metadata."""
|
||||
|
||||
client = await get_bisheng_information_client()
|
||||
result = await client.crawl_website(req_param.url)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.get("/my_channels")
|
||||
async def get_my_channels(
|
||||
query_type: QueryTypeEnum = Query(..., description='Query type: created / followed'),
|
||||
sort_by: SortByEnum = Query(SortByEnum.LATEST_UPDATE, description='Sort by, default latest update'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Endpoint to get channels related to the logged-in user, either created or followed, with sorting options."""
|
||||
query_data = MyChannelQueryRequest(query_type=query_type, sort_by=sort_by)
|
||||
result = await channel_service.get_my_channels(query_data, login_user)
|
||||
return resp_200(data=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
@router.get("/square")
|
||||
async def get_channel_square(
|
||||
keyword: Optional[str] = Query(None, description='Fuzzy search keyword (channel name/description)'),
|
||||
page: int = Query(1, ge=1, description='Page number, default 1'),
|
||||
page_size: int = Query(20, ge=1, le=100, description='Page size, default 20'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Channel square query: Paginated query of all released channels, supports fuzzy search, displays subscription status and subscriber count."""
|
||||
result = await channel_service.get_channel_square(
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
login_user=login_user
|
||||
)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.post("/subscribe")
|
||||
async def subscribe_channel(
|
||||
req_param: SubscribeChannelRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Subscribe channel request API: Handles subscription requests based on channel type (public, private, approval required)."""
|
||||
status = await channel_service.subscribe_channel(req_param, login_user)
|
||||
return resp_200(data=status.value)
|
||||
|
||||
|
||||
@router.post("/set_pin")
|
||||
async def set_channel_pin(
|
||||
req_param: SetPinRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Set channel pin status."""
|
||||
await channel_service.set_channel_pin(req_param, login_user)
|
||||
return resp_200(data=True)
|
||||
|
||||
|
||||
@router.get("/members")
|
||||
async def list_channel_members(
|
||||
channel_id: str = Query(..., description='Channel ID'),
|
||||
page: int = Query(1, ge=1, description='Page number, default 1'),
|
||||
page_size: int = Query(20, ge=1, le=100, description='Page size, default 20'),
|
||||
keyword: str = Query(None, description='Username fuzzy search keyword'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Paginated query of channel member list, supports fuzzy search by username."""
|
||||
result = await channel_service.list_channel_members(
|
||||
channel_id=channel_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
keyword=keyword,
|
||||
login_user=login_user
|
||||
)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.post("/update_member_role")
|
||||
async def update_member_role(
|
||||
req_param: UpdateMemberRoleRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Set member role (admin/member)."""
|
||||
|
||||
await channel_service.update_member_role(req_param, login_user)
|
||||
return resp_200(data=True)
|
||||
|
||||
|
||||
@router.post("/remove_member")
|
||||
async def remove_member(
|
||||
req_param: RemoveMemberRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Remove channel member."""
|
||||
|
||||
await channel_service.remove_member(req_param, login_user)
|
||||
return resp_200(data=True)
|
||||
|
||||
|
||||
@router.get("/articles")
|
||||
async def search_channel_articles(
|
||||
channel_id: str = Query(..., description='Channel ID'),
|
||||
keyword: Optional[str] = Query(None, description='Search keyword (title, content, source ID)'),
|
||||
source_ids: Optional[str] = Query(None, description='Specified source ID list, comma separated'),
|
||||
sub_channel_name: Optional[str] = Query(None, description='Sub-channel name'),
|
||||
page: int = Query(1, ge=1, description='Page number, default 1'),
|
||||
page_size: int = Query(20, ge=1, le=100, description='Page size, default 20'),
|
||||
only_unread: Optional[bool] = Query(False, description='Show unread only'),
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Paginated search of articles by channel, supports keyword search, source filtering, sub-channel filtering, results with highlighting."""
|
||||
# Parse comma-separated source ID list
|
||||
parsed_source_ids = None
|
||||
if source_ids:
|
||||
parsed_source_ids = [s.strip() for s in source_ids.split(',') if s.strip()]
|
||||
|
||||
result = await channel_service.search_channel_articles(
|
||||
channel_id=channel_id,
|
||||
keyword=keyword,
|
||||
source_ids=parsed_source_ids,
|
||||
sub_channel_name=sub_channel_name,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
login_user=login_user,
|
||||
only_unread=only_unread,
|
||||
)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.get("/articles/detail/{article_id}")
|
||||
async def get_article_detail(
|
||||
article_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Get article details by article ID and record read status."""
|
||||
result = await channel_service.get_article_detail(
|
||||
article_id=article_id,
|
||||
login_user=login_user,
|
||||
)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.get("/{channel_id}")
|
||||
async def get_channel_detail(
|
||||
channel_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Get channel details, including basic channel information, creator, subscriber count, article count, etc."""
|
||||
result = await channel_service.get_channel_detail(channel_id, login_user)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.put("/{channel_id}")
|
||||
async def update_channel_info(
|
||||
channel_id: str,
|
||||
req_param: UpdateChannelRequest,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Update channel information API"""
|
||||
result = await channel_service.update_channel(channel_id, req_param, login_user)
|
||||
return resp_200(data=result.model_dump())
|
||||
|
||||
|
||||
@router.delete("/{channel_id}")
|
||||
async def dismiss_channel(
|
||||
request: Request,
|
||||
channel_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Dismiss channel API"""
|
||||
await channel_service.dismiss_channel(channel_id, login_user, request)
|
||||
return resp_200(data=True)
|
||||
|
||||
|
||||
@router.post("/{channel_id}/unsubscribe")
|
||||
async def unsubscribe_channel(
|
||||
channel_id: str,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Unsubscribe channel API"""
|
||||
await channel_service.unsubscribe_channel(channel_id, login_user)
|
||||
return resp_200(data=True)
|
||||
|
||||
|
||||
@router.post("/articles/add_to_knowledge_space")
|
||||
async def add_articles_to_knowledge_space(
|
||||
req: AddArticlesToKnowledgeSpaceRequest,
|
||||
request: Request,
|
||||
login_user: UserPayload = Depends(UserPayload.get_login_user),
|
||||
channel_service: 'ChannelService' = Depends(get_channel_service)
|
||||
):
|
||||
"""Add channel articles to a knowledge space."""
|
||||
result = await channel_service.add_articles_to_knowledge_space(req, login_user, request)
|
||||
return resp_200(data=result)
|
||||
@@ -0,0 +1,8 @@
|
||||
from fastapi import APIRouter
|
||||
from .endpoints.channel_manager import router as channel_manager
|
||||
from .endpoints.channel_chat import router as channel_chat
|
||||
|
||||
router = APIRouter(prefix='/channel')
|
||||
|
||||
router.include_router(channel_manager)
|
||||
router.include_router(channel_chat)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user