chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+166
@@ -0,0 +1,166 @@
|
||||
# ALL
|
||||
*.dev.yaml
|
||||
|
||||
legacy/
|
||||
|
||||
# for server
|
||||
server/models/*
|
||||
!server/models/download.sh
|
||||
!server/models/download.ps1
|
||||
server/logs/
|
||||
server/models_dev
|
||||
server/public/*
|
||||
!server/public/examples/
|
||||
server/public/examples/*
|
||||
!server/public/examples/a.jpg
|
||||
!server/public/examples/b.jpg
|
||||
!server/public/examples/c.jpg
|
||||
!server/public/examples/d.jpg
|
||||
!server/public/examples/e.jpg
|
||||
!server/public/examples/f.jpg
|
||||
!server/public/examples/g.jpg
|
||||
|
||||
# docker
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
|
||||
# for gradio
|
||||
# server/run_gradio.py
|
||||
|
||||
# for web
|
||||
web/node_modules
|
||||
web/package-lock.json
|
||||
web/dist
|
||||
web/electron-dist
|
||||
web/yarn.lock
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
cff-version: 1.2.0
|
||||
message: "If you find this work useful in your method, you can cite the paper as below."
|
||||
authors:
|
||||
- family-names: Shen
|
||||
given-names: Yongliang
|
||||
- family-names: Song
|
||||
given-names: Kaitao
|
||||
- family-names: Tan
|
||||
given-names: Xu
|
||||
- family-names: Li
|
||||
given-names: Dongsheng
|
||||
- family-names: Lu
|
||||
given-names: Weiming
|
||||
- family-names: Zhuang
|
||||
given-names: Yueting
|
||||
title: "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace"
|
||||
version: 2.0.4
|
||||
license: MIT
|
||||
status: preprint
|
||||
date-released: 2023-03-30
|
||||
url: https://github.com/microsoft/JARVIS
|
||||
preferred-citation:
|
||||
type: article
|
||||
authors:
|
||||
- family-names: Shen
|
||||
given-names: Yongliang
|
||||
- family-names: Song
|
||||
given-names: Kaitao
|
||||
- family-names: Tan
|
||||
given-names: Xu
|
||||
- family-names: Li
|
||||
given-names: Dongsheng
|
||||
- family-names: Lu
|
||||
given-names: Weiming
|
||||
- family-names: Zhuang
|
||||
given-names: Yueting
|
||||
journal: "arXiv preprint arXiv:2303.17580"
|
||||
title: "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace"
|
||||
year: 2023
|
||||
url: https://arxiv.org/abs/2303.17580
|
||||
@@ -0,0 +1,9 @@
|
||||
# Microsoft Open Source Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
|
||||
Resources:
|
||||
|
||||
- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
|
||||
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
|
||||
@@ -0,0 +1,55 @@
|
||||
|
||||
To contribute to this GitHub project, you can follow these steps:
|
||||
|
||||
1. Fork the repository you want to contribute to by clicking the "Fork" button on the project page.
|
||||
|
||||
2. Clone the repository to your local machine and enter the newly created repo using the following commands:
|
||||
|
||||
```
|
||||
git clone https://github.com/YOUR-GITHUB-USERNAME/JARVIS
|
||||
cd JARVIS
|
||||
```
|
||||
3. Create a new branch for your changes using the following command:
|
||||
|
||||
```
|
||||
git checkout -b "branch-name"
|
||||
```
|
||||
4. Make your changes to the code or documentation.
|
||||
|
||||
5. Add the changes to the staging area using the following command:
|
||||
```
|
||||
git add .
|
||||
```
|
||||
|
||||
6. Commit the changes with a meaningful commit message using the following command:
|
||||
```
|
||||
git commit -m "your commit message"
|
||||
```
|
||||
7. Push the changes to your forked repository using the following command:
|
||||
```
|
||||
git push origin branch-name
|
||||
```
|
||||
8. Go to the GitHub website and navigate to your forked repository.
|
||||
|
||||
9. Click the "New pull request" button.
|
||||
|
||||
10. Select the branch you just pushed to and the branch you want to merge into on the original repository.
|
||||
|
||||
11. Add a description of your changes and click the "Create pull request" button.
|
||||
|
||||
12. Wait for the project maintainer to review your changes and provide feedback.
|
||||
|
||||
13. Make any necessary changes based on feedback and repeat steps 5-12 until your changes are accepted and merged into the main project.
|
||||
|
||||
14. Once your changes are merged, you can update your forked repository and local copy of the repository with the following commands:
|
||||
|
||||
```
|
||||
git fetch upstream
|
||||
git checkout main
|
||||
git merge upstream/main
|
||||
```
|
||||
Finally, delete the branch you created with the following command:
|
||||
```
|
||||
git branch -d branch-name
|
||||
```
|
||||
That's it you made it 🐣⭐⭐
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
@@ -0,0 +1,257 @@
|
||||
# JARVIS
|
||||
|
||||
|
||||
[](https://arxiv.org/abs/2303.17580)
|
||||
[](https://huggingface.co/spaces/microsoft/HuggingGPT)
|
||||
|
||||
The mission of JARVIS is to explore artificial general intelligence (AGI) and deliver cutting-edge research to the whole community.
|
||||
|
||||
## What's New
|
||||
|
||||
+ [2024.01.15] We release Easytool for easier tool usage.
|
||||
+ The code and datasets are available at [EasyTool](/easytool).
|
||||
+ The paper is available at [EasyTool: Enhancing LLM-based Agents with Concise Tool Instruction](https://arxiv.org/abs/2401.06201).
|
||||
+ [2023.11.30] We release TaskBench for evaluating task automation capability of LLMs.
|
||||
+ The code and datasets are available at [TaskBench](/taskbench).
|
||||
+ The paper is available at [TaskBench: Benchmarking Large Language Models for Task Automation](https://arxiv.org/abs/2311.18760).
|
||||
+ [2023.07.28] We are now in the process of planning evaluation and project rebuilding. We will release a new version of Jarvis in the near future.
|
||||
+ [2023.07.24] We released a light langchain version of Jarvis. See <a href="https://github.com/langchain-ai/langchain/tree/master/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt">here</a>.
|
||||
+ [2023.04.16] Jarvis now supports the OpenAI service on the Azure platform and the GPT-4 model.
|
||||
+ [2023.04.06] We added the Gradio demo and built the web API for `/tasks` and `/results` in `server` mode.
|
||||
+ The Gradio demo is now hosted on Hugging Face Space. (Build with `inference_mode=hybrid` and `local_deployment=standard`)
|
||||
+ The Web API `/tasks` and `/results` access intermediate results for `Stage #1`: task planning and `Stage #1-3`: model selection with execution results. See <a href="#Server">here</a>.
|
||||
+ [2023.04.03] We added the CLI mode and provided parameters for configuring the scale of local endpoints.
|
||||
+ You can enjoy a lightweight experience with Jarvis without deploying the models locally. See <a href="#Configuration">here</a>.
|
||||
+ Just run `python awesome_chat.py --config configs/config.lite.yaml` to experience it.
|
||||
+ [2023.04.01] We updated a version of code for building.
|
||||
|
||||
|
||||
### Overview
|
||||
|
||||
Language serves as an interface for LLMs to connect numerous AI models for solving complicated AI tasks!
|
||||
|
||||
<p align="center">
|
||||
<img width="100%" alt="image" src="./hugginggpt/assets/intro.png">
|
||||
</p>
|
||||
|
||||
|
||||
See our paper: [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace](http://arxiv.org/abs/2303.17580), Yongliang Shen, Kaitao Song, Xu Tan, Dongsheng Li, Weiming Lu and Yueting Zhuang (the first two authors contribute equally)
|
||||
|
||||
We introduce a collaborative system that consists of **an LLM as the controller** and **numerous expert models as collaborative executors** (from HuggingFace Hub). The workflow of our system consists of four stages:
|
||||
+ **Task Planning**: Using ChatGPT to analyze the requests of users to understand their intention, and disassemble them into possible solvable tasks.
|
||||
+ **Model Selection**: To solve the planned tasks, ChatGPT selects expert models hosted on Hugging Face based on their descriptions.
|
||||
+ **Task Execution**: Invokes and executes each selected model, and return the results to ChatGPT.
|
||||
+ **Response Generation**: Finally, using ChatGPT to integrate the prediction of all models, and generate responses.
|
||||
|
||||
<p align="center"><img src="./hugginggpt/assets/overview.jpg"></p>
|
||||
|
||||
### System Requirements
|
||||
|
||||
#### Default (Recommended)
|
||||
|
||||
For `configs/config.default.yaml`:
|
||||
|
||||
+ Ubuntu 16.04 LTS
|
||||
+ VRAM >= 24GB
|
||||
+ RAM > 12GB (minimal), 16GB (standard), 80GB (full)
|
||||
+ Disk > 284GB
|
||||
+ 42GB for `damo-vilab/text-to-video-ms-1.7b`
|
||||
+ 126GB for `ControlNet`
|
||||
+ 66GB for `stable-diffusion-v1-5`
|
||||
+ 50GB for others
|
||||
|
||||
#### Minimum (Lite)
|
||||
|
||||
For `configs/config.lite.yaml`:
|
||||
|
||||
+ Ubuntu 16.04 LTS
|
||||
+ Nothing else
|
||||
|
||||
The configuration `configs/config.lite.yaml` does not require any expert models to be downloaded and deployed locally. However, it means that Jarvis is restricted to models running stably on HuggingFace Inference Endpoints.
|
||||
|
||||
### Quick Start
|
||||
|
||||
First replace `openai.key` and `huggingface.token` in `server/configs/config.default.yaml` with **your personal OpenAI Key** and **your Hugging Face Token**, or put them in the environment variables `OPENAI_API_KEY` and `HUGGINGFACE_ACCESS_TOKEN` respectively. Then run the following commands:
|
||||
|
||||
<span id="Server"></span>
|
||||
|
||||
#### For Server:
|
||||
|
||||
```bash
|
||||
# setup env
|
||||
cd server
|
||||
conda create -n jarvis python=3.8
|
||||
conda activate jarvis
|
||||
conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia
|
||||
pip install -r requirements.txt
|
||||
|
||||
# download models. Make sure that `git-lfs` is installed.
|
||||
cd models
|
||||
bash download.sh # required when `inference_mode` is `local` or `hybrid`.
|
||||
|
||||
# run server
|
||||
cd ..
|
||||
python models_server.py --config configs/config.default.yaml # required when `inference_mode` is `local` or `hybrid`
|
||||
python awesome_chat.py --config configs/config.default.yaml --mode server # for text-davinci-003
|
||||
```
|
||||
|
||||
Now you can access Jarvis' services by the Web API.
|
||||
|
||||
+ `/hugginggpt` --method `POST`, access the full service.
|
||||
+ `/tasks` --method `POST`, access intermediate results for Stage #1.
|
||||
+ `/results` --method `POST`, access intermediate results for Stage #1-3.
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
# request
|
||||
curl --location 'http://localhost:8004/tasks' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "based on pose of /examples/d.jpg and content of /examples/e.jpg, please show me a new image"
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
# response
|
||||
[{"args":{"image":"/examples/d.jpg"},"dep":[-1],"id":0,"task":"openpose-control"},{"args":{"image":"/examples/e.jpg"},"dep":[-1],"id":1,"task":"image-to-text"},{"args":{"image":"<GENERATED>-0","text":"<GENERATED>-1"},"dep":[1,0],"id":2,"task":"openpose-text-to-image"}]
|
||||
```
|
||||
|
||||
|
||||
#### For Web:
|
||||
|
||||
We provide a user-friendly web page. After starting `awesome_chat.py` in a server mode, you can run the commands to communicate with Jarvis in your browser:
|
||||
|
||||
- you need to install `nodejs` and `npm` first.
|
||||
- [ IMPORTANT ] if you are running the web client on another machine, you need set `http://{LAN_IP_of_the_server}:{port}/` to `HUGGINGGPT_BASE_URL` of `web/src/config/index.ts`.
|
||||
- if you want to use the video generation feature, you need to compile `ffmpeg` manually with H.264.
|
||||
- you can switch to ChatGPT by `double click` on the setting icon!
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
```bash
|
||||
# Optional: Install ffmpeg
|
||||
# This command need be executed without errors.
|
||||
LD_LIBRARY_PATH=/usr/local/lib /usr/local/bin/ffmpeg -i input.mp4 -vcodec libx264 output.mp4
|
||||
```
|
||||
|
||||
<span id="Gradio"></span>
|
||||
|
||||
#### For Gradio
|
||||
|
||||
The Gradio demo is now hosted on Hugging Face Space. You can also run the following commands to start the demo locally:
|
||||
|
||||
```bash
|
||||
python models_server.py --config configs/config.gradio.yaml
|
||||
python run_gradio_demo.py --config configs/config.gradio.yaml
|
||||
|
||||
# or run the HF Space as a Docker image (Build with `inference_mode=hibrid` and `local_deployment=standard`)
|
||||
docker run -it -p 7860:7860 --platform=linux/amd64 registry.hf.space/microsoft-hugginggpt:latest python app.py
|
||||
```
|
||||
|
||||
#### For CLI:
|
||||
|
||||
You can also run Jarvis more easily in CLI mode:
|
||||
|
||||
```bash
|
||||
cd server
|
||||
python awesome_chat.py --config configs/config.default.yaml --mode cli
|
||||
```
|
||||
|
||||
Examples of CLI mode:
|
||||
|
||||
```
|
||||
Welcome to Jarvis! A collaborative system that consists of an LLM as the controller and numerous expert models as collaborative executors. Jarvis can plan tasks, schedule Hugging Face models, generate friendly responses based on your requests, and help you with many things. Please enter your request (`exit` to exit).
|
||||
[ User ]: Given a collection of image A: /examples/a.jpg, B: /examples/b.jpg, C: /examples/c.jpg, please tell me how many zebras in these picture?
|
||||
[ Jarvis ]: Based on your request, I have used several models to process the images you provided. For the image A: /examples/a.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "a cat sitting on a window sill looking out". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/a3dc.jpg', which contains two objects with labels 'potted plant' and 'cat'. For the image B: /examples/b.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "two zebras standing in a field of dry grass". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/f5df.jpg', which contains three objects with labels 'zebra'. For the image C: /examples/c.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "a zebra standing in a field of tall grass". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/69c7.jpg', which contains one object with label 'zebra'. Finally, I used the canny-control model canny-control to generate the image '/images/7e18.png' based on the image /examples/f.jpg. Therefore, there are four zebras in these pictures. Is there anything else I can help you with?
|
||||
[ User ]: Please answer all the named entities in the sentence: Iron Man is a superhero appearing in American comic books published by Marvel Comics. The character was co-created by writer and editor Stan Lee, developed by scripter Larry Lieber, and designed by artists Don Heck and Jack Kirby.
|
||||
[ Jarvis ]: Sure, I understand your request. Based on the inference results of the models, I will answer your request step by step. Firstly, I used the token-classification model dslim/bert-base-NER to recognize the named entities in the sentence. The inference result is that the named entities are Iron Man, Stan Lee, Larry Lieber, Don Heck and Jack Kirby. Then, I used the question-answering model deepset/roberta-base-squad2 to answer your request. The inference result is that there is no output since the context cannot be empty. Therefore, I cannot make it. I hope this answer your request. Is there anything else I can help you with?
|
||||
[ User ]: Please generate a canny image based on /examples/f.jpg
|
||||
[ Jarvis ]: Sure. I have generated a canny image based on /examples/f.jpg. To do this, I first used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text description of the image, which is "a herd of giraffes and zebras grazing in a field". Then I used the canny-control model to generate the canny image of the original image, which is stored in the path "/images/1ea2.png". Finally, I used the canny-text-to-image model lllyasviel/sd-controlnet-canny to generate the canny image based on the text description and the canny image, which is stored in the path "/images/ba66.png". Do you need anything else?
|
||||
```
|
||||
|
||||
|
||||
### Configuration
|
||||
|
||||
The server-side configuration file is `server/configs/config.default.yaml`, and some parameters are presented as follows:
|
||||
|
||||
+ `model`: LLM, currently supports `text-davinci-003`. We are working on integrating more open-source LLMs.
|
||||
+ `inference_mode`: mode of inference endpoints
|
||||
+ `local`: only use the local inference endpoints
|
||||
+ `huggingface`: only use the Hugging Face Inference Endpoints **(free of local inference endpoints)**
|
||||
+ `hybrid`: both of `local` and `huggingface`
|
||||
+ `local_deployment`: scale of locally deployed models, works under `local` or `hybrid` inference mode:
|
||||
+ `minimal` (RAM>12GB, ControlNet only)
|
||||
+ `standard` (RAM>16GB, ControlNet + Standard Pipelines)
|
||||
+ `full` (RAM>42GB, All registered models)
|
||||
|
||||
On a personal laptop, we recommend the configuration of `inference_mode: hybrid `and `local_deployment: minimal`. But the available models under this setting may be limited due to the instability of remote Hugging Face Inference Endpoints.
|
||||
|
||||
### NVIDIA Jetson Embedded Device Support
|
||||
A [Dockerfile](./Dockerfile.jetson) is included that provides experimental support for [NVIDIA Jetson embedded devices](https://developer.nvidia.com/embedded-computing). This image provides accelerated ffmpeg, pytorch, torchaudio, and torchvision dependencies. To build the docker image, [ensure that the default docker runtime is set to 'nvidia'](https://github.com/NVIDIA/nvidia-docker/wiki/Advanced-topics#default-runtime). A pre-built image is provided at https://hub.docker.com/r/toolboc/nv-jarvis.
|
||||
|
||||
```bash
|
||||
#Build the docker image
|
||||
docker build --pull --rm -f "Dockerfile.jetson" -t toolboc/nv-jarvis:r35.2.1
|
||||
```
|
||||
|
||||
Due to to memory requirements, JARVIS is required to run on Jetson AGX Orin family devices (64G on-board RAM device preferred) with config options set to:
|
||||
* `inference_mode: local`
|
||||
* `local_deployment: standard`
|
||||
|
||||
Models and configs are recommended to be provided through a volume mount from the host to the container as shown in the `docker run` step below. It is possible to uncomment the `# Download local models` section of the [Dockerfile](./Dockerfile.jetson) to build a container with models included.
|
||||
|
||||
#### Start the model server, awesomechat, and web app on Jetson Orin AGX
|
||||
|
||||
```bash
|
||||
# run the container which will automatically start the model server
|
||||
docker run --name jarvis --net=host --gpus all -v ~/jarvis/configs:/app/server/configs -v ~/src/JARVIS/server/models:/app/server/models toolboc/nv-jarvis:r35.2.1
|
||||
|
||||
# (wait for model server to complete initialization)
|
||||
|
||||
# start awesome_chat.py
|
||||
docker exec jarvis python3 awesome_chat.py --config configs/config.default.yaml --mode server
|
||||
|
||||
#start the web application (application will be acessible at http://localhost:9999)
|
||||
docker exec jarvis npm run dev --prefix=/app/web
|
||||
```
|
||||
|
||||
### Screenshots
|
||||
|
||||
<p align="center"><img src="./hugginggpt/assets/screenshot_q.jpg"><img src="./hugginggpt/assets/screenshot_a.jpg"></p>
|
||||
|
||||
|
||||
|
||||
|
||||
## Citation
|
||||
If you find this work useful in your method, you can cite the paper as below:
|
||||
|
||||
@inproceedings{shen2023hugginggpt,
|
||||
author = {Shen, Yongliang and Song, Kaitao and Tan, Xu and Li, Dongsheng and Lu, Weiming and Zhuang, Yueting},
|
||||
booktitle = {Advances in Neural Information Processing Systems},
|
||||
title = {HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace},
|
||||
year = {2023}
|
||||
}
|
||||
|
||||
>
|
||||
@article{shen2023taskbench,
|
||||
title = {TaskBench: Benchmarking Large Language Models for Task Automation},
|
||||
author = {Shen, Yongliang and Song, Kaitao and Tan, Xu and Zhang, Wenqi and Ren, Kan and Yuan, Siyu and Lu, Weiming and Li, Dongsheng and Zhuang, Yueting},
|
||||
journal = {arXiv preprint arXiv:2311.18760},
|
||||
year = {2023}
|
||||
}
|
||||
|
||||
>
|
||||
@article{yuan2024easytool,
|
||||
title = {EASYTOOL: Enhancing LLM-based Agents with Concise Tool Instruction},
|
||||
author = {Siyu Yuan and Kaitao Song and Jiangjie Chen and Xu Tan and Yongliang Shen and Ren Kan and Dongsheng Li and Deqing Yang},
|
||||
journal = {arXiv preprint arXiv:2401.06201},
|
||||
year = {2024}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`microsoft/JARVIS`
|
||||
- 原始仓库：https://github.com/microsoft/JARVIS
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.8 BLOCK -->
|
||||
|
||||
## Security
|
||||
|
||||
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
|
||||
|
||||
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
|
||||
|
||||
## Reporting Security Issues
|
||||
|
||||
**Please do not report security vulnerabilities through public GitHub issues.**
|
||||
|
||||
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
|
||||
|
||||
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
|
||||
|
||||
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
|
||||
|
||||
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
|
||||
|
||||
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help us triage your report more quickly.
|
||||
|
||||
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
|
||||
|
||||
## Preferred Languages
|
||||
|
||||
We prefer all communications to be in English.
|
||||
|
||||
## Policy
|
||||
|
||||
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
|
||||
|
||||
<!-- END MICROSOFT SECURITY.MD BLOCK -->
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# TODO: The maintainer of this repo has not yet edited this file
|
||||
|
||||
**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
|
||||
|
||||
- **No CSS support:** Fill out this template with information about how to file issues and get help.
|
||||
- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
|
||||
- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
|
||||
|
||||
*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
|
||||
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
|
||||
FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
|
||||
CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
@@ -0,0 +1,116 @@
|
||||
<p align="center">
|
||||
<img src="./assets/logo.png" width="15%"> <br>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
<h1>EasyTool</h1>
|
||||
<h3>Enhancing LLM-based Agents with Concise Tool Instruction<h3>
|
||||
</div>
|
||||
|
||||
## What's New
|
||||
+ [2024.01.15] We release Easytool for easier tool usage.
|
||||
+ The code and datasets are available at [easytool](#).
|
||||
+ The paper is available at [EASYTOOL: Enhancing LLM-based Agents with Concise Tool Instruction](https://arxiv.org/abs/2401.06201).
|
||||
|
||||
## Overview
|
||||
|
||||
LLM-based agents usually employ tool documentation to grasp the selection and usage of tools from different sources, but these documentations could be inconsistent in formats, redundant with excessive length, and lacking demonstrations for instructions.
|
||||
|
||||
EasyTool is an easy but effective method to create clear, structured, and unified instructions from tool documentations for improving LLM-based agents in using tools.
|
||||
|
||||
<p align="center">
|
||||
<img width="70%" alt="image" src="./assets/front.png">
|
||||
</p>
|
||||
|
||||
## Experiment
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Prepare requirements: `pip install -r requirements.txt`
|
||||
- Data Construction: `python3 data_process.py`
|
||||
|
||||
Before running any of the commands, ensure that you have set the necessary API keys. Replace `""` with your actual keys.
|
||||
```bash
|
||||
export OPENAI_API_KEY="your_openai_api_key_here"
|
||||
export RAPIDAPI_KEY="your_rapidapi_key_here"
|
||||
```
|
||||
### ToolBench
|
||||
You need first get the tool execution code (./data/toolenv/tools.) from the following link: [Google Drive](https://drive.google.com/drive/folders/1yBUQ732mPu-KclJnuQELEhtKakdXFc3J) or [Tsinghua Cloud](https://cloud.tsinghua.edu.cn/f/c9e50625743b40bfbe10/) and then save them to ./toolenv/tools
|
||||
To inference with LLMs, run the following commands:
|
||||
```bash
|
||||
unzip data_toolbench/tool_instruction/API_description_embeddings.zip -d data_toolbench/tool_instruction/
|
||||
|
||||
export OPENAI_API_KEY=""
|
||||
export RAPIDAPI_KEY=""
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task toolbench \
|
||||
--data_type G2 \
|
||||
--tool_root_dir ./toolenv/tools
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task toolbench \
|
||||
--data_type G3 \
|
||||
--tool_root_dir ./toolenv/tools
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task toolbench_retrieve \
|
||||
--data_type G2 \
|
||||
--tool_root_dir ./toolenv/tools
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task toolbench_retrieve \
|
||||
--data_type G3 \
|
||||
--tool_root_dir ./toolenv/tools
|
||||
```
|
||||
|
||||
### FuncQA
|
||||
|
||||
To inference with LLMs, run the following commands:
|
||||
```bash
|
||||
export OPENAI_API_KEY=""
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task funcqa \
|
||||
--data_type funcqa_mh
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task funcqa \
|
||||
--data_type funcqa_oh
|
||||
```
|
||||
|
||||
### RestBench
|
||||
|
||||
To inference with LLMs, run the following commands:
|
||||
```bash
|
||||
export OPENAI_API_KEY=""
|
||||
|
||||
python3 main.py \
|
||||
--model_name gpt-3.5-turbo \
|
||||
--task restbench
|
||||
```
|
||||
|
||||
## Citation
|
||||
|
||||
If you find this work useful in your method, you can cite the paper as below:
|
||||
|
||||
@article{yuan2024easytool,
|
||||
title = {EASYTOOL: Enhancing LLM-based Agents with Concise Tool Instruction},
|
||||
author = {Siyu Yuan and Kaitao Song and Jiangjie Chen and Xu Tan and Yongliang Shen and Ren Kan and Dongsheng Li and Deqing Yang},
|
||||
journal = {arXiv preprint arXiv:2401.06201},
|
||||
year = {2024}
|
||||
}
|
||||
|
||||
## Acknowledgement
|
||||
|
||||
- [ChatGPT](https://platform.openai.com/)
|
||||
- [Hugging Face](https://huggingface.co/)
|
||||
- [ToolBench](https://github.com/OpenBMB/ToolBench)
|
||||
- [RestBench](https://github.com/Yifan-Song793/RestGPT)
|
||||
- [FuncQA](https://github.com/Ber666/ToolkenGPT)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 110 KiB |
@@ -0,0 +1,162 @@
|
||||
import math
|
||||
|
||||
# this function is used to round the result to 2 decimal places
|
||||
# e.g. 52.3523 -> 52.35, 52.0011 -> 52, 0.00000233 -> 0.0000023
|
||||
def custom_round(x, decimal_places=2):
|
||||
str_x = f"{x:.10f}"
|
||||
before_decimal = str_x.split('.')[0]
|
||||
after_decimal = str_x.split('.')[1]
|
||||
leading_zeros = len(after_decimal) - len(after_decimal.lstrip('0'))
|
||||
|
||||
if leading_zeros >= 1 and before_decimal == "0":
|
||||
return round(x, leading_zeros + 2)
|
||||
else:
|
||||
return round(x, decimal_places)
|
||||
|
||||
# this function converts a number in scientific notation to decimal notation
|
||||
def scito_decimal(sci_str):
|
||||
def split_exponent(number_str):
|
||||
parts = number_str.split("e")
|
||||
coefficient = parts[0]
|
||||
exponent = int(parts[1]) if len(parts) == 2 else 0
|
||||
return coefficient, exponent
|
||||
|
||||
def multiplyby_10(number_str, exponent):
|
||||
if exponent == 0:
|
||||
return number_str
|
||||
|
||||
if exponent > 0:
|
||||
index = number_str.index(".") if "." in number_str else len(number_str)
|
||||
number_str = number_str.replace(".", "")
|
||||
new_index = index + exponent
|
||||
number_str += "0" * (new_index - len(number_str))
|
||||
if new_index < len(number_str):
|
||||
number_str = number_str[:new_index] + "." + number_str[new_index:]
|
||||
return number_str
|
||||
|
||||
if exponent < 0:
|
||||
index = number_str.index(".") if "." in number_str else len(number_str)
|
||||
number_str = number_str.replace(".", "")
|
||||
new_index = index + exponent
|
||||
number_str = "0" * (-new_index) + number_str
|
||||
number_str = "0." + number_str
|
||||
return number_str
|
||||
|
||||
coefficient, exponent = split_exponent(sci_str)
|
||||
decimal_str = multiplyby_10(coefficient, exponent)
|
||||
|
||||
# remove trailing zeros
|
||||
if "." in decimal_str:
|
||||
decimal_str = decimal_str.rstrip("0")
|
||||
|
||||
return decimal_str
|
||||
|
||||
# normalize the result to 2 decimal places and remove trailing zeros
|
||||
def normalize(res, round_to=2):
|
||||
# we round the result to 2 decimal places
|
||||
res = custom_round(res, round_to)
|
||||
res = str(res)
|
||||
if "." in res:
|
||||
while res[-1] == "0":
|
||||
res = res[:-1]
|
||||
res = res.strip(".")
|
||||
|
||||
# scientific notation
|
||||
if "e" in res:
|
||||
res = scito_decimal(res)
|
||||
|
||||
return res
|
||||
|
||||
# 1. add
|
||||
def add_(args):
|
||||
|
||||
return normalize(sum(args))
|
||||
|
||||
# 2. subtract
|
||||
def subtract_(args):
|
||||
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res -= arg
|
||||
return normalize(res)
|
||||
|
||||
# 3. multiply
|
||||
def multiply_(args):
|
||||
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res *= arg
|
||||
return normalize(res)
|
||||
|
||||
# 4. divide
|
||||
def divide_(args):
|
||||
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res /= arg
|
||||
return normalize(res)
|
||||
|
||||
# 5. power
|
||||
def power_(args):
|
||||
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res **= arg
|
||||
return normalize(res)
|
||||
|
||||
# 6. square root
|
||||
def sqrt_(args):
|
||||
res = args[0]
|
||||
return normalize(math.sqrt(res))
|
||||
|
||||
# 7. 10th log
|
||||
def log_(args):
|
||||
# if only one argument is passed, it is 10th log
|
||||
if len(args) == 1:
|
||||
res = args[0]
|
||||
return normalize(math.log10(res))
|
||||
# if two arguments are passed, it is log with base as the second argument
|
||||
elif len(args) == 2:
|
||||
res = args[0]
|
||||
base = args[1]
|
||||
return normalize(math.log(res, base))
|
||||
else:
|
||||
raise Exception("Invalid number of arguments passed to log function")
|
||||
|
||||
# 8. natural log
|
||||
def ln_(args):
|
||||
res = args[0]
|
||||
return normalize(math.log(res))
|
||||
|
||||
|
||||
# 9. choose
|
||||
def choose_(args):
|
||||
n = args[0]
|
||||
r = args[1]
|
||||
return normalize(math.comb(n, r))
|
||||
|
||||
# 10. permutation
|
||||
def permutate_(args):
|
||||
n = args[0]
|
||||
r = args[1]
|
||||
return normalize(math.perm(n, r))
|
||||
|
||||
# 11. greatest common divisor
|
||||
def gcd_(args):
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res = math.gcd(res, arg)
|
||||
return normalize(res)
|
||||
|
||||
# 12. least common multiple
|
||||
def lcm_(args):
|
||||
res = args[0]
|
||||
for arg in args[1:]:
|
||||
res = res * arg // math.gcd(res, arg)
|
||||
return normalize(res)
|
||||
|
||||
# 13. remainder
|
||||
def remainder_(args):
|
||||
dividend = args[0]
|
||||
divisor = args[1]
|
||||
return normalize(dividend % divisor)
|
||||
@@ -0,0 +1,249 @@
|
||||
{
|
||||
"0":{
|
||||
"ID":0,
|
||||
"standardized_name":"add_",
|
||||
"API_description":"'add_' returns the sum of all the arguments passed to it, normalized to 2 decimal places.",
|
||||
"Usage":{
|
||||
"required_parameters":[
|
||||
{
|
||||
"name":"input",
|
||||
"type":"List"
|
||||
}
|
||||
],
|
||||
"Example":{
|
||||
"Scenario":"if you want to add 2 to 1.",
|
||||
"Parameters":{
|
||||
"input":[2,1]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"1": {
|
||||
"ID": 1,
|
||||
"standardized_name": "subtract_",
|
||||
"API_description": "'subtract_' returns the difference of the arguments passed to it, starting with the first argument and subtracting all subsequent arguments, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to subtract 2 from 1.",
|
||||
"Parameters": {
|
||||
"input": [1,2]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"ID": 2,
|
||||
"standardized_name": "multiply_",
|
||||
"API_description": "'multiply_' returns the product of all the arguments passed to it, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to calculate 2*1.",
|
||||
"Parameters": {
|
||||
"input": [2,1]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"ID": 3,
|
||||
"standardized_name": "divide_",
|
||||
"API_description": "'divide_' returns the quotient of the first argument divided by all the subsequent arguments, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to calculate 4/2.",
|
||||
"Parameters": {
|
||||
"input": [4,2]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"ID": 4,
|
||||
"standardized_name": "power_",
|
||||
"API_description": "'power_' returns the result of raising the first argument to the power of all the subsequent arguments, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to calculate 2^3.",
|
||||
"Parameters": {
|
||||
"input": [2,3]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"ID": 5,
|
||||
"standardized_name": "sqrt_",
|
||||
"API_description": "'sqrt_' returns the square root of the first argument, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the square root of 9.",
|
||||
"Parameters": {
|
||||
"input": [9]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"ID": 6,
|
||||
"standardized_name": "log_",
|
||||
"API_description": "'log_' returns the base-10 logarithm of the first argument if one argument is provided, or the logarithm with base as the second argument if two arguments are provided, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the logarithm of 100 with base 10.",
|
||||
"Parameters": {
|
||||
"input": [100,10]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"ID": 7,
|
||||
"standardized_name": "ln_",
|
||||
"API_description": "'ln_' returns the natural logarithm of the first argument, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the natural logarithm of 2.718.",
|
||||
"Parameters": {
|
||||
"input": [2.718281828459045]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"ID": 8,
|
||||
"standardized_name": "lcm_",
|
||||
"API_description": "'lcm_' returns the least common multiple of all the arguments passed to it, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to find the LCM of 12 and 18.",
|
||||
"Parameters": {
|
||||
"input": [12,18]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"9": {
|
||||
"ID": 9,
|
||||
"standardized_name": "gcd_",
|
||||
"API_description": "'gcd_' returns the greatest common divisor of all the arguments passed to it, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to find the GCD of 54 and 24.",
|
||||
"Parameters": {
|
||||
"input": [54,24]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"ID": 10,
|
||||
"standardized_name": "remainder_",
|
||||
"API_description": "'remainder_' returns the remainder of the division of the first argument by the second argument, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to find the remainder of 10 divided by 3.",
|
||||
"Parameters": {
|
||||
"input": [10,3]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"11": {
|
||||
"ID": 11,
|
||||
"standardized_name": "choose_",
|
||||
"API_description": "'choose_' returns the number of ways to choose 'r' items from 'n' options without regard to order, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to choose 2 items out of 5.",
|
||||
"Parameters": {
|
||||
"input": [5,2]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"12": {
|
||||
"ID": 12,
|
||||
"standardized_name": "permutate_",
|
||||
"API_description": "'permutate_' returns the number of ways to arrange 'r' items out of 'n' options, normalized to 2 decimal places.",
|
||||
"Usage": {
|
||||
"required_parameters": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "List"
|
||||
}
|
||||
],
|
||||
"Example": {
|
||||
"Scenario": "if you want to find the number of arrangements of 3 items out of 5.",
|
||||
"Parameters": {
|
||||
"input": [5,3]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{"ID": 0, "description": "'add_' returns the sum of all the arguments passed to it, normalized to 2 decimal places."}
|
||||
{"ID": 1, "description": "'subtract_' returns the difference of the arguments passed to it, starting with the first argument and subtracting all subsequent arguments, normalized to 2 decimal places."}
|
||||
{"ID": 2, "description": "'multiply_' returns the product of all the arguments passed to it, normalized to 2 decimal places."}
|
||||
{"ID": 3, "description": "'divide_' returns the quotient of the first argument divided by all the subsequent arguments, normalized to 2 decimal places."}
|
||||
{"ID": 4, "description": "'power_' returns the result of raising the first argument to the power of all the subsequent arguments, normalized to 2 decimal places."}
|
||||
{"ID": 5, "description": "'sqrt_' returns the square root of the first argument, normalized to 2 decimal places."}
|
||||
{"ID": 6, "description": "'log_' returns the base-10 logarithm of the first argument if one argument is provided, or the logarithm with base as the second argument if two arguments are provided, normalized to 2 decimal places."}
|
||||
{"ID": 7, "description": "'ln_' returns the natural logarithm of the first argument, normalized to 2 decimal places."}
|
||||
{"ID": 8, "description": "'lcm_' returns the least common multiple of all the arguments passed to it, normalized to 2 decimal places."}
|
||||
{"ID": 9, "description": "'gcd_' returns the greatest common divisor of all the arguments passed to it, normalized to 2 decimal places."}
|
||||
{"ID": 10, "description": "'remainder_' returns the remainder of the division of the first argument by the second argument, normalized to 2 decimal places."}
|
||||
{"ID": 11, "description": "'choose_' returns the number of ways to choose 'r' items from 'n' options without regard to order, normalized to 2 decimal places."}
|
||||
{"ID": 12, "description": "'permutate_' returns the number of ways to arrange 'r' items out of 'n' options, normalized to 2 decimal places."}
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import gdown
|
||||
import shutil
|
||||
import json
|
||||
from zipfile import ZipFile
|
||||
|
||||
urls = {
|
||||
"funcqa": "https://drive.google.com/uc?id=13Sj7uIsyqWXoTh1ejWUviTzeQSES2Omd",
|
||||
"restbench": "https://raw.githubusercontent.com/Yifan-Song793/RestGPT/main/datasets/tmdb.json",
|
||||
"toolbench": "https://drive.google.com/uc?id=1XFjDxVZdUY7TXYF2yvzx3pJlS2fy78jk",
|
||||
}
|
||||
|
||||
|
||||
def read_jsonline(address):
|
||||
not_mark = []
|
||||
with open(address, 'r', encoding="utf-8") as f:
|
||||
for jsonstr in f.readlines():
|
||||
jsonstr = json.loads(jsonstr)
|
||||
not_mark.append(jsonstr)
|
||||
return not_mark
|
||||
|
||||
|
||||
def read_json(address):
|
||||
with open(address, 'r', encoding='utf-8') as json_file:
|
||||
json_data = json.load(json_file)
|
||||
return json_data
|
||||
|
||||
|
||||
def toolbench_process(data_file, dataset):
|
||||
ls = read_json(data_file)
|
||||
all_data = read_json(f"{dataset}/tool_instruction/toolbench_tool_instruction.json")
|
||||
all_dic = {}
|
||||
for ID in all_data.keys():
|
||||
all_dic[all_data[ID]["tool_name"]] = all_data[ID]
|
||||
|
||||
not_in = []
|
||||
for data in ls:
|
||||
Tool_dic = []
|
||||
data_dic = {}
|
||||
already = []
|
||||
for tool in data['api_list']:
|
||||
if tool['tool_name'] in all_dic:
|
||||
if all_dic[tool['tool_name']]["ID"] not in already:
|
||||
already.append(all_dic[tool['tool_name']]["ID"])
|
||||
Tool_dic.append({"ID": all_dic[tool['tool_name']]["ID"],
|
||||
"Description": all_dic[tool['tool_name']]["tool_description"], })
|
||||
data["Tool_dic"] = Tool_dic
|
||||
|
||||
json_str = json.dumps(ls, indent=4)
|
||||
with open(data_file, 'w', encoding='utf-8') as json_file:
|
||||
json.dump(ls, json_file, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
def main():
|
||||
curr_dir = os.path.dirname(__file__)
|
||||
|
||||
for dataset in [
|
||||
"funcqa",
|
||||
"restbench",
|
||||
"toolbench"
|
||||
]:
|
||||
dataset_path = os.path.join(curr_dir, "data_{}".format(dataset), "test_data")
|
||||
|
||||
if not os.path.exists(dataset_path):
|
||||
os.makedirs(dataset_path)
|
||||
|
||||
if dataset == "funcqa":
|
||||
print("Processing FuncQA dataset ...\n")
|
||||
# Processing FuncQA dataset ...
|
||||
temp_file = os.path.join(dataset_path, "data_toolkengpt_0918.zip")
|
||||
gdown.download(urls[dataset], temp_file, quiet=False)
|
||||
zf = ZipFile(temp_file, 'r')
|
||||
zf.extract("data/funcqa/funcqa_oh.json", ".")
|
||||
zf.extract("data/funcqa/funcqa_mh.json", ".")
|
||||
os.rename("data/funcqa/funcqa_oh.json", "{}/funcqa_oh.json".format(dataset_path))
|
||||
os.rename("data/funcqa/funcqa_mh.json", "{}/funcqa_mh.json".format(dataset_path))
|
||||
os.remove(temp_file)
|
||||
shutil.rmtree("data")
|
||||
|
||||
print("FuncQA dataset Done!\n")
|
||||
|
||||
if dataset == "restbench":
|
||||
print("Processing RestBench dataset ... \n")
|
||||
# Processing RestBench Dataset
|
||||
os.system("wget -P {} -c {}".format(dataset_path, urls[dataset]))
|
||||
|
||||
print("RestBench dataset Done!\n")
|
||||
|
||||
if dataset == "toolbench":
|
||||
print("Processing ToolBench dataset ... \n")
|
||||
# Processing ToolBench Dataset
|
||||
temp_file = os.path.join(dataset_path, "data.zip")
|
||||
gdown.download(urls[dataset], temp_file, quiet=False)
|
||||
zf = ZipFile(temp_file, 'r')
|
||||
zf.extract("data/test_instruction/G2_category.json", ".")
|
||||
zf.extract("data/test_instruction/G3_instruction.json", ".")
|
||||
|
||||
os.rename("data/test_instruction/G2_category.json", "{}/G2_category.json".format(dataset_path))
|
||||
os.rename("data/test_instruction/G3_instruction.json", "{}/G3_instruction.json".format(dataset_path))
|
||||
toolbench_process("{}/G2_category.json".format(dataset_path), "data_{}".format(dataset))
|
||||
toolbench_process("{}/G3_instruction.json".format(dataset_path), "data_{}".format(dataset))
|
||||
os.remove(temp_file)
|
||||
shutil.rmtree("data")
|
||||
|
||||
print("Toolbench dataset Done!\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,650 @@
|
||||
[
|
||||
{
|
||||
"ID": 1,
|
||||
"tool_name": "/movie/{movie_id}/keywords",
|
||||
"tool_description": "Get the keywords that have been added to a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/keywords",
|
||||
"Example": {
|
||||
"Scenario": "if you want to find out the keywords that have been added to a movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456/keywords"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 2,
|
||||
"tool_name": "/tv/popular",
|
||||
"tool_description": "Get a list of the current popular TV shows on TMDb.",
|
||||
"tool_usage": "GET /tv/popular",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get a list of the current popular TV shows on TMDb.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/popular"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 3,
|
||||
"tool_name": "/person/{person_id}",
|
||||
"tool_description": "Get the primary person details by id. You should first know the person_id and thus this tool should be used after /search/person.",
|
||||
"tool_usage": "GET /person/{person_id}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the primary details of person with person_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /person/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 4,
|
||||
"tool_name": "/movie/{movie_id}/reviews",
|
||||
"tool_description": "Get the user reviews for a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/reviews",
|
||||
"Example": {
|
||||
"Scenario": "If you want to read the user reviews for a movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456/reviews"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 5,
|
||||
"tool_name": "/movie/{movie_id}/release_dates",
|
||||
"tool_description": "Get the release date along with the certification for a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/release_dates",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the release dates and certification of the movie with movie_id 123.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/123/release_dates"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 6,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}/episode/{episode_number}/credits",
|
||||
"tool_description": "Get the credits (cast, crew and guest stars) for a TV episode. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}/episode/{episode_number}/credits",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get the credits of a TV episode with tv_id 456, season_number 2, and episode_number 3.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2/episode/3/credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 7,
|
||||
"tool_name": "/movie/{movie_id}/images",
|
||||
"tool_description": "Get the images that belong to a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the images of a movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 8,
|
||||
"tool_name": "/search/tv",
|
||||
"tool_description": "Search for a TV show, which can obtain tv_id.",
|
||||
"tool_usage": "GET /search/tv",
|
||||
"Example": {
|
||||
"Scenario": "If you want to search for the id of a TV show with the name 'Friends'",
|
||||
"Parameters": {
|
||||
"input": "GET /search/tv?query=Friends"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 9,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}/episode/{episode_number}",
|
||||
"tool_description": "Get the TV episode details by id.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}/episode/{episode_number}",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get details of the 3rd episode from the 2nd season of the TV show with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2/episode/3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 10,
|
||||
"tool_name": "/network/{network_id}/images",
|
||||
"tool_description": "Get the TV network logos by id.",
|
||||
"tool_usage": "GET /network/{network_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to retrieve the logos of a TV network with network_id 50.",
|
||||
"Parameters": {
|
||||
"input": "GET /network/50/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 11,
|
||||
"tool_name": "/genre/tv/list",
|
||||
"tool_description": "Get the list of official genres for TV shows.",
|
||||
"tool_usage": "GET /genre/tv/list",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the list of official genres for TV shows.",
|
||||
"Parameters": {
|
||||
"input": "GET /genre/tv/list"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 12,
|
||||
"tool_name": "/search/movie",
|
||||
"tool_description": "Search for movies, which can obtain movie_id.",
|
||||
"tool_usage": "GET /search/movie",
|
||||
"Example": {
|
||||
"Scenario": "If you want to search for the id of a movie with the name 'Avatar'",
|
||||
"Parameters": {
|
||||
"input": "GET /search/movie?query=Avatar"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 13,
|
||||
"tool_name": "/discover/movie",
|
||||
"tool_description": "Discover movies by different types of data like average rating, number of votes, genres and certifications",
|
||||
"tool_usage": "GET /discover/movie",
|
||||
"Example": {
|
||||
"Scenario": "If you want to discover movies by different types of data like average rating, number of votes, genres and certifications.",
|
||||
"Parameters": {
|
||||
"input": "GET /discover/movie"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 14,
|
||||
"tool_name": "/movie/upcoming",
|
||||
"tool_description": "Get a list of upcoming movies in theatres.",
|
||||
"tool_usage": "GET /movie/upcoming",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get a list of upcoming movies in theatres.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/upcoming"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 15,
|
||||
"tool_name": "/credit/{credit_id}",
|
||||
"tool_description": "Get a movie or TV credit details by id.",
|
||||
"tool_usage": "GET /credit/{credit_id}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the details of movie or TV credit with credit_id as 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /credit/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 16,
|
||||
"tool_name": "/person/{person_id}/tv_credits",
|
||||
"tool_description": "Get the TV show credits for a person. You should first know the person_id and thus this tool should be used after /search/person.",
|
||||
"tool_usage": "GET /person/{person_id}/tv_credits",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get the TV show credits of a person with person_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /person/456/tv_credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 17,
|
||||
"tool_name": "/tv/latest",
|
||||
"tool_description": "Get the most newly created TV show.",
|
||||
"tool_usage": "GET /tv/latest",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get the most newly created TV show.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/latest"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 18,
|
||||
"tool_name": "/company/{company_id}",
|
||||
"tool_description": "Get a companies details by id. You should first know the company_id and thus this tool should be used after /search/company.",
|
||||
"tool_usage": "GET /company/{company_id}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the details of a company with company_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /company/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 19,
|
||||
"tool_name": "/tv/{tv_id}/images",
|
||||
"tool_description": "Get the images that belong to a TV show. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the images of a TV show with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 20,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}/credits",
|
||||
"tool_description": "Get the credits for TV season. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}/credits",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the credits for a TV season with tv_id 456 and season_number 2.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2/credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 21,
|
||||
"tool_name": "/movie/now_playing",
|
||||
"tool_description": "Get a list of movies in theatres.",
|
||||
"tool_usage": "GET /movie/now_playing",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get a list of movies currently playing in theatres.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/now_playing"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 22,
|
||||
"tool_name": "/review/{review_id}",
|
||||
"tool_description": "Get review from a reviewer",
|
||||
"tool_usage": "GET /review/{review_id}",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get a review with review_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /review/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 23,
|
||||
"tool_name": "/tv/on_the_air",
|
||||
"tool_description": "Get a list of shows that are currently on the air.",
|
||||
"tool_usage": "GET /tv/on_the_air",
|
||||
"Example": {
|
||||
"Scenario": "If you want to know the list of shows currently on air.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/on_the_air"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 24,
|
||||
"tool_name": "/movie/{movie_id}",
|
||||
"tool_description": "Get the primary information about a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the primary information about a movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 25,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}/images",
|
||||
"tool_description": "Get the images that belong to a TV season. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the images of a specific season of a TV show with tv_id 456 and season_number 2.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 26,
|
||||
"tool_name": "/company/{company_id}/images",
|
||||
"tool_description": "Get a companies logos by id . You should first know the company_id and thus this tool should be used after /search/company.",
|
||||
"tool_usage": "GET /company/{company_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to fetch the logos of the company with company_id 789.",
|
||||
"Parameters": {
|
||||
"input": "GET /company/789/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 27,
|
||||
"tool_name": "/tv/{tv_id}/reviews",
|
||||
"tool_description": "Get the reviews for a TV show. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/reviews",
|
||||
"Example": {
|
||||
"Scenario": "if you want to read the reviews of a TV show with tv_id 567.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/567/reviews"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 28,
|
||||
"tool_name": "/tv/{tv_id}/similar",
|
||||
"tool_description": "Get a list of similar TV shows. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/similar",
|
||||
"Example": {
|
||||
"Scenario": "If you want to find similar TV shows to the one with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/similar"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 29,
|
||||
"tool_name": "/network/{network_id}",
|
||||
"tool_description": "Get the details of a network.",
|
||||
"tool_usage": "GET /network/{network_id}",
|
||||
"Example": {
|
||||
"Scenario": "If you want to know the details of a network with network_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /network/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 30,
|
||||
"tool_name": "/tv/{tv_id}/recommendations",
|
||||
"tool_description": "Get the list of TV show recommendations for this item. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/recommendations",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get recommendations for a TV show with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/recommendations"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 31,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}/episode/{episode_number}/images",
|
||||
"tool_description": "Get the images that belong to a TV episode. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}/episode/{episode_number}/images",
|
||||
"Example": {
|
||||
"Scenario": "If you want to know the images of a TV episode with tv_id 456, season_number 2 and episode_number 3.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2/episode/3/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 32,
|
||||
"tool_name": "/movie/popular",
|
||||
"tool_description": "Get a list of the current popular movies on TMDb.",
|
||||
"tool_usage": "GET /movie/popular",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get a list of the current popular movies on TMDb.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/popular"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 33,
|
||||
"tool_name": "/tv/airing_today",
|
||||
"tool_description": "Get a list of TV shows that are airing today.",
|
||||
"tool_usage": "GET /tv/airing_today",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get a list of TV shows that are airing today.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/airing_today"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 34,
|
||||
"tool_name": "/tv/{tv_id}/keywords",
|
||||
"tool_description": "Get the keywords that have been added to a TV show. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/keywords",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the keywords of a TV show with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/keywords"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 35,
|
||||
"tool_name": "/search/person",
|
||||
"tool_description": "Search for people, which can obtain person_id.",
|
||||
"tool_usage": "GET /search/person",
|
||||
"Example": {
|
||||
"Scenario": "If you want to search for the id of a person with the name 'Leonardo DiCaprio'",
|
||||
"Parameters": {
|
||||
"input": "GET /search/person?query=Leonardo%20DiCaprio"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 36,
|
||||
"tool_name": "/search/company",
|
||||
"tool_description": "Search for companies, which can obtain company_id.",
|
||||
"tool_usage": "GET /search/company",
|
||||
"Example": {
|
||||
"Scenario": "If you want to search for the id of a company with the name 'Disney'",
|
||||
"Parameters": {
|
||||
"input": "GET /search/company?query=Disney"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 37,
|
||||
"tool_name": "/discover/tv",
|
||||
"tool_description": "Discover TV shows by different types of data like average rating, number of votes, genres, the network they aired on and air dates.",
|
||||
"tool_usage": "GET /discover/tv",
|
||||
"Example": {
|
||||
"Scenario": "If you want to discover TV shows by their average rating, number of votes, genres, the network they aired on and air dates.",
|
||||
"Parameters": {
|
||||
"input": "GET /discover/tv"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 38,
|
||||
"tool_name": "/movie/top_rated",
|
||||
"tool_description": "Get the top rated movies on TMDb.",
|
||||
"tool_usage": "GET /movie/top_rated",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get the top rated movies on TMDb.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/top_rated"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 39,
|
||||
"tool_name": "/movie/latest",
|
||||
"tool_description": "Get the most newly created movie.",
|
||||
"tool_usage": "GET /movie/latest",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the details of the most recently created movie.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/latest"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 40,
|
||||
"tool_name": "/tv/top_rated",
|
||||
"tool_description": "Get a list of the top rated TV shows on TMDb.",
|
||||
"tool_usage": "GET /tv/top_rated",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get a list of top rated TV shows on TMDb.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/top_rated"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 41,
|
||||
"tool_name": "/trending/{media_type}/{time_window}",
|
||||
"tool_description": "Get the daily or weekly trending items.",
|
||||
"tool_usage": "GET /trending/{media_type}/{time_window}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the weekly trending movies.",
|
||||
"Parameters": {
|
||||
"input": "GET /trending/movie/week"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 42,
|
||||
"tool_name": "/genre/movie/list",
|
||||
"tool_description": "Get the list of official genres for movies.",
|
||||
"tool_usage": "GET /genre/movie/list",
|
||||
"Example": {
|
||||
"Scenario": "If you want to get the list of official genres for movies.",
|
||||
"Parameters": {
|
||||
"input": "GET /genre/movie/list"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 43,
|
||||
"tool_name": "/tv/{tv_id}/season/{season_number}",
|
||||
"tool_description": "Get the TV season details by id. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/season/{season_number}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the details of season 2 from the TV series with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/season/2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 44,
|
||||
"tool_name": "/collection/{collection_id}",
|
||||
"tool_description": "Get collection details by id. You should first know the collection_id and thus this tool should be used after /search/collection.",
|
||||
"tool_usage": "GET /collection/{collection_id}",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get details of a collection with collection_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /collection/456"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 45,
|
||||
"tool_name": "/person/{person_id}/images",
|
||||
"tool_description": "Get the images for a person. You should first know the person_id and thus this tool should be used after /search/person.",
|
||||
"tool_usage": "GET /person/{person_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the images of a person with person_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /person/456/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 46,
|
||||
"tool_name": "/tv/{tv_id}",
|
||||
"tool_description": "Get the primary TV show details by id. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}",
|
||||
"Example": {
|
||||
"Scenario": "If you want to retrieve the primary details of a TV show with tv_id 789.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/789"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 47,
|
||||
"tool_name": "/person/popular",
|
||||
"tool_description": "Get the list of popular people on TMDb.",
|
||||
"tool_usage": "GET /person/popular",
|
||||
"Example": {
|
||||
"Scenario": "If you want to fetch the list of popular people on TMDb.",
|
||||
"Parameters": {
|
||||
"input": "GET /person/popular"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 48,
|
||||
"tool_name": "/collection/{collection_id}/images",
|
||||
"tool_description": "Get the images for a collection by id. You should first know the collection_id and thus this tool should be used after /search/collection.",
|
||||
"tool_usage": "GET /collection/{collection_id}/images",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the images of a collection with collection_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /collection/456/images"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 49,
|
||||
"tool_name": "/tv/{tv_id}/credits",
|
||||
"tool_description": "Get the credits (cast and crew) that have been added to a TV show. You should first know the tv_id and thus this tool should be used after /search/tv.",
|
||||
"tool_usage": "GET /tv/{tv_id}/credits",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the credits of a TV show with tv_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /tv/456/credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 50,
|
||||
"tool_name": "/person/{person_id}/movie_credits",
|
||||
"tool_description": "Get the movie credits for a person, the results contains various information such as popularity and release date. You should first know the person_id and thus this tool should be used after /search/person.",
|
||||
"tool_usage": "GET /person/{person_id}/movie_credits",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get the movie credits for a person with person_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /person/456/movie_credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 51,
|
||||
"tool_name": "/movie/{movie_id}/recommendations",
|
||||
"tool_description": "Get a list of recommended movies for a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/recommendations",
|
||||
"Example": {
|
||||
"Scenario": "if you want to get a list of recommended movies for the movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456/recommendations"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 52,
|
||||
"tool_name": "/search/collection",
|
||||
"tool_description": "Search for collections, which can obtain collection_id.",
|
||||
"tool_usage": "GET /search/collection",
|
||||
"Example": {
|
||||
"Scenario": "if you want to search for a collection id of Star Wars.",
|
||||
"Parameters": {
|
||||
"input": "GET /search/collection?query=Star%20Wars"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 53,
|
||||
"tool_name": "/movie/{movie_id}/credits",
|
||||
"tool_description": "Get the cast and crew for a movie. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/credits",
|
||||
"Example": {
|
||||
"Scenario": "if you want to know the cast and crew of a movie with movie_id 789.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/789/credits"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": 54,
|
||||
"tool_name": "/movie/{movie_id}/similar",
|
||||
"tool_description": "Get a list of similar movies. You should first know the movie_id and thus this tool should be used after /search/movie.",
|
||||
"tool_usage": "GET /movie/{movie_id}/similar",
|
||||
"Example": {
|
||||
"Scenario": "if you want to find movies similar to the movie with movie_id 456.",
|
||||
"Parameters": {
|
||||
"input": "GET /movie/456/similar"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
# __init__.py
|
||||
@@ -0,0 +1,675 @@
|
||||
# — coding: utf-8 –
|
||||
import openai
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import (
|
||||
ChatPromptTemplate,
|
||||
MessagesPlaceholder,
|
||||
SystemMessagePromptTemplate,
|
||||
HumanMessagePromptTemplate
|
||||
)
|
||||
from langchain import LLMChain
|
||||
import numpy as np
|
||||
import requests
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import importlib.util
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
import pickle
|
||||
from util import *
|
||||
from tqdm import tqdm
|
||||
|
||||
openai.api_key = os.environ["OPENAI_API_KEY"]
|
||||
|
||||
|
||||
def get_last_processed_index(progress_file):
|
||||
"""Retrieve the last processed index from the progress file."""
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
last_index = f.read().strip()
|
||||
return int(last_index) if last_index else 0
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def update_progress(progress_file, index):
|
||||
"""Update the last processed index in the progress file."""
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
f.write(str(index))
|
||||
|
||||
|
||||
def choose_tool(question, Tool_dic, tool_used, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is the user's question: {question}\n"
|
||||
"These are the tools you can select to solve the question:\n"
|
||||
"Tool List:\n"
|
||||
"{Too_list}\n\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only chooce one tool the Tool List to solve this question.\n"
|
||||
"2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"ID\": 1}}\n"
|
||||
"Example 2: {{\"ID\": 2}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
Tool_list = []
|
||||
for ele in Tool_dic:
|
||||
for key in ele.keys():
|
||||
if str(key) not in tool_used:
|
||||
Tool_list.append(f'''ID: {key}\n{ele[key]}''')
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
Too_list=Tool_dic)
|
||||
clean_answer = eval(result.split("(")[0].strip())
|
||||
# clean_answer = lowercase_parameter_keys(clean_answer)
|
||||
# print(clean_answer)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"choose tool fails: {e}")
|
||||
print(result)
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return clean_answer
|
||||
|
||||
|
||||
def task_decompose(question, Tool_dic, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step.\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is tool list:\n"
|
||||
"{Tool_list}\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using one single tool in the tool list.\n"
|
||||
"2. If one subtask need the results from other subtask, you can should write clearly. For example:"
|
||||
"{{\"Tasks\": [\"Convert 23 km/h to X km/min by 'divide_'\", \"Multiply X km/min by 45 min to get Y by 'multiply_'\"]}}\n"
|
||||
"3. You must ONLY output in a parsible JSON format. An example output looks like:\n"
|
||||
"'''\n"
|
||||
"{{\"Tasks\": [\"Task 1\", \"Task 2\", ...]}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
Tool_list = []
|
||||
for ele in Tool_dic:
|
||||
Tool_list.append(str(ele))
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question, Tool_list=Tool_list)
|
||||
result = eval(result.split('\n\n')[0])
|
||||
a = result["Tasks"]
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"task decompose fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def task_topology(question, task_ls, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a complex user's question, I have decompose this question into some simple subtasks"
|
||||
"I think there exists a logical connections and order amontg the tasks. "
|
||||
"Thus you need to help me output this logical connections and order.\n"
|
||||
"You must ONLY output in a parsible JSON format with the following format:\n"
|
||||
"'''\n"
|
||||
"[{{\"task\": task, \"id\", task_id, \"dep\": [dependency_task_id1, dependency_task_id2, ...]}}]\n"
|
||||
"'''\n"
|
||||
"The \"dep\" field denotes the id of the previous task which generates a new resource upon which the current task depends. If there are no dependencies, set \"dep\" to -1.\n\n"
|
||||
"This is user's question: {question}\n"
|
||||
"These are subtasks of this question:\n"
|
||||
"{task_ls}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question, task_ls=task_ls)
|
||||
result = eval(result)
|
||||
for i in range(len(result)):
|
||||
if isinstance(result[i]['dep'], str):
|
||||
temp = []
|
||||
for ele in result[i]['dep'].split(','):
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif isinstance(result[i]['dep'], int):
|
||||
result[i]['dep'] = [result[i]['dep']]
|
||||
elif isinstance(result[i]['dep'], list):
|
||||
temp = []
|
||||
for ele in result[i]['dep']:
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif result[i]['dep'] == -1:
|
||||
result[i]['dep'] = [-1]
|
||||
a = result[i]['dep'][0]
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"task topology fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_direct(task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to answer the user's question.\n"
|
||||
"This is the user's question: {task}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(task=task)
|
||||
return result
|
||||
|
||||
|
||||
def choose_parameter(API_instruction, api, api_dic, question, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is an API tool documentation. Given a user's question, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. If you need to use this API multiple times, please set \"Parameters\" to a list.\n"
|
||||
"5. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"input\": [1,2,3]}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"input\": [1,2,3]}}, {{\"input\": [2,3,4]}}]}}\n"
|
||||
"'''\n"
|
||||
"This is user's question: {question}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question, )
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(":true", ": True").replace(":false", ": False").replace(
|
||||
": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"Choose Parameter fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def choose_parameter_depend(API_instruction, api, api_dic, question, model_name, previous_log):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.\n"
|
||||
"5. If you need to use this API multiple times,, please set \"Parameters\" to a list.\n"
|
||||
"6. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"input\": [1,2,3]}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"input\": [1,2,3]}}, {{\"input\": [2,3,4]}}]}}\n"
|
||||
"'''\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the current user's question: {question}\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question,
|
||||
previous_log=previous_log)
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"choose parameter depend fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def Call_function(B, arg, id):
|
||||
app_path = 'data_funcqa/funchub/math.py'
|
||||
spec = importlib.util.spec_from_file_location('math', app_path)
|
||||
app_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(app_module)
|
||||
if hasattr(app_module, B):
|
||||
function_B = getattr(app_module, B)
|
||||
try:
|
||||
call_result = function_B(arg['input'])
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.lower()): v for k, v in arg.items()}
|
||||
call_result = function_B(arg['input'])
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.lower()): v for k, v in arg.items()}
|
||||
arg = {change_name(k.replace("-", "_")): v for k, v in arg.items()}
|
||||
call_result = function_B(arg['input'])
|
||||
return call_result
|
||||
except Exception as e:
|
||||
print(f"fails: {e}")
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": str(e)
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return -1
|
||||
else:
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": f"No function named {B} in {app_path}"
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return (f"No function named {B} in {app_path}")
|
||||
|
||||
|
||||
def retrieval(question, Tool_dic, dataset, tool_used, ind, model_name, previous_log=None):
|
||||
tool_id = choose_tool(question, Tool_dic, tool_used, model_name)
|
||||
if tool_id == -1:
|
||||
return tool_id, "", "", "", ""
|
||||
tool_instruction = dataset[str(tool_id["ID"])]
|
||||
API_instruction = tool_instruction["API_description"]
|
||||
API_tool = tool_instruction["standardized_name"]
|
||||
|
||||
api_selection = [API_tool]
|
||||
api_result = []
|
||||
for api in api_selection:
|
||||
if previous_log is None:
|
||||
parameter = choose_parameter(API_instruction, api,
|
||||
tool_instruction["Usage"], question, model_name)
|
||||
else:
|
||||
parameter = choose_parameter_depend(API_instruction, api,
|
||||
tool_instruction["Usage"],
|
||||
question, model_name, previous_log)
|
||||
if parameter == -1:
|
||||
continue
|
||||
api_result.append({"api_name": api, "parameters": parameter})
|
||||
if len(api_result) == 0:
|
||||
call_result = ""
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
if isinstance(api_result, set) or isinstance(api_result, list):
|
||||
call_results = []
|
||||
for api in api_result:
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, parameters, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
elif isinstance(api["parameters"], list):
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, parameters, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
elif isinstance(api_result, dict):
|
||||
api = api_result
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, parameters, ind)
|
||||
elif isinstance(api["parameters"], list):
|
||||
call_results = []
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, parameters, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
|
||||
|
||||
def answer_generation(question, API_instruction, call_result, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Answer the question in natural language based on the API response reasonably and effectively.\n"
|
||||
"2. The user cannot directly get API response, "
|
||||
"so you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"3. If the API tool does not provide useful information in the response, "
|
||||
"please answer with your knowledge.\n"
|
||||
"This is the user's question:\n {question}\n"
|
||||
"This is the API response:\n {call_result}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
API_instruction=API_instruction,
|
||||
call_result=call_result, )
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation fails: {e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_depend(question, API_instruction, call_result, previous_log, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Try to organize the response into a natural language answer.\n"
|
||||
"2. We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"3. If the API tool does not provide useful information in the response, "
|
||||
"please answer with your knowledge.\n"
|
||||
"4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers.\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response output by the API tool: \n{call_result}\n"
|
||||
"We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
API_instruction=API_instruction,
|
||||
call_result=call_result,
|
||||
previous_log=previous_log)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation depend fails: {e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_summarize(question, answer_task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. "
|
||||
"You need to organize these answers to each subtask and form a self-consistent final answer to the user's question\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"These are subtasks and their answers: {answer_task}\n"
|
||||
"Final answer:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer_task=answer_task)
|
||||
return result
|
||||
|
||||
|
||||
def answer_check(question, answer, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Please check whether the response can reasonably and accurately answer the question."
|
||||
"If can, please output 'YES'; If not, please output 'NO'\n"
|
||||
"You need to give reasons first and then decide whether the response can reasonably and accurately answer the question. You must only output in a parsible JSON format. Two example outputs look like:\n"
|
||||
"Example 1: {{\"Reason\": \"The reason why you think the response can reasonably and accurately answer the question\", \"Choice\": \"Yes\"}}\n"
|
||||
"Example 2: {{\"Reason\": \"The reason why you think the response cannot reasonably and accurately answer the question\", \"Choice\": \"No\"}}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response: {answer}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer=answer)
|
||||
if 'yes'.lower() in eval(result)["Choice"].lower():
|
||||
return 1
|
||||
else:
|
||||
return -1
|
||||
|
||||
|
||||
def task_execution_mh(data_type, start_index, total_files,
|
||||
retrieval_num, ind, model_name, dataset,
|
||||
Tool_dic, test_data, progress_file):
|
||||
with tqdm(total=total_files, desc="Processing files", initial=start_index) as pbar:
|
||||
for i, data in enumerate(test_data[start_index:], start=start_index):
|
||||
answer_ls = []
|
||||
question = data["question"]
|
||||
print(question)
|
||||
temp = task_decompose(question, Tool_dic, model_name)['Tasks']
|
||||
task_ls = []
|
||||
for t in range(len(temp)):
|
||||
task_ls.append({"task": temp[t], "id": t + 1})
|
||||
task_ls = task_topology(question, task_ls, model_name)
|
||||
task_depend = {'Original Question': question}
|
||||
for task_dic in task_ls:
|
||||
task_depend[task_dic['id']] = {'task': task_dic['task'], 'answer': ''}
|
||||
answer_task = []
|
||||
tool_instruction_ls = []
|
||||
api_result_ls = []
|
||||
call_result_ls = []
|
||||
tool_check_reason_ls = []
|
||||
for task_dic in task_ls:
|
||||
task = task_dic['task']
|
||||
print("Do need tool.")
|
||||
tool_used = []
|
||||
depend_id = [1]
|
||||
for r in range(retrieval_num):
|
||||
if depend_id[0] == -1:
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task, Tool_dic,
|
||||
dataset,
|
||||
tool_used, ind,
|
||||
model_name)
|
||||
if len(str(call_result)) > 5000:
|
||||
call_result = str(call_result)[:5000]
|
||||
answer = answer_generation(task, API_instruction, call_result, model_name)
|
||||
else:
|
||||
previous_log = task_depend
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task, Tool_dic,
|
||||
dataset,
|
||||
tool_used, ind,
|
||||
model_name,
|
||||
previous_log=previous_log)
|
||||
if len(str(call_result)) > 5000:
|
||||
call_result = str(call_result)[:5000]
|
||||
answer = answer_generation_depend(task, API_instruction, call_result, previous_log, model_name)
|
||||
|
||||
check_index = 1
|
||||
if str(call_result).strip() == '-1' or str(call_result).strip() == '':
|
||||
check_index = -1
|
||||
if check_index == 1:
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
tool_instruction_ls.append(tool_instruction)
|
||||
api_result_ls.append(api_result)
|
||||
call_result_ls.append(call_result)
|
||||
break
|
||||
else:
|
||||
answer_ls.append({'task': task, 'answer': answer})
|
||||
try:
|
||||
tool_used.append(str(tool_id["ID"]))
|
||||
except:
|
||||
continue
|
||||
print('****Try Again****')
|
||||
|
||||
task_depend[task_dic['id']]['answer'] = answer
|
||||
final_answer = answer_summarize(question, answer_task, model_name)
|
||||
check_index = answer_check(question, final_answer, model_name)
|
||||
ind = ind + 1
|
||||
with open(f"FuncQA_{data_type}_{model_name}_easytool.jsonl", 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"ID": ind,
|
||||
"question": question,
|
||||
"final_answer": final_answer,
|
||||
"subtask": task_ls,
|
||||
"answer_subtask": answer_task,
|
||||
"answer_wrong": answer_ls,
|
||||
"check_index": check_index,
|
||||
"execute_log": {
|
||||
"api_result_ls": api_result_ls,
|
||||
"call_result_ls": call_result_ls,
|
||||
"tool_check_reason_ls": tool_check_reason_ls,
|
||||
"tool_instruction_ls": tool_instruction_ls,
|
||||
},
|
||||
"check": 0
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
|
||||
print(final_answer)
|
||||
update_progress(progress_file, i + 1)
|
||||
pbar.update(1)
|
||||
|
||||
|
||||
def task_execution_oh(data_type, start_index, total_files,
|
||||
retrieval_num, ind, model_name, dataset,
|
||||
Tool_dic, test_data, progress_file):
|
||||
with tqdm(total=total_files, desc="Processing files", initial=start_index) as pbar:
|
||||
for i, data in enumerate(test_data[start_index:], start=start_index):
|
||||
answer_ls = []
|
||||
question = data["question"]
|
||||
print(question)
|
||||
task_ls = [{"task": question}]
|
||||
answer_task = []
|
||||
tool_instruction_ls = []
|
||||
api_result_ls = []
|
||||
call_result_ls = []
|
||||
tool_check_reason_ls = []
|
||||
for task_dic in task_ls:
|
||||
task = task_dic['task']
|
||||
print("Do need tool.")
|
||||
tool_used = []
|
||||
depend_id = [1]
|
||||
for r in range(retrieval_num):
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task, Tool_dic,
|
||||
dataset,
|
||||
tool_used, ind,
|
||||
model_name)
|
||||
if len(str(call_result)) > 5000:
|
||||
call_result = str(call_result)[:5000]
|
||||
answer = answer_generation(task, API_instruction, call_result, model_name)
|
||||
|
||||
check_index = 1
|
||||
if str(call_result).strip() == '-1' or str(call_result).strip() == '':
|
||||
check_index = -1
|
||||
if check_index == 1:
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
tool_instruction_ls.append(tool_instruction)
|
||||
api_result_ls.append(api_result)
|
||||
call_result_ls.append(call_result)
|
||||
break
|
||||
else:
|
||||
answer_ls.append({'task': task, 'answer': answer})
|
||||
try:
|
||||
tool_used.append(str(tool_id["ID"]))
|
||||
except:
|
||||
continue
|
||||
print('****Try Again****')
|
||||
|
||||
final_answer = answer_summarize(question, answer_task, model_name)
|
||||
check_index = answer_check(question, final_answer, model_name)
|
||||
ind = ind + 1
|
||||
with open(f"FuncQA_{data_type}_{model_name}_easytool.jsonl", 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"ID": ind,
|
||||
"question": question,
|
||||
"final_answer": final_answer,
|
||||
"subtask": task_ls,
|
||||
"answer_subtask": answer_task,
|
||||
"answer_wrong": answer_ls,
|
||||
"check_index": check_index,
|
||||
"execute_log": {
|
||||
"api_result_ls": api_result_ls,
|
||||
"call_result_ls": call_result_ls,
|
||||
"tool_check_reason_ls": tool_check_reason_ls,
|
||||
"tool_instruction_ls": tool_instruction_ls,
|
||||
},
|
||||
"check": 0
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
|
||||
print(final_answer)
|
||||
update_progress(progress_file, i + 1)
|
||||
pbar.update(1)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# — coding: utf-8 –
|
||||
import openai
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
import ast
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import (
|
||||
ChatPromptTemplate,
|
||||
MessagesPlaceholder,
|
||||
SystemMessagePromptTemplate,
|
||||
HumanMessagePromptTemplate
|
||||
)
|
||||
from langchain import LLMChain
|
||||
import numpy as np
|
||||
import requests
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import importlib.util
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
import pickle
|
||||
from util import *
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
openai.api_key = os.environ["OPENAI_API_KEY"]
|
||||
|
||||
|
||||
def get_last_processed_index(progress_file):
|
||||
"""Retrieve the last processed index from the progress file."""
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
last_index = f.read().strip()
|
||||
return int(last_index) if last_index else 0
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def update_progress(progress_file, index):
|
||||
"""Update the last processed index in the progress file."""
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
f.write(str(index))
|
||||
|
||||
|
||||
def task_decompose(question, Tool_dic, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"We have spotify database and the following tools:\n"
|
||||
"{Tool_dic}"
|
||||
"You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step with these tools.\n"
|
||||
"Please note that: \n"
|
||||
"1. you should break down tasks into appropriate subtasks to use the tools mentioned above.\n"
|
||||
"2. You should not only list the subtask, but also list the ID of the tool used to solve this subtask.\n"
|
||||
"3. If you think you do not need to use the tool to solve the subtask, just leave it as {{\"ID\": -1}}\n"
|
||||
"4. You must consider the logical connections, order and constraints among the tools to achieve a correct tool path."
|
||||
"5. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two examples output look like:\n"
|
||||
"'''\n"
|
||||
"Question: Pause the player"
|
||||
"Example 1: [{{\"Task\":\"Get information about the user’s current playback state\", \"ID\":15}}, {{\"Task\":\"Pause playback on the user's account\", \"ID\":19}}]\n"
|
||||
"'''\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question, Tool_dic=Tool_dic)
|
||||
result = ast.literal_eval(result.split('\n\n')[0])
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"task decompose fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def task_execution(
|
||||
Tool_dic, dic_tool, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name):
|
||||
with tqdm(total=total_files, desc="Processing files", initial=start_index) as pbar:
|
||||
for i, data in enumerate(test_data[start_index:], start=start_index):
|
||||
question = data["query"]
|
||||
print(question)
|
||||
task_path = task_decompose(question, Tool_dic, model_name)
|
||||
tool_choice_ls = []
|
||||
for task in task_path:
|
||||
if isinstance(task["ID"], list):
|
||||
for ele in task["ID"]:
|
||||
tool_choice_ls.append(dic_tool[ele]['tool_usage'])
|
||||
elif int(task["ID"]) in dic_tool.keys():
|
||||
tool_choice_ls.append(dic_tool[task["ID"]]['tool_usage'])
|
||||
ind = ind + 1
|
||||
with open(f"restbench_{model_name}_Easytool.jsonl", 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"ID": ind,
|
||||
"question": question,
|
||||
"task_path": task_path,
|
||||
"tool_choice_ls": tool_choice_ls
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
print(tool_choice_ls)
|
||||
update_progress(progress_file, i + 1)
|
||||
pbar.update(1)
|
||||
@@ -0,0 +1,718 @@
|
||||
# — coding: utf-8 –
|
||||
import openai
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import (
|
||||
ChatPromptTemplate,
|
||||
MessagesPlaceholder,
|
||||
SystemMessagePromptTemplate,
|
||||
HumanMessagePromptTemplate
|
||||
)
|
||||
from langchain import LLMChain
|
||||
import numpy as np
|
||||
import requests
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import importlib.util
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
import pickle
|
||||
from util import *
|
||||
from tqdm import tqdm
|
||||
|
||||
openai.api_key = os.environ["OPENAI_API_KEY"]
|
||||
|
||||
|
||||
def get_last_processed_index(progress_file):
|
||||
"""Retrieve the last processed index from the progress file."""
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
last_index = f.read().strip()
|
||||
return int(last_index) if last_index else 0
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def update_progress(progress_file, index):
|
||||
"""Update the last processed index in the progress file."""
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
f.write(str(index))
|
||||
|
||||
|
||||
def choose_tool(question, Tool_dic, tool_used, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is the user's question: {question}\n"
|
||||
"These are the tools you can select to solve the question:\n"
|
||||
"Tool List:\n"
|
||||
"{Too_list}\n\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only chooce one tool the Tool List to solve this question.\n"
|
||||
"2. You must ONLY output the ID of the tool you chose in a parsible JSON format. An example output looks like:\n"
|
||||
"'''\n"
|
||||
"Example: {{\"ID\": XX}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
Tool_list = []
|
||||
for ele in Tool_dic:
|
||||
for key in ele.keys():
|
||||
if str(key) not in tool_used:
|
||||
Tool_list.append(f'''ID: {key}\n{ele[key]}''')
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
Too_list='\n'.join(Tool_list))
|
||||
clean_answer = eval(result.split("\n\n")[-1].strip())
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"choose tool fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return clean_answer
|
||||
|
||||
|
||||
def choose_API(API_instruction, API_list, question, model_name):
|
||||
input_execute_rapidapi_api_note = '''
|
||||
This is an API Tool instruction. Given a question, you should choose APIs from the API list you want to use for this question in this instruction.
|
||||
you must only output in a parsible Python List Format. An example output looks like:
|
||||
```
|
||||
["api1", "api2", ...]
|
||||
```
|
||||
'''.strip()
|
||||
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"{API_instruction}\n"
|
||||
"{input_execute_rapidapi_api_note}\n"
|
||||
"This is the API list: {API_list}\n"
|
||||
"Please note that: \n"
|
||||
"1. The APIs you choose must in the API list.\n"
|
||||
"2. You must ONLY output in the following parsible Python List Format.\n"
|
||||
"```\n"
|
||||
"Output_Example: [\"api1\", \"api2\", ...]\n"
|
||||
"```\n"
|
||||
"Question: {question}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(API_instruction=API_instruction,
|
||||
API_list=API_list,
|
||||
question=question,
|
||||
input_execute_rapidapi_api_note=input_execute_rapidapi_api_note)
|
||||
clean_answer = eval(result.replace("```", "").strip().split("\n\n")[0].strip())
|
||||
if isinstance(clean_answer, str):
|
||||
ls = [clean_answer]
|
||||
elif isinstance(clean_answer, list):
|
||||
ls = clean_answer
|
||||
temp = []
|
||||
for ele in ls:
|
||||
if ele in API_list:
|
||||
temp.append(ele)
|
||||
ls = temp
|
||||
return ls
|
||||
except Exception as e:
|
||||
print(f"Choose API fails: {e}")
|
||||
print(result)
|
||||
if ind > 10:
|
||||
return []
|
||||
ind += 1
|
||||
continue
|
||||
return ls
|
||||
|
||||
|
||||
def choose_parameter(API_instruction, api, api_dic, question, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is an API tool documentation. Given a user's question, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. If you need to use this API multiple times, please set \"Parameters\" to a list.\n"
|
||||
"5. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}, {{\"keyword\": \"Machine Learning\", \"language\": \"English\"}}]}}\n"
|
||||
"'''\n"
|
||||
"This is user's question: {question}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question, )
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(":true", ": True").replace(":false", ": False").replace(
|
||||
": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"Choose Parameter fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def choose_parameter_depend(API_instruction, api, api_dic, question, previous_log, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.\n"
|
||||
"5. If you need to use this API multiple times,, please set \"Parameters\" to a list.\n"
|
||||
"6. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}, {{\"keyword\": \"Machine Learning\", \"language\": \"English\"}}]}}\n"
|
||||
"'''\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the current user's question: {question}\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question,
|
||||
previous_log=previous_log)
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"choose parameter depend fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def answer_generation(question, API_instruction, call_result, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Answer the question in natural language based on the API response reasonably and effectively.\n"
|
||||
"2. The user cannot directly get API response, "
|
||||
"so you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"This is the user's question:\n {question}\n"
|
||||
"This is the API response:\n {call_result}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
call_result=call_result)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation fails: {e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_depend(question, API_instruction, call_result, model_name, previous_log):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Try to organize the response into a natural language answer.\n"
|
||||
"2. We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"3. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers.\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response output by the API tool: \n{call_result}\n"
|
||||
"We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
call_result=call_result,
|
||||
previous_log=previous_log)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation depend fails: {e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_check(question, answer, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Please check whether the response can reasonably and accurately answer the question."
|
||||
"If can, please output 'YES'; If not, please output 'NO'\n"
|
||||
"You need to give reasons first and then decide whether the response can reasonably and accurately answer the question. You must only output in a parsible JSON format. Two example outputs look like:\n"
|
||||
"Example 1: {{\"Reason\": \"The reason why you think the response can reasonably and accurately answer the question\", \"Choice\": \"Yes\"}}\n"
|
||||
"Example 2: {{\"Reason\": \"The reason why you think the response cannot reasonably and accurately answer the question\", \"Choice\": \"No\"}}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response: {answer}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer=answer)
|
||||
if 'yes'.lower() in str(result).lower():
|
||||
return 1
|
||||
else:
|
||||
return -1
|
||||
|
||||
|
||||
def Call_function(A, B, arg, index, id):
|
||||
if A in index:
|
||||
for path in index[A]:
|
||||
app_path = os.path.join(path, A, 'api.py')
|
||||
if os.path.isfile(app_path):
|
||||
spec = importlib.util.spec_from_file_location('api', app_path)
|
||||
app_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(app_module)
|
||||
arg['toolbench_rapidapi_key'] = os.environ['RAPIDAPI_KEY']
|
||||
# Check if B is a function in app
|
||||
if hasattr(app_module, B):
|
||||
function_B = getattr(app_module, B)
|
||||
try:
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.lower()): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.replace("-", "_")): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.replace("\\", "")): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
print(f"Call function fails: {e}")
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": str(e)
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return -1
|
||||
else:
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": f"No function named {B} in {app_path}"
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return (f"No function named {B} in {app_path}")
|
||||
|
||||
|
||||
def retrieval(question, Tool_dic, dataset, tool_used, ind, model_name, index, previous_log=None):
|
||||
tool_id = choose_tool(question, Tool_dic, tool_used, model_name)
|
||||
if tool_id == -1:
|
||||
return tool_id, "", "", "", ""
|
||||
if str(tool_id["ID"]) not in dataset:
|
||||
return tool_id, "", "", "", ""
|
||||
tool_instruction = dataset[str(tool_id["ID"])]
|
||||
API_instruction = tool_instruction["tool_description"]
|
||||
API_tool = tool_instruction["standardized_name"]
|
||||
API_list = []
|
||||
for ele in tool_instruction["tool_guidelines"].keys():
|
||||
API_list.append(ele)
|
||||
|
||||
api_selection = choose_API(API_instruction, API_list, question, model_name)
|
||||
api_result = []
|
||||
if len(api_selection) == 0:
|
||||
call_result = ""
|
||||
print("No Calling")
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
for api in api_selection:
|
||||
if previous_log is None:
|
||||
parameter = choose_parameter(API_instruction, api,
|
||||
tool_instruction["tool_guidelines"][api], question,
|
||||
model_name)
|
||||
else:
|
||||
parameter = choose_parameter_depend(API_instruction, api,
|
||||
tool_instruction["tool_guidelines"][api],
|
||||
question, previous_log,
|
||||
model_name)
|
||||
if parameter == -1:
|
||||
continue
|
||||
api_result.append({"api_name": api, "parameters": parameter})
|
||||
if len(api_result) == 0:
|
||||
call_result = ""
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
if isinstance(api_result, set) or isinstance(api_result, list):
|
||||
call_results = []
|
||||
for api in api_result:
|
||||
api_name = change_name(standardize(api["api_name"]))
|
||||
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
elif isinstance(api["parameters"], list):
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
elif isinstance(api_result, dict):
|
||||
api_name = change_name(standardize(api_result["api_name"]))
|
||||
api = api_result
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
elif isinstance(api["parameters"], list):
|
||||
call_results = []
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
|
||||
|
||||
def task_decompose(question, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step.\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using a single tool.\n"
|
||||
"2. Each simple subtask should be expressed into natural language.\n"
|
||||
"3. Each subtask should contain the necessary information from the original question and should be complete, explicit and self-consistent.\n"
|
||||
"4. You must ONLY output the ID of the tool you chose in a parsible JSON format. An example output looks like:\n"
|
||||
"'''\n"
|
||||
"{{\"Tasks\": [\"Task 1\", \"Task 2\", ...]}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question)
|
||||
result = eval(result.split('\n\n')[0])
|
||||
a = result["Tasks"]
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"task decompose fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def task_topology(question, task_ls, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a complex user's question, I have decompose this question into some simple subtasks"
|
||||
"I think there exists a logical connections and order amontg the tasks. "
|
||||
"Thus you need to help me output this logical connections and order.\n"
|
||||
"You must ONLY output in a parsible JSON format with the following format:\n"
|
||||
"'''\n"
|
||||
"[{{\"task\": task, \"id\", task_id, \"dep\": [dependency_task_id1, dependency_task_id2, ...]}}]\n"
|
||||
"'''\n"
|
||||
"The \"dep\" field denotes the id of the previous task which generates a new resource upon which the current task depends. If there are no dependencies, set \"dep\" to -1.\n\n"
|
||||
"This is user's question: {question}\n"
|
||||
"These are subtasks of this question:\n"
|
||||
"{task_ls}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question, task_ls=task_ls)
|
||||
result = eval(result)
|
||||
for i in range(len(result)):
|
||||
if isinstance(result[i]['dep'], str):
|
||||
temp = []
|
||||
for ele in result[i]['dep'].split(','):
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif isinstance(result[i]['dep'], int):
|
||||
result[i]['dep'] = [result[i]['dep']]
|
||||
elif isinstance(result[i]['dep'], list):
|
||||
temp = []
|
||||
for ele in result[i]['dep']:
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif result[i]['dep'] == -1:
|
||||
result[i]['dep'] = [-1]
|
||||
a = result[i]['dep'][0]
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"task topology fails: {e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_summarize(question, answer_task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. "
|
||||
"You need to organize these answers to each subtask and form a self-consistent final answer to the user's question\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"These are subtasks and their answers: {answer_task}\n"
|
||||
"Final answer:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer_task=answer_task)
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_direct(task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to answer the user's question.\n"
|
||||
"This is the user's question: {task}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(task=task)
|
||||
return result
|
||||
|
||||
|
||||
def tool_check(task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful language model which can use external APIs to solve user's question."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"As a powerful language model, you're equipped to answer user's question with accumulated knowledge. "
|
||||
"However, in some cases, you need to use external APIs to answer accurately."
|
||||
"Thus, you need to check whether the user's question requires you to call an external API to solve it.\n"
|
||||
"Here are some tips to help you check: \n"
|
||||
"1. If the user's question requires real-time information, since your knowledge base isn't updated in real-time, any such question will demand an API call.\n"
|
||||
"2. If you need to obtain information (e.g., ID, name, phone number, geographical location, rank, etc.), you need to call the database APIs if you are not sure.\n"
|
||||
"3. If the question demand a database search or internet research to generate an answer, this is another situation where an API call is necessary.\n"
|
||||
"If need, please output 'YES'; If not, please output 'NO'\n"
|
||||
"You need to give reasons first and then decide whether to keep it or not. You must only output in a parsible JSON format. Two example outputs look like:\n"
|
||||
"Example 1: {{\"Reason\": \"The reason why you think you do not need to call an external API to solve the user's question\", \"Choice\": \"No\"}}\n"
|
||||
"Example 2: {{\"Reason\": \"The reason why you think you need to call an external API to solve the user's question\", \"Choice\": \"Yes\"}}\n"
|
||||
"This is the user's question: {task}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(task=task)
|
||||
result = eval(result)
|
||||
a = result["Reason"]
|
||||
b = result["Choice"]
|
||||
if 'yes' in b.lower():
|
||||
return result, -1
|
||||
else:
|
||||
return result, 1
|
||||
except Exception as e:
|
||||
print(f"tool check fails: {e}")
|
||||
if ind > 10:
|
||||
return "", -1
|
||||
ind += 1
|
||||
continue
|
||||
return result, -1
|
||||
|
||||
|
||||
def task_execution(data_type,
|
||||
base_path, index, dataset, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name):
|
||||
with tqdm(total=total_files, desc="Processing files", initial=start_index) as pbar:
|
||||
for i, data in enumerate(test_data[start_index:], start=start_index):
|
||||
answer_ls = []
|
||||
question = data["query"]
|
||||
print(question)
|
||||
temp = task_decompose(question, model_name)['Tasks']
|
||||
task_ls = []
|
||||
for t in range(len(temp)):
|
||||
task_ls.append({"task": temp[t], "id": t + 1})
|
||||
task_ls = task_topology(question, task_ls, model_name)
|
||||
task_depend = {}
|
||||
for task_dic in task_ls:
|
||||
task_depend[task_dic['id']] = {'task': task_dic['task'], 'answer': ''}
|
||||
answer_task = []
|
||||
api_result_ls = []
|
||||
call_result_ls = []
|
||||
tool_check_reason_ls = []
|
||||
parameter_ls = []
|
||||
for task_dic in task_ls:
|
||||
task = task_dic['task']
|
||||
tool_check_reason, tool_check_result = tool_check(task, model_name)
|
||||
tool_check_reason_ls.append(tool_check_reason)
|
||||
if tool_check_result == 1:
|
||||
print("Do not need tool.")
|
||||
answer = answer_generation_direct(task)
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
else:
|
||||
print("Do need tool.")
|
||||
depend_id = task_dic['dep']
|
||||
tool_used = []
|
||||
for r in range(retrieval_num):
|
||||
Tool_dic = data["Tool_dic"]
|
||||
if depend_id[0] == -1:
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task,
|
||||
Tool_dic,
|
||||
dataset,
|
||||
tool_used,
|
||||
ind,
|
||||
model_name,
|
||||
index)
|
||||
call_result = str(call_result)[:1000]
|
||||
answer = answer_generation(task, API_instruction,
|
||||
call_result, model_name)
|
||||
else:
|
||||
previous_log = []
|
||||
for ids in depend_id:
|
||||
previous_log.append(task_depend[ids])
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task,
|
||||
Tool_dic,
|
||||
dataset,
|
||||
tool_used,
|
||||
ind,
|
||||
model_name,
|
||||
index,
|
||||
previous_log=previous_log)
|
||||
call_result = str(call_result)[:1000]
|
||||
answer = answer_generation_depend(task, API_instruction, call_result, model_name,
|
||||
previous_log=previous_log)
|
||||
|
||||
check_index = answer_check(task, answer, model_name)
|
||||
if check_index == 1:
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
api_result_ls.append(api_result)
|
||||
call_result_ls.append(call_result)
|
||||
break
|
||||
else:
|
||||
answer_ls.append({'task': task, 'answer': answer})
|
||||
try:
|
||||
tool_used.append(str(tool_id["ID"]))
|
||||
except:
|
||||
continue
|
||||
print('****Try Again****')
|
||||
task_depend[task_dic['id']]['answer'] = answer
|
||||
final_answer = answer_summarize(question, answer_task, model_name)
|
||||
check_index = answer_check(question, final_answer, model_name)
|
||||
|
||||
ind = ind + 1
|
||||
with open(f'''{data_type}_{model_name}_Easytool.jsonl''', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"ID": ind,
|
||||
"question": question,
|
||||
"final_answer": final_answer,
|
||||
"subtask": task_ls,
|
||||
"answer_subtask": answer_task,
|
||||
"answer_wrong": answer_ls,
|
||||
"check_index": check_index,
|
||||
"execute_log": {
|
||||
"api_result_ls": api_result_ls,
|
||||
"parameter_ls": parameter_ls,
|
||||
"call_result_ls": call_result_ls,
|
||||
"tool_check_reason_ls": tool_check_reason_ls,
|
||||
}
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
|
||||
print(final_answer)
|
||||
update_progress(progress_file, i + 1)
|
||||
pbar.update(1)
|
||||
@@ -0,0 +1,736 @@
|
||||
# — coding: utf-8 –
|
||||
import openai
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import (
|
||||
ChatPromptTemplate,
|
||||
MessagesPlaceholder,
|
||||
SystemMessagePromptTemplate,
|
||||
HumanMessagePromptTemplate
|
||||
)
|
||||
from langchain import LLMChain
|
||||
import numpy as np
|
||||
import requests
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import importlib.util
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
import pickle
|
||||
from util import *
|
||||
from tqdm import tqdm
|
||||
|
||||
openai.api_key = os.environ["OPENAI_API_KEY"]
|
||||
|
||||
|
||||
def get_last_processed_index(progress_file):
|
||||
"""Retrieve the last processed index from the progress file."""
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
last_index = f.read().strip()
|
||||
return int(last_index) if last_index else 0
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def update_progress(progress_file, index):
|
||||
"""Update the last processed index in the progress file."""
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
f.write(str(index))
|
||||
|
||||
|
||||
def get_embedding(text):
|
||||
a = openai.Embedding.create(
|
||||
engine="text-embedding-ada-002",
|
||||
input=text
|
||||
)
|
||||
return a['data'][0]["embedding"]
|
||||
|
||||
|
||||
def retrieve_reference(embedded_texts, filenames, question, k):
|
||||
input_text = question
|
||||
input_embedding = get_embedding(input_text)
|
||||
similarities = [cosine_similarity([input_embedding], [emb])[0][0] for emb in embedded_texts]
|
||||
top_k_indices = sorted(range(len(similarities)), key=lambda i: similarities[i], reverse=True)[:k]
|
||||
return [filenames[i] for i in top_k_indices]
|
||||
|
||||
|
||||
def choose_tool(question, Tool_dic, tool_used, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is the user's question: {question}\n"
|
||||
"These are the tools you can select to solve the question:\n"
|
||||
"Tool List:\n"
|
||||
"{Too_list}\n\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only chooce one tool the Tool List to solve this question.\n"
|
||||
"2. You must ONLY output the ID of the tool you chose in a parsible JSON format. An example output looks like:\n"
|
||||
"'''\n"
|
||||
"Example: {{\"ID\": XX}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
Tool_list = []
|
||||
for ele in Tool_dic:
|
||||
for key in ele.keys():
|
||||
if str(key) not in tool_used:
|
||||
Tool_list.append(f'''ID: {key}\n{ele[key]}''')
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
Too_list='\n'.join(Tool_list))
|
||||
clean_answer = eval(result.split("\n\n")[-1].strip())
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"choose tool fails:{e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return clean_answer
|
||||
|
||||
|
||||
def choose_API(API_instruction, API_list, question, model_name):
|
||||
input_execute_rapidapi_api_note = '''
|
||||
This is an API Tool instruction. Given a question, you should choose APIs from the API list you want to use for this question in this instruction.
|
||||
you must only output in a parsible Python List Format. An example output looks like:
|
||||
```
|
||||
["api1", "api2", ...]
|
||||
```
|
||||
'''.strip()
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"{API_instruction}\n"
|
||||
"{input_execute_rapidapi_api_note}\n"
|
||||
"This is the API list: {API_list}\n"
|
||||
"Please note that: \n"
|
||||
"1. The APIs you choose must in the API list.\n"
|
||||
"2. You must ONLY output in the following parsible Python List Format.\n"
|
||||
"```\n"
|
||||
"Output_Example: [\"api1\", \"api2\", ...]\n"
|
||||
"```\n"
|
||||
"Question: {question}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(API_instruction=API_instruction,
|
||||
API_list=API_list,
|
||||
question=question,
|
||||
input_execute_rapidapi_api_note=input_execute_rapidapi_api_note)
|
||||
clean_answer = eval(result.replace("```", "").strip().split("\n\n")[0].strip())
|
||||
if isinstance(clean_answer, str):
|
||||
ls = [clean_answer]
|
||||
elif isinstance(clean_answer, list):
|
||||
ls = clean_answer
|
||||
temp = []
|
||||
for ele in ls:
|
||||
if ele in API_list:
|
||||
temp.append(ele)
|
||||
ls = temp
|
||||
return ls
|
||||
except Exception as e:
|
||||
print(f"Choose API fails:{e}")
|
||||
print(result)
|
||||
if ind > 10:
|
||||
return []
|
||||
ind += 1
|
||||
continue
|
||||
return ls
|
||||
|
||||
|
||||
def choose_parameter(API_instruction, api, api_dic, question, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"This is an API tool documentation. Given a user's question, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. If you need to use this API multiple times, please set \"Parameters\" to a list.\n"
|
||||
"5. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}, {{\"keyword\": \"Machine Learning\", \"language\": \"English\"}}]}}\n"
|
||||
"'''\n"
|
||||
"This is user's question: {question}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question, )
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(":true", ": True").replace(":false", ": False").replace(
|
||||
": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"Choose Parameter fails:{e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def choose_parameter_depend(API_instruction, api, api_dic, question, previous_log, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n"
|
||||
"Please note that: \n"
|
||||
"1. The Example in the API tool documentation can help you better understand the use of the API.\n"
|
||||
"2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{\"Parameters\":{{}}}}\n"
|
||||
"3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n"
|
||||
"4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.\n"
|
||||
"5. If you need to use this API multiple times,, please set \"Parameters\" to a list.\n"
|
||||
"6. You must ONLY output in a parsible JSON format. Two examples output looks like:\n"
|
||||
"'''\n"
|
||||
"Example 1: {{\"Parameters\":{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}}}\n"
|
||||
"Example 2: {{\"Parameters\":[{{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}, {{\"keyword\": \"Machine Learning\", \"language\": \"English\"}}]}}\n"
|
||||
"'''\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the current user's question: {question}\n"
|
||||
"This is API tool documentation: {api_dic}\n"
|
||||
"Output:\n"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(api_dic=api_dic,
|
||||
question=question,
|
||||
previous_log=previous_log)
|
||||
clean_answer = eval(
|
||||
result.replace(": true", ": True").replace(": false", ": False").replace("```", "").strip())
|
||||
a = clean_answer["Parameters"]
|
||||
|
||||
return a
|
||||
except Exception as e:
|
||||
print(f"choose parameter depend fails:{e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return a
|
||||
|
||||
|
||||
def answer_generation(question, API_instruction, call_result, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Answer the question in natural language based on the API response reasonably and effectively.\n"
|
||||
"2. The user cannot directly get API response, "
|
||||
"so you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"This is the user's question:\n {question}\n"
|
||||
"This is the API response:\n {call_result}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
call_result=call_result)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation fails:{e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_depend(question, API_instruction, call_result, model_name, previous_log):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You should answer the question based on the response output by the API tool."
|
||||
"Please note that:\n"
|
||||
"1. Try to organize the response into a natural language answer.\n"
|
||||
"2. We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"3. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers.\n"
|
||||
"There are logs of previous questions and answers: \n {previous_log}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response output by the API tool: \n{call_result}\n"
|
||||
"We will not show the API response to the user, "
|
||||
"thus you need to make full use of the response and give the information "
|
||||
"in the response that can satisfy the user's question in as much detail as possible.\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question,
|
||||
call_result=call_result,
|
||||
previous_log=previous_log)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"answer generation depend fails:{e}")
|
||||
if ind > 2:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_check(question, answer, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Please check whether the response can reasonably and accurately answer the question."
|
||||
"If can, please output 'YES'; If not, please output 'NO'\n"
|
||||
"You need to give reasons first and then decide whether the response can reasonably and accurately answer the question. You must only output in a parsible JSON format. Two example outputs look like:\n"
|
||||
"Example 1: {{\"Reason\": \"The reason why you think the response can reasonably and accurately answer the question\", \"Choice\": \"Yes\"}}\n"
|
||||
"Example 2: {{\"Reason\": \"The reason why you think the response cannot reasonably and accurately answer the question\", \"Choice\": \"No\"}}\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"This is the response: {answer}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer=answer)
|
||||
if 'yes'.lower() in str(result).lower():
|
||||
return 1
|
||||
else:
|
||||
return -1
|
||||
|
||||
|
||||
def Call_function(A, B, arg, index, id):
|
||||
if A in index:
|
||||
for path in index[A]:
|
||||
app_path = os.path.join(path, A, 'api.py')
|
||||
if os.path.isfile(app_path):
|
||||
spec = importlib.util.spec_from_file_location('api', app_path)
|
||||
app_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(app_module)
|
||||
arg['toolbench_rapidapi_key'] = os.environ['RAPIDAPI_KEY']
|
||||
# Check if B is a function in app
|
||||
if hasattr(app_module, B):
|
||||
function_B = getattr(app_module, B)
|
||||
try:
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.lower()): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.replace("-", "_")): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
try:
|
||||
arg = {change_name(k.replace("\\", "")): v for k, v in arg.items()}
|
||||
call_result = function_B(**arg)
|
||||
return call_result
|
||||
except Exception as e:
|
||||
print(f"Call function fails:{e}")
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": str(e)
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return -1
|
||||
else:
|
||||
with open('wrong_log.json', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"id": id,
|
||||
"parameters": arg,
|
||||
"wrong": f"No function named {B} in {app_path}"
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
return (f"No function named {B} in {app_path}")
|
||||
|
||||
|
||||
def retrieval(question, Tool_dic, dataset, tool_used, ind, model_name, index, previous_log=None):
|
||||
tool_id = choose_tool(question, Tool_dic, tool_used, model_name)
|
||||
if tool_id == -1:
|
||||
return tool_id, "", "", "", ""
|
||||
if str(tool_id["ID"]) not in dataset:
|
||||
return tool_id, "", "", "", ""
|
||||
tool_instruction = dataset[str(tool_id["ID"])]
|
||||
API_instruction = tool_instruction["tool_description"]
|
||||
API_tool = tool_instruction["standardized_name"]
|
||||
API_list = []
|
||||
for ele in tool_instruction["tool_guidelines"].keys():
|
||||
API_list.append(ele)
|
||||
|
||||
api_selection = choose_API(API_instruction, API_list, question, model_name)
|
||||
api_result = []
|
||||
if len(api_selection) == 0:
|
||||
call_result = ""
|
||||
print("No Calling")
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
for api in api_selection:
|
||||
if previous_log is None:
|
||||
parameter = choose_parameter(API_instruction, api,
|
||||
tool_instruction["tool_guidelines"][api], question,
|
||||
model_name)
|
||||
else:
|
||||
parameter = choose_parameter_depend(API_instruction, api,
|
||||
tool_instruction["tool_guidelines"][api],
|
||||
question, previous_log,
|
||||
model_name)
|
||||
if parameter == -1:
|
||||
continue
|
||||
api_result.append({"api_name": api, "parameters": parameter})
|
||||
if len(api_result) == 0:
|
||||
call_result = ""
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
if isinstance(api_result, set) or isinstance(api_result, list):
|
||||
call_results = []
|
||||
for api in api_result:
|
||||
api_name = change_name(standardize(api["api_name"]))
|
||||
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
elif isinstance(api["parameters"], list):
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
elif isinstance(api_result, dict):
|
||||
api_name = change_name(standardize(api_result["api_name"]))
|
||||
api = api_result
|
||||
if isinstance(api["parameters"], dict):
|
||||
parameters = {}
|
||||
for key in api["parameters"]:
|
||||
value = api["parameters"][key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
elif isinstance(api["parameters"], list):
|
||||
call_results = []
|
||||
for para_ls in api["parameters"]:
|
||||
parameters = {}
|
||||
for key in para_ls:
|
||||
value = para_ls[key]
|
||||
key = change_name(key)
|
||||
parameters[key] = value
|
||||
call_result = Call_function(API_tool, api_name, parameters, index, ind)
|
||||
if call_result == -1:
|
||||
continue
|
||||
call_results.append(str(call_result))
|
||||
call_result = '\n\n'.join(call_results)
|
||||
|
||||
return tool_id, api_result, call_result, tool_instruction, API_instruction
|
||||
|
||||
|
||||
def task_decompose(question, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step.\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"Please note that: \n"
|
||||
"1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using a single tool.\n"
|
||||
"2. Each simple subtask should be expressed into natural language.\n"
|
||||
"3. Each subtask should contain the necessary information from the original question and should be complete, explicit and self-consistent.\n"
|
||||
"4. You must ONLY output the ID of the tool you chose in a parsible JSON format. An example output looks like:\n"
|
||||
"'''\n"
|
||||
"{{\"Tasks\": [\"Task 1\", \"Task 2\", ...]}}\n"
|
||||
"'''\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question)
|
||||
result = eval(result.split('\n\n')[0])
|
||||
a = result["Tasks"]
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"task decompose fails:{e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def task_topology(question, task_ls, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"Given a complex user's question, I have decompose this question into some simple subtasks"
|
||||
"I think there exists a logical connections and order amontg the tasks. "
|
||||
"Thus you need to help me output this logical connections and order.\n"
|
||||
"You must ONLY output in a parsible JSON format with the following format:\n"
|
||||
"'''\n"
|
||||
"[{{\"task\": task, \"id\", task_id, \"dep\": [dependency_task_id1, dependency_task_id2, ...]}}]\n"
|
||||
"'''\n"
|
||||
"The \"dep\" field denotes the id of the previous task which generates a new resource upon which the current task depends. If there are no dependencies, set \"dep\" to -1.\n\n"
|
||||
"This is user's question: {question}\n"
|
||||
"These are subtasks of this question:\n"
|
||||
"{task_ls}\n"
|
||||
"Output: "
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(question=question, task_ls=task_ls)
|
||||
result = eval(result)
|
||||
for i in range(len(result)):
|
||||
if isinstance(result[i]['dep'], str):
|
||||
temp = []
|
||||
for ele in result[i]['dep'].split(','):
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif isinstance(result[i]['dep'], int):
|
||||
result[i]['dep'] = [result[i]['dep']]
|
||||
elif isinstance(result[i]['dep'], list):
|
||||
temp = []
|
||||
for ele in result[i]['dep']:
|
||||
temp.append(int(ele))
|
||||
result[i]['dep'] = temp
|
||||
elif result[i]['dep'] == -1:
|
||||
result[i]['dep'] = [-1]
|
||||
a = result[i]['dep'][0]
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"task topology fails:{e}")
|
||||
if ind > 10:
|
||||
return -1
|
||||
ind += 1
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def answer_summarize(question, answer_task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. "
|
||||
"You need to organize these answers to each subtask and form a self-consistent final answer to the user's question\n"
|
||||
"This is the user's question: {question}\n"
|
||||
"These are subtasks and their answers: {answer_task}\n"
|
||||
"Final answer:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(question=question, answer_task=answer_task)
|
||||
return result
|
||||
|
||||
|
||||
def answer_generation_direct(task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful assistant."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"You need to answer the user's question.\n"
|
||||
"This is the user's question: {task}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
result = chain.run(task=task)
|
||||
return result
|
||||
|
||||
|
||||
def tool_check(task, model_name):
|
||||
chat = ChatOpenAI(model_name=model_name)
|
||||
template = "You are a helpful language model which can use external APIs to solve user's question."
|
||||
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
|
||||
human_message_prompt = HumanMessagePromptTemplate.from_template(
|
||||
"As a powerful language model, you're equipped to answer user's question with accumulated knowledge. "
|
||||
"However, in some cases, you need to use external APIs to answer accurately."
|
||||
"Thus, you need to check whether the user's question requires you to call an external API to solve it.\n"
|
||||
"Here are some tips to help you check: \n"
|
||||
"1. If the user's question requires real-time information, since your knowledge base isn't updated in real-time, any such question will demand an API call.\n"
|
||||
"2. If you need to obtain information (e.g., ID, name, phone number, geographical location, rank, etc.), you need to call the database APIs if you are not sure.\n"
|
||||
"3. If the question demand a database search or internet research to generate an answer, this is another situation where an API call is necessary.\n"
|
||||
"If need, please output 'YES'; If not, please output 'NO'\n"
|
||||
"You need to give reasons first and then decide whether to keep it or not. You must only output in a parsible JSON format. Two example outputs look like:\n"
|
||||
"Example 1: {{\"Reason\": \"The reason why you think you do not need to call an external API to solve the user's question\", \"Choice\": \"No\"}}\n"
|
||||
"Example 2: {{\"Reason\": \"The reason why you think you need to call an external API to solve the user's question\", \"Choice\": \"Yes\"}}\n"
|
||||
"This is the user's question: {task}\n"
|
||||
"Output:"
|
||||
)
|
||||
chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
|
||||
chain = LLMChain(llm=chat, prompt=chat_prompt)
|
||||
ind = 0
|
||||
while True:
|
||||
try:
|
||||
result = chain.run(task=task)
|
||||
result = eval(result)
|
||||
a = result["Reason"]
|
||||
b = result["Choice"]
|
||||
if 'yes' in b.lower():
|
||||
return result, -1
|
||||
else:
|
||||
return result, 1
|
||||
except Exception as e:
|
||||
print(f"tool check fails:{e}")
|
||||
if ind > 10:
|
||||
return "", -1
|
||||
ind += 1
|
||||
continue
|
||||
return result, -1
|
||||
|
||||
|
||||
def task_execution(data_type,
|
||||
base_path, index, dataset, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name):
|
||||
with open("data_toolbench/tool_instruction/API_description_embeddings.pkl", "rb") as file:
|
||||
filenames, embedded_texts = pickle.load(file)
|
||||
with tqdm(total=total_files, desc="Processing files", initial=start_index) as pbar:
|
||||
for i, data in enumerate(test_data[start_index:], start=start_index):
|
||||
answer_ls = []
|
||||
question = data["query"]
|
||||
print(question)
|
||||
temp = task_decompose(question, model_name)['Tasks']
|
||||
task_ls = []
|
||||
for t in range(len(temp)):
|
||||
task_ls.append({"task": temp[t], "id": t + 1})
|
||||
task_ls = task_topology(question, task_ls, model_name)
|
||||
task_depend = {}
|
||||
for task_dic in task_ls:
|
||||
task_depend[task_dic['id']] = {'task': task_dic['task'], 'answer': ''}
|
||||
answer_task = []
|
||||
api_result_ls = []
|
||||
call_result_ls = []
|
||||
tool_check_reason_ls = []
|
||||
parameter_ls = []
|
||||
for task_dic in task_ls:
|
||||
task = task_dic['task']
|
||||
tool_check_reason, tool_check_result = tool_check(task, model_name)
|
||||
tool_check_reason_ls.append(tool_check_reason)
|
||||
if tool_check_result == 1:
|
||||
print("Do not need tool.")
|
||||
answer = answer_generation_direct(task)
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
else:
|
||||
print("Do need tool.")
|
||||
depend_id = task_dic['dep']
|
||||
tool_used = []
|
||||
Tool_dic = [{tool: dataset[str(tool)]["tool_description"]} for tool in
|
||||
retrieve_reference(embedded_texts, filenames, task, k=5)]
|
||||
for r in range(retrieval_num):
|
||||
if depend_id[0] == -1:
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task,
|
||||
Tool_dic,
|
||||
dataset,
|
||||
tool_used,
|
||||
ind,
|
||||
model_name,
|
||||
index)
|
||||
call_result = str(call_result)[:1000]
|
||||
answer = answer_generation(task, API_instruction,
|
||||
call_result, model_name)
|
||||
else:
|
||||
previous_log = []
|
||||
for ids in depend_id:
|
||||
previous_log.append(task_depend[ids])
|
||||
tool_id, api_result, call_result, tool_instruction, API_instruction = retrieval(task,
|
||||
Tool_dic,
|
||||
dataset,
|
||||
tool_used,
|
||||
ind,
|
||||
model_name,
|
||||
index,
|
||||
previous_log=previous_log)
|
||||
call_result = str(call_result)[:1000]
|
||||
answer = answer_generation_depend(task, API_instruction, call_result, model_name,
|
||||
previous_log=previous_log)
|
||||
|
||||
check_index = answer_check(task, answer, model_name)
|
||||
if check_index == 1:
|
||||
answer_task.append({'task': task, 'answer': answer})
|
||||
api_result_ls.append(api_result)
|
||||
call_result_ls.append(call_result)
|
||||
break
|
||||
else:
|
||||
answer_ls.append({'task': task, 'answer': answer})
|
||||
try:
|
||||
tool_used.append(str(tool_id["ID"]))
|
||||
except:
|
||||
continue
|
||||
print('****Try Again****')
|
||||
task_depend[task_dic['id']]['answer'] = answer
|
||||
final_answer = answer_summarize(question, answer_task, model_name)
|
||||
check_index = answer_check(question, final_answer, model_name)
|
||||
|
||||
ind = ind + 1
|
||||
with open(f'''{data_type}_{model_name}_retrieve_Easytool.jsonl''', 'a+', encoding='utf-8') as f:
|
||||
line = json.dumps({
|
||||
"ID": ind,
|
||||
"question": question,
|
||||
"final_answer": final_answer,
|
||||
"subtask": task_ls,
|
||||
"answer_subtask": answer_task,
|
||||
"answer_wrong": answer_ls,
|
||||
"check_index": check_index,
|
||||
"execute_log": {
|
||||
"api_result_ls": api_result_ls,
|
||||
"parameter_ls": parameter_ls,
|
||||
"call_result_ls": call_result_ls,
|
||||
"tool_check_reason_ls": tool_check_reason_ls,
|
||||
}
|
||||
}, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
|
||||
print(final_answer)
|
||||
update_progress(progress_file, i + 1)
|
||||
pbar.update(1)
|
||||
@@ -0,0 +1,109 @@
|
||||
# — coding: utf-8 –
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
def read_jsonline(address):
|
||||
not_mark = []
|
||||
with open(address, 'r', encoding="utf-8") as f:
|
||||
for jsonstr in f.readlines():
|
||||
jsonstr = json.loads(jsonstr)
|
||||
not_mark.append(jsonstr)
|
||||
return not_mark
|
||||
|
||||
|
||||
def save_json(ls, address):
|
||||
json_str = json.dumps(ls, indent=4)
|
||||
with open(address, 'w', encoding='utf-8') as json_file:
|
||||
json.dump(ls, json_file, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
def read_json(address):
|
||||
with open(address, 'r', encoding='utf-8') as json_file:
|
||||
json_data = json.load(json_file)
|
||||
return json_data
|
||||
|
||||
|
||||
def remove_key(item, key_to_remove):
|
||||
if isinstance(item, dict):
|
||||
if key_to_remove in item:
|
||||
del item[key_to_remove]
|
||||
for key, value in list(item.items()): # 使用list包裹，防止字典大小改变时引发错误
|
||||
item[key] = remove_key(value, key_to_remove)
|
||||
elif isinstance(item, list):
|
||||
for index, value in enumerate(item):
|
||||
item[index] = remove_key(value, key_to_remove)
|
||||
return item
|
||||
|
||||
|
||||
def data_clean(dic, key):
|
||||
dic = remove_key(dic, key)
|
||||
return dic
|
||||
|
||||
|
||||
def lowercase_parameter_keys(input_dict):
|
||||
if "parameters" in input_dict and isinstance(input_dict["parameters"], dict):
|
||||
# Convert all keys in the "parameters" dictionary to uppercase
|
||||
input_dict["parameters"] = {change_name(k.lower()): v for k, v in input_dict["parameters"].items()}
|
||||
return input_dict
|
||||
|
||||
|
||||
def build_index(base_path):
|
||||
index = {}
|
||||
for root, dirs, files in os.walk(base_path):
|
||||
for dir_name in dirs:
|
||||
if dir_name not in index:
|
||||
index[dir_name] = []
|
||||
index[dir_name].append(root)
|
||||
return index
|
||||
|
||||
|
||||
def change_name(name):
|
||||
change_list = ["from", "class", "return", "false", "true", "id", "and", "", "ID"]
|
||||
if name in change_list:
|
||||
name = "is_" + name.lower()
|
||||
return name
|
||||
|
||||
|
||||
def standardize(string):
|
||||
res = re.compile("[^\\u4e00-\\u9fa5^a-z^A-Z^0-9^_]")
|
||||
string = res.sub("_", string)
|
||||
string = re.sub(r"(_)\1+", "_", string).lower()
|
||||
while True:
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[0] == "_":
|
||||
string = string[1:]
|
||||
else:
|
||||
break
|
||||
while True:
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[-1] == "_":
|
||||
string = string[:-1]
|
||||
else:
|
||||
break
|
||||
if string[0].isdigit():
|
||||
string = "get_" + string
|
||||
return string
|
||||
|
||||
|
||||
def get_last_processed_index(progress_file):
|
||||
"""Retrieve the last processed index from the progress file."""
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
last_index = f.read().strip()
|
||||
return int(last_index) if last_index else 0
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def update_progress(progress_file, index):
|
||||
"""Update the last processed index in the progress file."""
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
f.write(str(index))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("util.py")
|
||||
@@ -0,0 +1,90 @@
|
||||
# — coding: utf-8 –
|
||||
import openai
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
from easytool import funcQA, restbench, toolbench_retrieve, toolbench
|
||||
from easytool.util import *
|
||||
openai.api_key = os.environ["OPENAI_API_KEY"]
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--model_name', type=str, default='gpt-3.5-turbo')
|
||||
parser.add_argument('--task', type=str, default='funcqa_mh', help='funcqa, toolbench_retrieve, toolbench, restbench')
|
||||
parser.add_argument('--data_type', type=str, default='G3', help='G2 or G3 or funcqa_mh or funcqa_oh')
|
||||
parser.add_argument('--tool_root_dir', type=str, default='.toolenv/tools/')
|
||||
parser.add_argument('--retrieval_num', type=int, default=5)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.task == 'funcqa':
|
||||
dataset = read_json('data_funcqa/tool_instruction/functions_data.json')
|
||||
Tool_dic = read_jsonline('data_funcqa/tool_instruction/tool_dic.jsonl')
|
||||
test_data = read_json(f"data_funcqa/test_data/{args.data_type}.json")
|
||||
progress_file = f"FuncQA_{args.data_type}_{args.model_name}_Easytool.txt"
|
||||
|
||||
|
||||
elif 'toolbench' in args.task:
|
||||
base_path = args.tool_root_dir
|
||||
index = build_index(base_path)
|
||||
dataset = read_json('data_toolbench/tool_instruction/toolbench_tool_instruction.json')
|
||||
if args.data_type == 'G2':
|
||||
test_data = read_json(f'''data_toolbench/test_data/{args.data_type}_category.json''')
|
||||
elif args.data_type == 'G3':
|
||||
test_data = read_json(f'''data_toolbench/test_data/{args.data_type}_instruction.json''')
|
||||
progress_file = f'''{args.data_type}_{args.model_name}_Easytool.txt'''
|
||||
|
||||
|
||||
elif args.task == 'restbench':
|
||||
Tool_dic = read_json('data_restbench/tool_instruction/tmdb_tool.json')
|
||||
dic_tool = {}
|
||||
for data in Tool_dic:
|
||||
dic_tool[data['ID']] = data
|
||||
test_data = read_json('data_restbench/test_data/tmdb.json')
|
||||
progress_file = f"restbench_{args.model_name}_Easytool.txt"
|
||||
|
||||
else:
|
||||
print("Wrong task name")
|
||||
exit()
|
||||
|
||||
start_index = get_last_processed_index(progress_file)
|
||||
total_files = len(test_data)
|
||||
retrieval_num = args.retrieval_num
|
||||
ind = start_index
|
||||
model_name = args.model_name
|
||||
|
||||
print("-------Start Execution-------")
|
||||
if args.data_type == 'funcqa_mh':
|
||||
funcQA.task_execution_mh(args.data_type, start_index, total_files,
|
||||
retrieval_num, ind, model_name, dataset,
|
||||
Tool_dic, test_data, progress_file)
|
||||
elif args.data_type == 'funcqa_oh':
|
||||
funcQA.task_execution_oh(args.data_type, start_index, total_files,
|
||||
retrieval_num, ind, model_name, dataset,
|
||||
Tool_dic, test_data, progress_file)
|
||||
|
||||
|
||||
elif args.task == 'toolbench_retrieve':
|
||||
toolbench_retrieve.task_execution(args.data_type,
|
||||
base_path, index, dataset, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name)
|
||||
|
||||
|
||||
|
||||
elif args.task == 'toolbench':
|
||||
toolbench.task_execution(args.data_type,
|
||||
base_path, index, dataset, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name)
|
||||
|
||||
|
||||
|
||||
elif args.task == 'restbench':
|
||||
restbench.task_execution(
|
||||
Tool_dic, dic_tool, test_data, progress_file,
|
||||
start_index, total_files, retrieval_num, ind, model_name)
|
||||
|
||||
|
||||
else:
|
||||
print("Wrong task name")
|
||||
exit()
|
||||
@@ -0,0 +1,9 @@
|
||||
openai==0.27.8
|
||||
langchain==0.0.260
|
||||
gdown==4.6.0
|
||||
tqdm
|
||||
argparse
|
||||
numpy
|
||||
requests
|
||||
pickle-mixin
|
||||
scikit-learn
|
||||
@@ -0,0 +1,4 @@
|
||||
.git
|
||||
server/models/*
|
||||
!server/models/download.sh
|
||||
!server/models/download.ps1
|
||||
@@ -0,0 +1,103 @@
|
||||
# NVIDIA Jetson embedded device support with GPU accelerated local model execution for https://github.com/microsoft/JARVIS
|
||||
|
||||
# Base image for ffmpeg build env: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/l4t-jetpack/tags
|
||||
FROM nvcr.io/nvidia/l4t-jetpack:r35.2.1 AS build
|
||||
|
||||
RUN apt update && apt install -y --no-install-recommends \
|
||||
build-essential git libass-dev libx264-dev cmake && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Build ffmpeg dependency libraries
|
||||
RUN git clone https://github.com/jocover/jetson-ffmpeg.git && \
|
||||
cd jetson-ffmpeg && \
|
||||
sed -i 's=Libs: -L${libdir} -lnvmpi=Libs: -L${libdir} -lnvmpi -L/usr/lib/aarch64-linux-gnu/tegra -lnvbufsurface=g' nvmpi.pc.in && \
|
||||
mkdir build && \
|
||||
cd build && \
|
||||
cmake .. && \
|
||||
make -j$(nproc) && \
|
||||
sudo make install && \
|
||||
sudo ldconfig && \
|
||||
git clone git://source.ffmpeg.org/ffmpeg.git -b release/4.2 --depth=1 && \
|
||||
cd ffmpeg && \
|
||||
wget https://github.com/jocover/jetson-ffmpeg/raw/master/ffmpeg_nvmpi.patch && \
|
||||
git apply ffmpeg_nvmpi.patch && \
|
||||
./configure --enable-nvmpi --enable-libass --enable-libx264 --enable-gpl && \
|
||||
make -j$(nproc)
|
||||
|
||||
# Base image: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/l4t-pytorch/tags
|
||||
# For running JARVIS application layer
|
||||
from nvcr.io/nvidia/l4t-pytorch:r35.2.1-pth2.0-py3
|
||||
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
|
||||
COPY --from=build /usr/local/lib/libnvmpi.a /usr/local/lib
|
||||
COPY --from=build /usr/local/lib/libnvmpi.so.1.0.0 /usr/local/lib
|
||||
COPY --from=build jetson-ffmpeg/build/ffmpeg/ffmpeg /usr/local/bin
|
||||
COPY --from=build jetson-ffmpeg/build/ffmpeg/ffprobe /usr/local/bin
|
||||
RUN ln /usr/local/lib/libnvmpi.so.1.0.0 /usr/local/lib/libnvmpi.so
|
||||
ENV MAKEFLAGS="-j$(nproc)"
|
||||
|
||||
COPY ./server/requirements.txt .
|
||||
|
||||
# Install model server dependencies
|
||||
RUN apt update && apt remove -y \
|
||||
opencv-dev opencv-libs opencv-licenses opencv-main opencv-python opencv-scripts python3-numpy && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -m pip install importlib-metadata==4.13.0 && \
|
||||
python3 -m pip install -r requirements.txt && \
|
||||
rm -rf requirements.txt
|
||||
|
||||
# Update torch deps via reinstall
|
||||
RUN python3 -m pip install torch==2.0.0a0+ec3941ad.nv23.2 torchaudio==0.13.1+b90d798 torchvision==0.14.1a0+5e8e2f1
|
||||
|
||||
# Downgrade opencv-python to v4.5
|
||||
RUN python3 -m pip install opencv-python==4.5.5.64
|
||||
|
||||
# Install nvidia-opencv-dev
|
||||
RUN apt update && apt install -y --no-install-recommends \
|
||||
nvidia-opencv-dev && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Fix loading of scikit dep at runtime
|
||||
ENV LD_PRELOAD='/usr/local/lib/python3.8/dist-packages/scikit_learn.libs/libgomp-d22c30c5.so.1.0.0'
|
||||
|
||||
# Install nodejs npm from nodesource
|
||||
ENV NVM_DIR /root/.nvm
|
||||
ENV NODE_VERSION v18.16.0
|
||||
RUN wget -q -O - https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash && \
|
||||
. "$NVM_DIR/nvm.sh" && \
|
||||
nvm install $NODE_VERSION && \
|
||||
nvm alias default $NODE_VERSION && \
|
||||
nvm use default
|
||||
ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules
|
||||
ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy source files
|
||||
COPY . .
|
||||
|
||||
# Install web server dependencies
|
||||
RUN apt update && apt install -y --no-install-recommends \
|
||||
xdg-utils && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
cd web && \
|
||||
npm install
|
||||
|
||||
# Download local models
|
||||
# RUN apt update && apt install -y --no-install-recommends \
|
||||
# git-lfs && \
|
||||
# rm -rf /var/lib/apt/lists/* && \
|
||||
# cd server/models && \
|
||||
# bash download.sh
|
||||
|
||||
# Expose the model server ports
|
||||
EXPOSE 8004
|
||||
EXPOSE 8005
|
||||
# Expose the web server port
|
||||
EXPOSE 9999
|
||||
|
||||
WORKDIR /app/server
|
||||
|
||||
# Start the model and web server
|
||||
CMD python3 models_server.py --config configs/config.default.yaml;
|
||||
@@ -0,0 +1,254 @@
|
||||
<p align="center">
|
||||
<img src="./assets/logo.png" width="30%"> <br>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
<h1>HuggingGPT</h1>
|
||||
<div align="center">
|
||||
<a href="https://opensource.org/licenses/Apache-2.0">
|
||||
<img alt="License: Apache 2.0" src="https://img.shields.io/badge/License-Apache%202.0-4E94CE.svg">
|
||||
</a>
|
||||
<a href="https://arxiv.org/abs/2303.17580">
|
||||
<img alt="License: Apache 2.0" src="https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg">
|
||||
</a>
|
||||
<a href="https://huggingface.co/spaces/microsoft/HuggingGPT">
|
||||
<img alt="License: Apache 2.0" src="https://img.shields.io/badge/%F0%9F%A4%97-Open%20in%20Spaces-blue">
|
||||
</a>
|
||||
</div>
|
||||
<h3>Solving AI Tasks with ChatGPT and its Friends in HuggingFace<h3>
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
<img width="100%" alt="image" src="./assets/intro.png">
|
||||
</p>
|
||||
|
||||
|
||||
<!--
|
||||
[](https://arxiv.org/abs/2303.17580)
|
||||
[](https://huggingface.co/spaces/microsoft/HuggingGPT) -->
|
||||
|
||||
|
||||
## Updates
|
||||
+ [2023.07.28] We are now in the process of planning evaluation and project rebuilding. We will release a new version of Jarvis in the near future.
|
||||
+ [2023.07.24] We released a light langchain version of Jarvis. See <a href="https://github.com/langchain-ai/langchain/tree/master/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt">here</a>.
|
||||
+ [2023.04.16] Jarvis now supports the OpenAI service on the Azure platform and the GPT-4 model.
|
||||
+ [2023.04.06] We added the Gradio demo and built the web API for `/tasks` and `/results` in `server` mode.
|
||||
+ The Gradio demo is now hosted on Hugging Face Space. (Build with `inference_mode=hybrid` and `local_deployment=standard`)
|
||||
+ The Web API `/tasks` and `/results` access intermediate results for `Stage #1`: task planning and `Stage #1-3`: model selection with execution results. See <a href="#Server">here</a>.
|
||||
+ [2023.04.03] We added the CLI mode and provided parameters for configuring the scale of local endpoints.
|
||||
+ You can enjoy a lightweight experience with Jarvis without deploying the models locally. See <a href="#Configuration">here</a>.
|
||||
+ Just run `python awesome_chat.py --config configs/config.lite.yaml` to experience it.
|
||||
+ [2023.04.01] We updated a version of code for building.
|
||||
|
||||
## Overview
|
||||
|
||||
Language serves as an interface for LLMs to connect numerous AI models for solving complicated AI tasks!
|
||||
|
||||
See our paper: [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace](http://arxiv.org/abs/2303.17580), Yongliang Shen, Kaitao Song, Xu Tan, Dongsheng Li, Weiming Lu and Yueting Zhuang (the first two authors contribute equally)
|
||||
|
||||
<p align="center"><img src="./assets/overview.jpg"></p>
|
||||
|
||||
We introduce a collaborative system that consists of **an LLM as the controller** and **numerous expert models as collaborative executors** (from HuggingFace Hub). The workflow of our system consists of four stages:
|
||||
+ **Task Planning**: Using ChatGPT to analyze the requests of users to understand their intention, and disassemble them into possible solvable tasks.
|
||||
+ **Model Selection**: To solve the planned tasks, ChatGPT selects expert models hosted on Hugging Face based on their descriptions.
|
||||
+ **Task Execution**: Invokes and executes each selected model, and return the results to ChatGPT.
|
||||
+ **Response Generation**: Finally, using ChatGPT to integrate the prediction of all models, and generate responses.
|
||||
|
||||
## System Requirements
|
||||
|
||||
### Default (Recommended)
|
||||
|
||||
For `configs/config.default.yaml`:
|
||||
|
||||
+ Ubuntu 16.04 LTS
|
||||
+ VRAM >= 24GB
|
||||
+ RAM > 12GB (minimal), 16GB (standard), 80GB (full)
|
||||
+ Disk > 284GB
|
||||
+ 42GB for `damo-vilab/text-to-video-ms-1.7b`
|
||||
+ 126GB for `ControlNet`
|
||||
+ 66GB for `stable-diffusion-v1-5`
|
||||
+ 50GB for others
|
||||
|
||||
### Minimum (Lite)
|
||||
|
||||
For `configs/config.lite.yaml`:
|
||||
|
||||
+ Ubuntu 16.04 LTS
|
||||
+ Nothing else
|
||||
|
||||
The configuration `configs/config.lite.yaml` does not require any expert models to be downloaded and deployed locally. However, it means that Jarvis is restricted to models running stably on HuggingFace Inference Endpoints.
|
||||
|
||||
## Quick Start
|
||||
|
||||
First replace `openai.key` and `huggingface.token` in `server/configs/config.default.yaml` with **your personal OpenAI Key** and **your Hugging Face Token**, or put them in the environment variables `OPENAI_API_KEY` and `HUGGINGFACE_ACCESS_TOKEN` respectively. Then run the following commands:
|
||||
|
||||
<span id="Server"></span>
|
||||
|
||||
### For Server:
|
||||
|
||||
```bash
|
||||
# setup env
|
||||
cd server
|
||||
conda create -n jarvis python=3.8
|
||||
conda activate jarvis
|
||||
conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia
|
||||
pip install -r requirements.txt
|
||||
|
||||
# download models. Make sure that `git-lfs` is installed.
|
||||
cd models
|
||||
bash download.sh # required when `inference_mode` is `local` or `hybrid`.
|
||||
|
||||
# run server
|
||||
cd ..
|
||||
python models_server.py --config configs/config.default.yaml # required when `inference_mode` is `local` or `hybrid`
|
||||
python awesome_chat.py --config configs/config.default.yaml --mode server # for text-davinci-003
|
||||
```
|
||||
|
||||
Now you can access Jarvis' services by the Web API.
|
||||
|
||||
+ `/hugginggpt` --method `POST`, access the full service.
|
||||
+ `/tasks` --method `POST`, access intermediate results for Stage #1.
|
||||
+ `/results` --method `POST`, access intermediate results for Stage #1-3.
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
# request
|
||||
curl --location 'http://localhost:8004/tasks' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "based on pose of /examples/d.jpg and content of /examples/e.jpg, please show me a new image"
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
# response
|
||||
[{"args":{"image":"/examples/d.jpg"},"dep":[-1],"id":0,"task":"openpose-control"},{"args":{"image":"/examples/e.jpg"},"dep":[-1],"id":1,"task":"image-to-text"},{"args":{"image":"<GENERATED>-0","text":"<GENERATED>-1"},"dep":[1,0],"id":2,"task":"openpose-text-to-image"}]
|
||||
```
|
||||
|
||||
|
||||
### For Web:
|
||||
|
||||
We provide a user-friendly web page. After starting `awesome_chat.py` in a server mode, you can run the commands to communicate with Jarvis in your browser:
|
||||
|
||||
- you need to install `nodejs` and `npm` first.
|
||||
- [ IMPORTANT ] if you are running the web client on another machine, you need set `http://{LAN_IP_of_the_server}:{port}/` to `HUGGINGGPT_BASE_URL` of `web/src/config/index.ts`.
|
||||
- if you want to use the video generation feature, you need to compile `ffmpeg` manually with H.264.
|
||||
- you can switch to ChatGPT by `double click` on the setting icon!
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
```bash
|
||||
# Optional: Install ffmpeg
|
||||
# This command need be executed without errors.
|
||||
LD_LIBRARY_PATH=/usr/local/lib /usr/local/bin/ffmpeg -i input.mp4 -vcodec libx264 output.mp4
|
||||
```
|
||||
|
||||
<span id="Gradio"></span>
|
||||
|
||||
### For Gradio
|
||||
|
||||
The Gradio demo is now hosted on Hugging Face Space. You can also run the following commands to start the demo locally:
|
||||
|
||||
```bash
|
||||
python models_server.py --config configs/config.gradio.yaml
|
||||
python run_gradio_demo.py --config configs/config.gradio.yaml
|
||||
|
||||
# or run the HF Space as a Docker image (Build with `inference_mode=hibrid` and `local_deployment=standard`)
|
||||
docker run -it -p 7860:7860 --platform=linux/amd64 registry.hf.space/microsoft-hugginggpt:latest python app.py
|
||||
```
|
||||
|
||||
### For CLI:
|
||||
|
||||
You can also run Jarvis more easily in CLI mode:
|
||||
|
||||
```bash
|
||||
cd server
|
||||
python awesome_chat.py --config configs/config.default.yaml --mode cli
|
||||
```
|
||||
|
||||
Examples of CLI mode:
|
||||
|
||||
```
|
||||
Welcome to Jarvis! A collaborative system that consists of an LLM as the controller and numerous expert models as collaborative executors. Jarvis can plan tasks, schedule Hugging Face models, generate friendly responses based on your requests, and help you with many things. Please enter your request (`exit` to exit).
|
||||
[ User ]: Given a collection of image A: /examples/a.jpg, B: /examples/b.jpg, C: /examples/c.jpg, please tell me how many zebras in these picture?
|
||||
[ Jarvis ]: Based on your request, I have used several models to process the images you provided. For the image A: /examples/a.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "a cat sitting on a window sill looking out". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/a3dc.jpg', which contains two objects with labels 'potted plant' and 'cat'. For the image B: /examples/b.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "two zebras standing in a field of dry grass". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/f5df.jpg', which contains three objects with labels 'zebra'. For the image C: /examples/c.jpg, I used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text "a zebra standing in a field of tall grass". Then I used the object-detection model facebook/detr-resnet-50 to generate the image with predicted box '/images/69c7.jpg', which contains one object with label 'zebra'. Finally, I used the canny-control model canny-control to generate the image '/images/7e18.png' based on the image /examples/f.jpg. Therefore, there are four zebras in these pictures. Is there anything else I can help you with?
|
||||
[ User ]: Please answer all the named entities in the sentence: Iron Man is a superhero appearing in American comic books published by Marvel Comics. The character was co-created by writer and editor Stan Lee, developed by scripter Larry Lieber, and designed by artists Don Heck and Jack Kirby.
|
||||
[ Jarvis ]: Sure, I understand your request. Based on the inference results of the models, I will answer your request step by step. Firstly, I used the token-classification model dslim/bert-base-NER to recognize the named entities in the sentence. The inference result is that the named entities are Iron Man, Stan Lee, Larry Lieber, Don Heck and Jack Kirby. Then, I used the question-answering model deepset/roberta-base-squad2 to answer your request. The inference result is that there is no output since the context cannot be empty. Therefore, I cannot make it. I hope this answer your request. Is there anything else I can help you with?
|
||||
[ User ]: Please generate a canny image based on /examples/f.jpg
|
||||
[ Jarvis ]: Sure. I have generated a canny image based on /examples/f.jpg. To do this, I first used the image-to-text model nlpconnect/vit-gpt2-image-captioning to generate the text description of the image, which is "a herd of giraffes and zebras grazing in a field". Then I used the canny-control model to generate the canny image of the original image, which is stored in the path "/images/1ea2.png". Finally, I used the canny-text-to-image model lllyasviel/sd-controlnet-canny to generate the canny image based on the text description and the canny image, which is stored in the path "/images/ba66.png". Do you need anything else?
|
||||
```
|
||||
|
||||
|
||||
## Configuration
|
||||
|
||||
The server-side configuration file is `server/configs/config.default.yaml`, and some parameters are presented as follows:
|
||||
|
||||
+ `model`: LLM, currently supports `text-davinci-003`. We are working on integrating more open-source LLMs.
|
||||
+ `inference_mode`: mode of inference endpoints
|
||||
+ `local`: only use the local inference endpoints
|
||||
+ `huggingface`: only use the Hugging Face Inference Endpoints **(free of local inference endpoints)**
|
||||
+ `hybrid`: both of `local` and `huggingface`
|
||||
+ `local_deployment`: scale of locally deployed models, works under `local` or `hybrid` inference mode:
|
||||
+ `minimal` (RAM>12GB, ControlNet only)
|
||||
+ `standard` (RAM>16GB, ControlNet + Standard Pipelines)
|
||||
+ `full` (RAM>42GB, All registered models)
|
||||
|
||||
On a personal laptop, we recommend the configuration of `inference_mode: hybrid `and `local_deployment: minimal`. But the available models under this setting may be limited due to the instability of remote Hugging Face Inference Endpoints.
|
||||
|
||||
## NVIDIA Jetson Embedded Device Support
|
||||
A [Dockerfile](./Dockerfile.jetson) is included that provides experimental support for [NVIDIA Jetson embedded devices](https://developer.nvidia.com/embedded-computing). This image provides accelerated ffmpeg, pytorch, torchaudio, and torchvision dependencies. To build the docker image, [ensure that the default docker runtime is set to 'nvidia'](https://github.com/NVIDIA/nvidia-docker/wiki/Advanced-topics#default-runtime). A pre-built image is provided at https://hub.docker.com/r/toolboc/nv-jarvis.
|
||||
|
||||
```bash
|
||||
#Build the docker image
|
||||
docker build --pull --rm -f "Dockerfile.jetson" -t toolboc/nv-jarvis:r35.2.1
|
||||
```
|
||||
|
||||
Due to to memory requirements, JARVIS is required to run on Jetson AGX Orin family devices (64G on-board RAM device preferred) with config options set to:
|
||||
* `inference_mode: local`
|
||||
* `local_deployment: standard`
|
||||
|
||||
Models and configs are recommended to be provided through a volume mount from the host to the container as shown in the `docker run` step below. It is possible to uncomment the `# Download local models` section of the [Dockerfile](./Dockerfile.jetson) to build a container with models included.
|
||||
|
||||
### Start the model server, awesomechat, and web app on Jetson Orin AGX
|
||||
|
||||
```bash
|
||||
# run the container which will automatically start the model server
|
||||
docker run --name jarvis --net=host --gpus all -v ~/jarvis/configs:/app/server/configs -v ~/src/JARVIS/server/models:/app/server/models toolboc/nv-jarvis:r35.2.1
|
||||
|
||||
# (wait for model server to complete initialization)
|
||||
|
||||
# start awesome_chat.py
|
||||
docker exec jarvis python3 awesome_chat.py --config configs/config.default.yaml --mode server
|
||||
|
||||
#start the web application (application will be acessible at http://localhost:9999)
|
||||
docker exec jarvis npm run dev --prefix=/app/web
|
||||
```
|
||||
|
||||
## Screenshots
|
||||
|
||||
<p align="center"><img src="./assets/screenshot_q.jpg"><img src="./assets/screenshot_a.jpg"></p>
|
||||
|
||||
## Citation
|
||||
If you find this work useful in your method, you can cite the paper as below:
|
||||
|
||||
@inproceedings{shen2023hugginggpt,
|
||||
author = {Shen, Yongliang and Song, Kaitao and Tan, Xu and Li, Dongsheng and Lu, Weiming and Zhuang, Yueting},
|
||||
booktitle = {Advances in Neural Information Processing Systems},
|
||||
title = {HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in HuggingFace},
|
||||
year = {2023}
|
||||
}
|
||||
|
||||
## Acknowledgement
|
||||
|
||||
- [ChatGPT](https://platform.openai.com/)
|
||||
- [Hugging Face](https://huggingface.co/)
|
||||
- [ControlNet](https://github.com/lllyasviel/ControlNet)
|
||||
- [ChatGPT-vue](https://github.com/lianginx/chatgpt-vue)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 592 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 387 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 86 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
azure:
|
||||
api_key: REPLACE_WITH_YOUR_AZURE_API_KEY_HERE
|
||||
base_url: REPLACE_WITH_YOUR_ENDPOINT_HERE
|
||||
deployment_name: REPLACE_WITH_YOUR_DEPLOYMENT_NAME_HERE
|
||||
api_version: "2022-12-01"
|
||||
huggingface:
|
||||
token: REPLACE_WITH_YOUR_HUGGINGFACE_TOKEN_HERE # required: huggingface token @ https://huggingface.co/settings/tokens
|
||||
dev: false
|
||||
debug: false
|
||||
log_file: logs/debug.log
|
||||
model: text-davinci-003 # currently only support text-davinci-003, gpt-4, we will support more open-source LLMs in the future
|
||||
use_completion: true
|
||||
inference_mode: huggingface # local, huggingface or hybrid, prefer hybrid
|
||||
local_deployment: full # minimal, standard or full, prefer full
|
||||
device: cuda:0 # cuda:id or cpu
|
||||
num_candidate_models: 5
|
||||
max_description_length: 100
|
||||
proxy: # optional: your proxy server "http://ip:port"
|
||||
http_listen:
|
||||
host: 0.0.0.0 # if you use web as the client, please set `http://{LAN_IP_of_the_server}:{port}/` to `BASE_URL` of `web/src/config/index.ts`.
|
||||
port: 8004
|
||||
local_inference_endpoint:
|
||||
host: localhost
|
||||
port: 8005
|
||||
logit_bias:
|
||||
parse_task: 0.1
|
||||
choose_model: 5
|
||||
tprompt:
|
||||
parse_task: >-
|
||||
#1 Task Planning Stage: The AI assistant can parse user input to several tasks: [{"task": task, "id": task_id, "dep": dependency_task_id, "args": {"text": text or <GENERATED>-dep_id, "image": image_url or <GENERATED>-dep_id, "audio": audio_url or <GENERATED>-dep_id}}]. The special tag "<GENERATED>-dep_id" refer to the one generated text/image/audio in the dependency task (Please consider whether the dependency task generates resources of this type.) and "dep_id" must be in "dep" list. The "dep" field denotes the ids of the previous prerequisite tasks which generate a new resource that the current task relies on. The "args" field must in ["text", "image", "audio"], nothing else. The task MUST be selected from the following options: "token-classification", "text2text-generation", "summarization", "translation", "question-answering", "conversational", "text-generation", "sentence-similarity", "tabular-classification", "object-detection", "image-classification", "image-to-image", "image-to-text", "text-to-image", "text-to-video", "visual-question-answering", "document-question-answering", "image-segmentation", "depth-estimation", "text-to-speech", "automatic-speech-recognition", "audio-to-audio", "audio-classification", "canny-control", "hed-control", "mlsd-control", "normal-control", "openpose-control", "canny-text-to-image", "depth-text-to-image", "hed-text-to-image", "mlsd-text-to-image", "normal-text-to-image", "openpose-text-to-image", "seg-text-to-image". There may be multiple tasks of the same type. Think step by step about all the tasks needed to resolve the user's request. Parse out as few tasks as possible while ensuring that the user request can be resolved. Pay attention to the dependencies and order among tasks. If the user input can't be parsed, you need to reply empty JSON [].
|
||||
choose_model: >-
|
||||
#2 Model Selection Stage: Given the user request and the parsed tasks, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The assistant should focus more on the description of the model and find the model that has the most potential to solve requests and tasks. Also, prefer models with local inference endpoints for speed and stability.
|
||||
response_results: >-
|
||||
#4 Response Generation Stage: With the task execution logs, the AI assistant needs to describe the process and inference results.
|
||||
demos_or_presteps:
|
||||
parse_task: demos/demo_parse_task.json
|
||||
choose_model: demos/demo_choose_model.json
|
||||
response_results: demos/demo_response_results.json
|
||||
prompt:
|
||||
parse_task: The chat log [ {{context}} ] may contain the resources I mentioned. Now I input { {{input}} }. Pay attention to the input and output types of tasks and the dependencies between tasks.
|
||||
choose_model: >-
|
||||
Please choose the most suitable model from {{metas}} for the task {{task}}. The output must be in a strict JSON format: {"id": "id", "reason": "your detail reasons for the choice"}.
|
||||
response_results: >-
|
||||
Yes. Please first think carefully and directly answer my request based on the inference results. Some of the inferences may not always turn out to be correct and require you to make careful consideration in making decisions. Then please detail your workflow including the used models and inference results for my request in your friendly tone. Please filter out information that is not relevant to my request. Tell me the complete path or urls of files in inference results. If there is nothing in the results, please tell me you can't make it. }
|
||||
@@ -0,0 +1,46 @@
|
||||
openai:
|
||||
api_key: REPLACE_WITH_YOUR_OPENAI_API_KEY_HERE
|
||||
# azure:
|
||||
# api_key: REPLACE_WITH_YOUR_AZURE_API_KEY_HERE
|
||||
# base_url: REPLACE_WITH_YOUR_ENDPOINT_HERE
|
||||
# deployment_name: REPLACE_WITH_YOUR_DEPLOYMENT_NAME_HERE
|
||||
# api_version: "2022-12-01"
|
||||
huggingface:
|
||||
token: REPLACE_WITH_YOUR_HUGGINGFACE_TOKEN_HERE # required: huggingface token @ https://huggingface.co/settings/tokens
|
||||
dev: false
|
||||
debug: false
|
||||
log_file: logs/debug.log
|
||||
model: text-davinci-003 # currently only support text-davinci-003, gpt-4, we will support more open-source LLMs in the future
|
||||
use_completion: true
|
||||
inference_mode: hybrid # local, huggingface or hybrid, prefer hybrid
|
||||
local_deployment: full # minimal, standard or full, prefer full
|
||||
device: cuda:0 # cuda:id or cpu
|
||||
num_candidate_models: 5
|
||||
max_description_length: 100
|
||||
proxy: # optional: your proxy server "http://ip:port"
|
||||
http_listen:
|
||||
host: 0.0.0.0 # if you use web as the client, please set `http://{LAN_IP_of_the_server}:{port}/` to `BASE_URL` of `web/src/config/index.ts`.
|
||||
port: 8004
|
||||
local_inference_endpoint:
|
||||
host: localhost
|
||||
port: 8005
|
||||
logit_bias:
|
||||
parse_task: 0.1
|
||||
choose_model: 5
|
||||
tprompt:
|
||||
parse_task: >-
|
||||
#1 Task Planning Stage: The AI assistant can parse user input to several tasks: [{"task": task, "id": task_id, "dep": dependency_task_id, "args": {"text": text or <GENERATED>-dep_id, "image": image_url or <GENERATED>-dep_id, "audio": audio_url or <GENERATED>-dep_id}}]. The special tag "<GENERATED>-dep_id" refer to the one generated text/image/audio in the dependency task (Please consider whether the dependency task generates resources of this type.) and "dep_id" must be in "dep" list. The "dep" field denotes the ids of the previous prerequisite tasks which generate a new resource that the current task relies on. The "args" field must in ["text", "image", "audio"], nothing else. The task MUST be selected from the following options: "token-classification", "text2text-generation", "summarization", "translation", "question-answering", "conversational", "text-generation", "sentence-similarity", "tabular-classification", "object-detection", "image-classification", "image-to-image", "image-to-text", "text-to-image", "text-to-video", "visual-question-answering", "document-question-answering", "image-segmentation", "depth-estimation", "text-to-speech", "automatic-speech-recognition", "audio-to-audio", "audio-classification", "canny-control", "hed-control", "mlsd-control", "normal-control", "openpose-control", "canny-text-to-image", "depth-text-to-image", "hed-text-to-image", "mlsd-text-to-image", "normal-text-to-image", "openpose-text-to-image", "seg-text-to-image". There may be multiple tasks of the same type. Think step by step about all the tasks needed to resolve the user's request. Parse out as few tasks as possible while ensuring that the user request can be resolved. Pay attention to the dependencies and order among tasks. If the user input can't be parsed, you need to reply empty JSON [].
|
||||
choose_model: >-
|
||||
#2 Model Selection Stage: Given the user request and the parsed tasks, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The assistant should focus more on the description of the model and find the model that has the most potential to solve requests and tasks. Also, prefer models with local inference endpoints for speed and stability.
|
||||
response_results: >-
|
||||
#4 Response Generation Stage: With the task execution logs, the AI assistant needs to describe the process and inference results.
|
||||
demos_or_presteps:
|
||||
parse_task: demos/demo_parse_task.json
|
||||
choose_model: demos/demo_choose_model.json
|
||||
response_results: demos/demo_response_results.json
|
||||
prompt:
|
||||
parse_task: The chat log [ {{context}} ] may contain the resources I mentioned. Now I input { {{input}} }. Pay attention to the input and output types of tasks and the dependencies between tasks.
|
||||
choose_model: >-
|
||||
Please choose the most suitable model from {{metas}} for the task {{task}}. The output must be in a strict JSON format: {"id": "id", "reason": "your detail reasons for the choice"}.
|
||||
response_results: >-
|
||||
Yes. Please first think carefully and directly answer my request based on the inference results. Some of the inferences may not always turn out to be correct and require you to make careful consideration in making decisions. Then please detail your workflow including the used models and inference results for my request in your friendly tone. Please filter out information that is not relevant to my request. Tell me the complete path or urls of files in inference results. If there is nothing in the results, please tell me you can't make it. }
|
||||
@@ -0,0 +1,36 @@
|
||||
huggingface:
|
||||
token: REPLACE_WITH_YOUR_HUGGINGFACE_TOKEN_HERE # required: huggingface token @ https://huggingface.co/settings/tokens
|
||||
dev: false
|
||||
debug: true
|
||||
log_file: logs/debug.log
|
||||
model: text-davinci-003 # currently only support text-davinci-003, we will support more open-source LLMs in the future
|
||||
use_completion: true
|
||||
inference_mode: huggingface # local, huggingface or hybrid, prefer hybrid
|
||||
local_deployment: full # minimal, standard or full, prefer full
|
||||
device: cuda:0 # cuda:id or cpu
|
||||
num_candidate_models: 5
|
||||
max_description_length: 100
|
||||
proxy: # optional: your proxy server "http://ip:port"
|
||||
local_inference_endpoint:
|
||||
host: localhost
|
||||
port: 8005
|
||||
logit_bias:
|
||||
parse_task: 0.1
|
||||
choose_model: 5
|
||||
tprompt:
|
||||
parse_task: >-
|
||||
#1 Task Planning Stage: The AI assistant can parse user input to several tasks: [{"task": task, "id": task_id, "dep": dependency_task_id, "args": {"text": text or <GENERATED>-dep_id, "image": image_url or <GENERATED>-dep_id, "audio": audio_url or <GENERATED>-dep_id}}]. The special tag "<GENERATED>-dep_id" refer to the one generated text/image/audio in the dependency task (Please consider whether the dependency task generates resources of this type.) and "dep_id" must be in "dep" list. The "dep" field denotes the ids of the previous prerequisite tasks which generate a new resource that the current task relies on. The "args" field must in ["text", "image", "audio"], nothing else. The task MUST be selected from the following options: "token-classification", "text2text-generation", "summarization", "translation", "question-answering", "conversational", "text-generation", "sentence-similarity", "tabular-classification", "object-detection", "image-classification", "image-to-image", "image-to-text", "text-to-image", "text-to-video", "visual-question-answering", "document-question-answering", "image-segmentation", "depth-estimation", "text-to-speech", "automatic-speech-recognition", "audio-to-audio", "audio-classification", "canny-control", "hed-control", "mlsd-control", "normal-control", "openpose-control", "canny-text-to-image", "depth-text-to-image", "hed-text-to-image", "mlsd-text-to-image", "normal-text-to-image", "openpose-text-to-image", "seg-text-to-image". There may be multiple tasks of the same type. Think step by step about all the tasks needed to resolve the user's request. Parse out as few tasks as possible while ensuring that the user request can be resolved. Pay attention to the dependencies and order among tasks. If the user input can't be parsed, you need to reply empty JSON [].
|
||||
choose_model: >-
|
||||
#2 Model Selection Stage: Given the user request and the parsed tasks, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The assistant should focus more on the description of the model and find the model that has the most potential to solve requests and tasks. Also, prefer models with local inference endpoints for speed and stability.
|
||||
response_results: >-
|
||||
#4 Response Generation Stage: With the task execution logs, the AI assistant needs to describe the process and inference results.
|
||||
demos_or_presteps:
|
||||
parse_task: demos/demo_parse_task.json
|
||||
choose_model: demos/demo_choose_model.json
|
||||
response_results: demos/demo_response_results.json
|
||||
prompt:
|
||||
parse_task: The chat log [ {{context}} ] may contain the resources I mentioned. Now I input { {{input}} }. Pay attention to the input and output types of tasks and the dependencies between tasks.
|
||||
choose_model: >-
|
||||
Please choose the most suitable model from {{metas}} for the task {{task}}. The output must be in a strict JSON format: {"id": "id", "reason": "your detail reasons for the choice"}.
|
||||
response_results: >-
|
||||
Yes. Please first think carefully and directly answer my request based on the inference results. Some of the inferences may not always turn out to be correct and require you to make careful consideration in making decisions. Then please detail your workflow including the used models and inference results for my request in your friendly tone. Please filter out information that is not relevant to my request. Tell me the complete path or urls of files in inference results. If there is nothing in the results, please tell me you can't make it. }
|
||||
@@ -0,0 +1,37 @@
|
||||
openai:
|
||||
api_key: REPLACE_WITH_YOUR_OPENAI_API_KEY_HERE
|
||||
huggingface:
|
||||
token: REPLACE_WITH_YOUR_HUGGINGFACE_TOKEN_HERE # required: huggingface token @ https://huggingface.co/settings/tokens
|
||||
dev: false
|
||||
debug: false
|
||||
log_file: logs/debug.log
|
||||
model: text-davinci-003 # currently only support text-davinci-003, gpt-4, we will support more open-source LLMs in the future
|
||||
use_completion: true
|
||||
inference_mode: huggingface # local, huggingface or hybrid, prefer hybrid
|
||||
local_deployment: minimal # minimal, standard or full, prefer full
|
||||
num_candidate_models: 5
|
||||
max_description_length: 100
|
||||
proxy: # optional: your proxy server "http://ip:port"
|
||||
http_listen:
|
||||
host: 0.0.0.0 # if you use web as the client, please set `http://{LAN_IP_of_the_server}:{port}/` to `BASE_URL` of `web/src/config/index.ts`.
|
||||
port: 8004
|
||||
logit_bias:
|
||||
parse_task: 0.1
|
||||
choose_model: 5
|
||||
tprompt:
|
||||
parse_task: >-
|
||||
#1 Task Planning Stage: The AI assistant can parse user input to several tasks: [{"task": task, "id": task_id, "dep": dependency_task_id, "args": {"text": text or <GENERATED>-dep_id, "image": image_url or <GENERATED>-dep_id, "audio": audio_url or <GENERATED>-dep_id}}]. The special tag "<GENERATED>-dep_id" refer to the one generated text/image/audio in the dependency task (Please consider whether the dependency task generates resources of this type.) and "dep_id" must be in "dep" list. The "dep" field denotes the ids of the previous prerequisite tasks which generate a new resource that the current task relies on. The "args" field must in ["text", "image", "audio"], nothing else. The task MUST be selected from the following options: "token-classification", "text2text-generation", "summarization", "translation", "question-answering", "conversational", "text-generation", "sentence-similarity", "tabular-classification", "object-detection", "image-classification", "image-to-image", "image-to-text", "text-to-image", "text-to-video", "visual-question-answering", "document-question-answering", "image-segmentation", "depth-estimation", "text-to-speech", "automatic-speech-recognition", "audio-to-audio", "audio-classification", "canny-control", "hed-control", "mlsd-control", "normal-control", "openpose-control", "canny-text-to-image", "depth-text-to-image", "hed-text-to-image", "mlsd-text-to-image", "normal-text-to-image", "openpose-text-to-image", "seg-text-to-image". There may be multiple tasks of the same type. Think step by step about all the tasks needed to resolve the user's request. Parse out as few tasks as possible while ensuring that the user request can be resolved. Pay attention to the dependencies and order among tasks. If the user input can't be parsed, you need to reply empty JSON [].
|
||||
choose_model: >-
|
||||
#2 Model Selection Stage: Given the user request and the parsed tasks, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The assistant should focus more on the description of the model and find the model that has the most potential to solve requests and tasks. Also, prefer models with local inference endpoints for speed and stability.
|
||||
response_results: >-
|
||||
#4 Response Generation Stage: With the task execution logs, the AI assistant needs to describe the process and inference results.
|
||||
demos_or_presteps:
|
||||
parse_task: demos/demo_parse_task.json
|
||||
choose_model: demos/demo_choose_model.json
|
||||
response_results: demos/demo_response_results.json
|
||||
prompt:
|
||||
parse_task: The chat log [ {{context}} ] may contain the resources I mentioned. Now I input { {{input}} }. Pay attention to the input and output types of tasks and the dependencies between tasks.
|
||||
choose_model: >-
|
||||
Please choose the most suitable model from {{metas}} for the task {{task}}. The output must be in a strict JSON format: {"id": "id", "reason": "your detail reasons for the choice"}.
|
||||
response_results: >-
|
||||
Yes. Please first think carefully and directly answer my request based on the inference results. Some of the inferences may not always turn out to be correct and require you to make careful consideration in making decisions. Then please detail your workflow including the used models and inference results for my request in your friendly tone. Please filter out information that is not relevant to my request. Tell me the complete path or urls of files in inference results. If there is nothing in the results, please tell me you can't make it. }
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "{{input}}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "{{task}}"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,56 @@
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Give you some pictures e1.jpg, e2.png, e3.jpg, help me count the number of sheep?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "[{\"task\": \"image-to-text\", \"id\": 0, \"dep\": [-1], \"args\": {\"image\": \"e1.jpg\" }}, {\"task\": \"object-detection\", \"id\": 1, \"dep\": [-1], \"args\": {\"image\": \"e1.jpg\" }}, {\"task\": \"visual-question-answering\", \"id\": 2, \"dep\": [1], \"args\": {\"image\": \"<GENERATED>-1\", \"text\": \"How many sheep in the picture\"}} }}, {\"task\": \"image-to-text\", \"id\": 3, \"dep\": [-1], \"args\": {\"image\": \"e2.png\" }}, {\"task\": \"object-detection\", \"id\": 4, \"dep\": [-1], \"args\": {\"image\": \"e2.png\" }}, {\"task\": \"visual-question-answering\", \"id\": 5, \"dep\": [4], \"args\": {\"image\": \"<GENERATED>-4\", \"text\": \"How many sheep in the picture\"}} }}, {\"task\": \"image-to-text\", \"id\": 6, \"dep\": [-1], \"args\": {\"image\": \"e3.jpg\" }}, {\"task\": \"object-detection\", \"id\": 7, \"dep\": [-1], \"args\": {\"image\": \"e3.jpg\" }}, {\"task\": \"visual-question-answering\", \"id\": 8, \"dep\": [7], \"args\": {\"image\": \"<GENERATED>-7\", \"text\": \"How many sheep in the picture\"}}]"
|
||||
},
|
||||
|
||||
{
|
||||
"role":"user",
|
||||
"content":"Look at /e.jpg, can you tell me how many objects in the picture? Give me a picture and video similar to this one."
|
||||
},
|
||||
{
|
||||
"role":"assistant",
|
||||
"content":"[{\"task\": \"image-to-text\", \"id\": 0, \"dep\": [-1], \"args\": {\"image\": \"/e.jpg\" }}, {\"task\": \"object-detection\", \"id\": 1, \"dep\": [-1], \"args\": {\"image\": \"/e.jpg\" }}, {\"task\": \"visual-question-answering\", \"id\": 2, \"dep\": [1], \"args\": {\"image\": \"<GENERATED>-1\", \"text\": \"how many objects in the picture?\" }}, {\"task\": \"text-to-image\", \"id\": 3, \"dep\": [0], \"args\": {\"text\": \"<GENERATED-0>\" }}, {\"task\": \"image-to-image\", \"id\": 4, \"dep\": [-1], \"args\": {\"image\": \"/e.jpg\" }}, {\"task\": \"text-to-video\", \"id\": 5, \"dep\": [0], \"args\": {\"text\": \"<GENERATED-0>\" }}]"
|
||||
},
|
||||
|
||||
{
|
||||
"role":"user",
|
||||
"content":"given a document /images/e.jpeg, answer me what is the student amount? And describe the image with your voice"
|
||||
},
|
||||
{
|
||||
"role":"assistant",
|
||||
"content":"{\"task\": \"document-question-answering\", \"id\": 0, \"dep\": [-1], \"args\": {\"image\": \"/images/e.jpeg\", \"text\": \"what is the student amount?\" }}, {\"task\": \"visual-question-answering\", \"id\": 1, \"dep\": [-1], \"args\": {\"image\": \"/images/e.jpeg\", \"text\": \"what is the student amount?\" }}, {\"task\": \"image-to-text\", \"id\": 2, \"dep\": [-1], \"args\": {\"image\": \"/images/e.jpg\" }}, {\"task\": \"text-to-speech\", \"id\": 3, \"dep\": [2], \"args\": {\"text\": \"<GENERATED>-2\" }}]"
|
||||
},
|
||||
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Given an image /example.jpg, first generate a hed image, then based on the hed image generate a new image where a girl is reading a book"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "[{\"task\": \"openpose-control\", \"id\": 0, \"dep\": [-1], \"args\": {\"image\": \"/example.jpg\" }}, {\"task\": \"openpose-text-to-image\", \"id\": 1, \"dep\": [0], \"args\": {\"text\": \"a girl is reading a book\", \"image\": \"<GENERATED>-0\" }}]"
|
||||
},
|
||||
|
||||
{
|
||||
"role": "user",
|
||||
"content": "please show me a video and an image of (based on the text) 'a boy is running' and dub it"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "[{\"task\": \"text-to-video\", \"id\": 0, \"dep\": [-1], \"args\": {\"text\": \"a boy is running\" }}, {\"task\": \"text-to-speech\", \"id\": 1, \"dep\": [-1], \"args\": {\"text\": \"a boy is running\" }}, {\"task\": \"text-to-image\", \"id\": 2, \"dep\": [-1], \"args\": {\"text\": \"a boy is running\" }}]"
|
||||
},
|
||||
|
||||
|
||||
{
|
||||
"role": "user",
|
||||
"content": "please show me a joke and an image of cat"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "[{\"task\": \"conversational\", \"id\": 0, \"dep\": [-1], \"args\": {\"text\": \"please show me a joke of cat\" }}, {\"task\": \"text-to-image\", \"id\": 1, \"dep\": [-1], \"args\": {\"text\": \"a photo of cat\" }}]"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "{{input}}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Before give you a response, I want to introduce my workflow for your request, which is shown in the following JSON data: {{processes}}. Do you have any demands regarding my response?"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
import tiktoken
|
||||
|
||||
encodings = {
|
||||
"gpt-4": tiktoken.get_encoding("cl100k_base"),
|
||||
"gpt-4-32k": tiktoken.get_encoding("cl100k_base"),
|
||||
"gpt-3.5-turbo": tiktoken.get_encoding("cl100k_base"),
|
||||
"gpt-3.5-turbo-0301": tiktoken.get_encoding("cl100k_base"),
|
||||
"text-davinci-003": tiktoken.get_encoding("p50k_base"),
|
||||
"text-davinci-002": tiktoken.get_encoding("p50k_base"),
|
||||
"text-davinci-001": tiktoken.get_encoding("r50k_base"),
|
||||
"text-curie-001": tiktoken.get_encoding("r50k_base"),
|
||||
"text-babbage-001": tiktoken.get_encoding("r50k_base"),
|
||||
"text-ada-001": tiktoken.get_encoding("r50k_base"),
|
||||
"davinci": tiktoken.get_encoding("r50k_base"),
|
||||
"curie": tiktoken.get_encoding("r50k_base"),
|
||||
"babbage": tiktoken.get_encoding("r50k_base"),
|
||||
"ada": tiktoken.get_encoding("r50k_base"),
|
||||
}
|
||||
|
||||
max_length = {
|
||||
"gpt-4": 8192,
|
||||
"gpt-4-32k": 32768,
|
||||
"gpt-3.5-turbo": 4096,
|
||||
"gpt-3.5-turbo-0301": 4096,
|
||||
"text-davinci-003": 4096,
|
||||
"text-davinci-002": 4096,
|
||||
"text-davinci-001": 2049,
|
||||
"text-curie-001": 2049,
|
||||
"text-babbage-001": 2049,
|
||||
"text-ada-001": 2049,
|
||||
"davinci": 2049,
|
||||
"curie": 2049,
|
||||
"babbage": 2049,
|
||||
"ada": 2049
|
||||
}
|
||||
|
||||
def count_tokens(model_name, text):
|
||||
return len(encodings[model_name].encode(text))
|
||||
|
||||
def get_max_context_length(model_name):
|
||||
return max_length[model_name]
|
||||
|
||||
def get_token_ids_for_task_parsing(model_name):
|
||||
text = '''{"task": "text-classification", "token-classification", "text2text-generation", "summarization", "translation", "question-answering", "conversational", "text-generation", "sentence-similarity", "tabular-classification", "object-detection", "image-classification", "image-to-image", "image-to-text", "text-to-image", "visual-question-answering", "document-question-answering", "image-segmentation", "text-to-speech", "text-to-video", "automatic-speech-recognition", "audio-to-audio", "audio-classification", "canny-control", "hed-control", "mlsd-control", "normal-control", "openpose-control", "canny-text-to-image", "depth-text-to-image", "hed-text-to-image", "mlsd-text-to-image", "normal-text-to-image", "openpose-text-to-image", "seg-text-to-image", "args", "text", "path", "dep", "id", "<GENERATED>-"}'''
|
||||
res = encodings[model_name].encode(text)
|
||||
res = list(set(res))
|
||||
return res
|
||||
|
||||
def get_token_ids_for_choose_model(model_name):
|
||||
text = '''{"id": "reason"}'''
|
||||
res = encodings[model_name].encode(text)
|
||||
res = list(set(res))
|
||||
return res
|
||||
@@ -0,0 +1,60 @@
|
||||
$models = @(
|
||||
"nlpconnect/vit-gpt2-image-captioning",
|
||||
"lllyasviel/ControlNet",
|
||||
"lllyasviel/sd-controlnet-canny",
|
||||
"lllyasviel/sd-controlnet-depth",
|
||||
"lllyasviel/sd-controlnet-hed",
|
||||
"lllyasviel/sd-controlnet-mlsd",
|
||||
"lllyasviel/sd-controlnet-openpose",
|
||||
"lllyasviel/sd-controlnet-scribble",
|
||||
"lllyasviel/sd-controlnet-seg",
|
||||
"runwayml/stable-diffusion-v1-5",
|
||||
"damo-vilab/text-to-video-ms-1.7b",
|
||||
"microsoft/speecht5_asr",
|
||||
"JorisCos/DCCRNet_Libri1Mix_enhsingle_16k",
|
||||
"espnet/kan-bayashi_ljspeech_vits",
|
||||
"facebook/detr-resnet-101",
|
||||
"microsoft/speecht5_hifigan",
|
||||
"microsoft/speecht5_vc",
|
||||
"openai/whisper-base",
|
||||
"Intel/dpt-large",
|
||||
"facebook/detr-resnet-50-panoptic",
|
||||
"facebook/detr-resnet-50",
|
||||
"google/owlvit-base-patch32",
|
||||
"impira/layoutlm-document-qa",
|
||||
"ydshieh/vit-gpt2-coco-en",
|
||||
"dandelin/vilt-b32-finetuned-vqa",
|
||||
"lambdalabs/sd-image-variations-diffusers",
|
||||
"facebook/maskformer-swin-base-coco",
|
||||
"Intel/dpt-hybrid-midas"
|
||||
)
|
||||
|
||||
$CURRENT_DIR = Get-Location
|
||||
|
||||
foreach ($model in $models) {
|
||||
Write-Host "----- Downloading from https://huggingface.co/$model -----"
|
||||
if (Test-Path $model) {
|
||||
Set-Location $model
|
||||
git pull
|
||||
git lfs pull
|
||||
Set-Location $CURRENT_DIR
|
||||
} else {
|
||||
git clone "https://huggingface.co/$model" $model
|
||||
}
|
||||
}
|
||||
|
||||
$datasets = @(
|
||||
"Matthijs/cmu-arctic-xvectors"
|
||||
)
|
||||
|
||||
foreach ($dataset in $datasets) {
|
||||
Write-Host "----- Downloading from https://huggingface.co/datasets/$dataset -----"
|
||||
if (Test-Path $dataset) {
|
||||
Set-Location $dataset
|
||||
git pull
|
||||
git lfs pull
|
||||
Set-Location $CURRENT_DIR
|
||||
} else {
|
||||
git clone "https://huggingface.co/datasets/$dataset" $dataset
|
||||
}
|
||||
}
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Set models and datasets to download
|
||||
models=(
|
||||
"nlpconnect/vit-gpt2-image-captioning"
|
||||
"lllyasviel/ControlNet"
|
||||
"lllyasviel/sd-controlnet-canny"
|
||||
"lllyasviel/sd-controlnet-depth"
|
||||
"lllyasviel/sd-controlnet-hed"
|
||||
"lllyasviel/sd-controlnet-mlsd"
|
||||
"lllyasviel/sd-controlnet-openpose"
|
||||
"lllyasviel/sd-controlnet-scribble"
|
||||
"lllyasviel/sd-controlnet-seg"
|
||||
"runwayml/stable-diffusion-v1-5"
|
||||
"damo-vilab/text-to-video-ms-1.7b"
|
||||
"microsoft/speecht5_asr"
|
||||
"JorisCos/DCCRNet_Libri1Mix_enhsingle_16k"
|
||||
"espnet/kan-bayashi_ljspeech_vits"
|
||||
"facebook/detr-resnet-101"
|
||||
"microsoft/speecht5_hifigan"
|
||||
"microsoft/speecht5_vc"
|
||||
"openai/whisper-base"
|
||||
"Intel/dpt-large"
|
||||
"facebook/detr-resnet-50-panoptic"
|
||||
"facebook/detr-resnet-50"
|
||||
"google/owlvit-base-patch32"
|
||||
"impira/layoutlm-document-qa"
|
||||
"ydshieh/vit-gpt2-coco-en"
|
||||
"dandelin/vilt-b32-finetuned-vqa"
|
||||
"lambdalabs/sd-image-variations-diffusers"
|
||||
"facebook/maskformer-swin-base-coco"
|
||||
"Intel/dpt-hybrid-midas"
|
||||
)
|
||||
datasets=("Matthijs/cmu-arctic-xvectors")
|
||||
|
||||
# Set the current directory
|
||||
CURRENT_DIR=$(pwd)
|
||||
|
||||
# Download models
|
||||
for model in "${models[@]}"; do
|
||||
echo "----- Downloading from https://huggingface.co/${model} -----"
|
||||
if [ -d "${model}" ]; then
|
||||
(cd "${model}" && git pull && git lfs pull)
|
||||
else
|
||||
git clone --recurse-submodules "https://huggingface.co/${model}" "${model}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Download datasets
|
||||
for dataset in "${datasets[@]}"; do
|
||||
echo "----- Downloading from https://huggingface.co/datasets/${dataset} -----"
|
||||
if [ -d "${dataset}" ]; then
|
||||
(cd "${dataset}" && git pull && git lfs pull)
|
||||
else
|
||||
git clone --recurse-submodules "https://huggingface.co/datasets/${dataset}" "${dataset}"
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,635 @@
|
||||
import argparse
|
||||
import logging
|
||||
import random
|
||||
import uuid
|
||||
import numpy as np
|
||||
from transformers import pipeline
|
||||
from diffusers import DiffusionPipeline, StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
|
||||
from diffusers.utils import load_image
|
||||
from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
|
||||
from diffusers.utils import export_to_video
|
||||
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5ForSpeechToSpeech
|
||||
from transformers import BlipProcessor, BlipForConditionalGeneration
|
||||
from transformers import TrOCRProcessor, VisionEncoderDecoderModel, ViTImageProcessor, AutoTokenizer
|
||||
from datasets import load_dataset
|
||||
from PIL import Image
|
||||
import flask
|
||||
from flask import request, jsonify
|
||||
import waitress
|
||||
from flask_cors import CORS
|
||||
import io
|
||||
from torchvision import transforms
|
||||
import torch
|
||||
import torchaudio
|
||||
from speechbrain.pretrained import WaveformEnhancement
|
||||
import joblib
|
||||
from huggingface_hub import hf_hub_url, cached_download
|
||||
from transformers import AutoImageProcessor, TimesformerForVideoClassification
|
||||
from transformers import MaskFormerFeatureExtractor, MaskFormerForInstanceSegmentation, AutoFeatureExtractor
|
||||
from controlnet_aux import OpenposeDetector, MLSDdetector, HEDdetector, CannyDetector, MidasDetector
|
||||
from controlnet_aux.open_pose.body import Body
|
||||
from controlnet_aux.mlsd.models.mbv2_mlsd_large import MobileV2_MLSD_Large
|
||||
from controlnet_aux.hed import Network
|
||||
from transformers import DPTForDepthEstimation, DPTFeatureExtractor
|
||||
import warnings
|
||||
import time
|
||||
from espnet2.bin.tts_inference import Text2Speech
|
||||
import soundfile as sf
|
||||
from asteroid.models import BaseModel
|
||||
import traceback
|
||||
import os
|
||||
import yaml
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=str, default="configs/config.default.yaml")
|
||||
args = parser.parse_args()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
config = yaml.load(open(args.config, "r"), Loader=yaml.FullLoader)
|
||||
|
||||
# host = config["local_inference_endpoint"]["host"]
|
||||
port = config["local_inference_endpoint"]["port"]
|
||||
|
||||
local_deployment = config["local_deployment"]
|
||||
device = config.get("device", "cuda:0")
|
||||
|
||||
PROXY = None
|
||||
if config["proxy"]:
|
||||
PROXY = {
|
||||
"https": config["proxy"],
|
||||
}
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
start = time.time()
|
||||
|
||||
local_fold = "models"
|
||||
# if args.config.endswith(".dev"):
|
||||
# local_fold = "models_dev"
|
||||
|
||||
|
||||
def load_pipes(local_deployment):
|
||||
other_pipes = {}
|
||||
standard_pipes = {}
|
||||
controlnet_sd_pipes = {}
|
||||
if local_deployment in ["full"]:
|
||||
other_pipes = {
|
||||
"nlpconnect/vit-gpt2-image-captioning":{
|
||||
"model": VisionEncoderDecoderModel.from_pretrained(f"{local_fold}/nlpconnect/vit-gpt2-image-captioning"),
|
||||
"feature_extractor": ViTImageProcessor.from_pretrained(f"{local_fold}/nlpconnect/vit-gpt2-image-captioning"),
|
||||
"tokenizer": AutoTokenizer.from_pretrained(f"{local_fold}/nlpconnect/vit-gpt2-image-captioning"),
|
||||
"device": device
|
||||
},
|
||||
# "Salesforce/blip-image-captioning-large": {
|
||||
# "model": BlipForConditionalGeneration.from_pretrained(f"{local_fold}/Salesforce/blip-image-captioning-large"),
|
||||
# "processor": BlipProcessor.from_pretrained(f"{local_fold}/Salesforce/blip-image-captioning-large"),
|
||||
# "device": device
|
||||
# },
|
||||
"damo-vilab/text-to-video-ms-1.7b": {
|
||||
"model": DiffusionPipeline.from_pretrained(f"{local_fold}/damo-vilab/text-to-video-ms-1.7b", torch_dtype=torch.float16, variant="fp16"),
|
||||
"device": device
|
||||
},
|
||||
# "facebook/maskformer-swin-large-ade": {
|
||||
# "model": MaskFormerForInstanceSegmentation.from_pretrained(f"{local_fold}/facebook/maskformer-swin-large-ade"),
|
||||
# "feature_extractor" : AutoFeatureExtractor.from_pretrained("facebook/maskformer-swin-large-ade"),
|
||||
# "device": device
|
||||
# },
|
||||
# "microsoft/trocr-base-printed": {
|
||||
# "processor": TrOCRProcessor.from_pretrained(f"{local_fold}/microsoft/trocr-base-printed"),
|
||||
# "model": VisionEncoderDecoderModel.from_pretrained(f"{local_fold}/microsoft/trocr-base-printed"),
|
||||
# "device": device
|
||||
# },
|
||||
# "microsoft/trocr-base-handwritten": {
|
||||
# "processor": TrOCRProcessor.from_pretrained(f"{local_fold}/microsoft/trocr-base-handwritten"),
|
||||
# "model": VisionEncoderDecoderModel.from_pretrained(f"{local_fold}/microsoft/trocr-base-handwritten"),
|
||||
# "device": device
|
||||
# },
|
||||
"JorisCos/DCCRNet_Libri1Mix_enhsingle_16k": {
|
||||
"model": BaseModel.from_pretrained("JorisCos/DCCRNet_Libri1Mix_enhsingle_16k"),
|
||||
"device": device
|
||||
},
|
||||
"espnet/kan-bayashi_ljspeech_vits": {
|
||||
"model": Text2Speech.from_pretrained(f"espnet/kan-bayashi_ljspeech_vits"),
|
||||
"device": device
|
||||
},
|
||||
"lambdalabs/sd-image-variations-diffusers": {
|
||||
"model": DiffusionPipeline.from_pretrained(f"{local_fold}/lambdalabs/sd-image-variations-diffusers"), #torch_dtype=torch.float16
|
||||
"device": device
|
||||
},
|
||||
# "CompVis/stable-diffusion-v1-4": {
|
||||
# "model": DiffusionPipeline.from_pretrained(f"{local_fold}/CompVis/stable-diffusion-v1-4"),
|
||||
# "device": device
|
||||
# },
|
||||
# "stabilityai/stable-diffusion-2-1": {
|
||||
# "model": DiffusionPipeline.from_pretrained(f"{local_fold}/stabilityai/stable-diffusion-2-1"),
|
||||
# "device": device
|
||||
# },
|
||||
"runwayml/stable-diffusion-v1-5": {
|
||||
"model": DiffusionPipeline.from_pretrained(f"{local_fold}/runwayml/stable-diffusion-v1-5"),
|
||||
"device": device
|
||||
},
|
||||
# "microsoft/speecht5_tts":{
|
||||
# "processor": SpeechT5Processor.from_pretrained(f"{local_fold}/microsoft/speecht5_tts"),
|
||||
# "model": SpeechT5ForTextToSpeech.from_pretrained(f"{local_fold}/microsoft/speecht5_tts"),
|
||||
# "vocoder": SpeechT5HifiGan.from_pretrained(f"{local_fold}/microsoft/speecht5_hifigan"),
|
||||
# "embeddings_dataset": load_dataset(f"{local_fold}/Matthijs/cmu-arctic-xvectors", split="validation"),
|
||||
# "device": device
|
||||
# },
|
||||
# "speechbrain/mtl-mimic-voicebank": {
|
||||
# "model": WaveformEnhancement.from_hparams(source="speechbrain/mtl-mimic-voicebank", savedir="models/mtl-mimic-voicebank"),
|
||||
# "device": device
|
||||
# },
|
||||
"microsoft/speecht5_vc":{
|
||||
"processor": SpeechT5Processor.from_pretrained(f"{local_fold}/microsoft/speecht5_vc"),
|
||||
"model": SpeechT5ForSpeechToSpeech.from_pretrained(f"{local_fold}/microsoft/speecht5_vc"),
|
||||
"vocoder": SpeechT5HifiGan.from_pretrained(f"{local_fold}/microsoft/speecht5_hifigan"),
|
||||
"embeddings_dataset": load_dataset(f"{local_fold}/Matthijs/cmu-arctic-xvectors", split="validation"),
|
||||
"device": device
|
||||
},
|
||||
# "julien-c/wine-quality": {
|
||||
# "model": joblib.load(cached_download(hf_hub_url("julien-c/wine-quality", "sklearn_model.joblib")))
|
||||
# },
|
||||
# "facebook/timesformer-base-finetuned-k400": {
|
||||
# "processor": AutoImageProcessor.from_pretrained(f"{local_fold}/facebook/timesformer-base-finetuned-k400"),
|
||||
# "model": TimesformerForVideoClassification.from_pretrained(f"{local_fold}/facebook/timesformer-base-finetuned-k400"),
|
||||
# "device": device
|
||||
# },
|
||||
"facebook/maskformer-swin-base-coco": {
|
||||
"feature_extractor": MaskFormerFeatureExtractor.from_pretrained(f"{local_fold}/facebook/maskformer-swin-base-coco"),
|
||||
"model": MaskFormerForInstanceSegmentation.from_pretrained(f"{local_fold}/facebook/maskformer-swin-base-coco"),
|
||||
"device": device
|
||||
},
|
||||
"Intel/dpt-hybrid-midas": {
|
||||
"model": DPTForDepthEstimation.from_pretrained(f"{local_fold}/Intel/dpt-hybrid-midas", low_cpu_mem_usage=True),
|
||||
"feature_extractor": DPTFeatureExtractor.from_pretrained(f"{local_fold}/Intel/dpt-hybrid-midas"),
|
||||
"device": device
|
||||
}
|
||||
}
|
||||
|
||||
if local_deployment in ["full", "standard"]:
|
||||
standard_pipes = {
|
||||
# "superb/wav2vec2-base-superb-ks": {
|
||||
# "model": pipeline(task="audio-classification", model=f"{local_fold}/superb/wav2vec2-base-superb-ks"),
|
||||
# "device": device
|
||||
# },
|
||||
"openai/whisper-base": {
|
||||
"model": pipeline(task="automatic-speech-recognition", model=f"{local_fold}/openai/whisper-base"),
|
||||
"device": device
|
||||
},
|
||||
"microsoft/speecht5_asr": {
|
||||
"model": pipeline(task="automatic-speech-recognition", model=f"{local_fold}/microsoft/speecht5_asr"),
|
||||
"device": device
|
||||
},
|
||||
"Intel/dpt-large": {
|
||||
"model": pipeline(task="depth-estimation", model=f"{local_fold}/Intel/dpt-large"),
|
||||
"device": device
|
||||
},
|
||||
# "microsoft/beit-base-patch16-224-pt22k-ft22k": {
|
||||
# "model": pipeline(task="image-classification", model=f"{local_fold}/microsoft/beit-base-patch16-224-pt22k-ft22k"),
|
||||
# "device": device
|
||||
# },
|
||||
"facebook/detr-resnet-50-panoptic": {
|
||||
"model": pipeline(task="image-segmentation", model=f"{local_fold}/facebook/detr-resnet-50-panoptic"),
|
||||
"device": device
|
||||
},
|
||||
"facebook/detr-resnet-101": {
|
||||
"model": pipeline(task="object-detection", model=f"{local_fold}/facebook/detr-resnet-101"),
|
||||
"device": device
|
||||
},
|
||||
# "openai/clip-vit-large-patch14": {
|
||||
# "model": pipeline(task="zero-shot-image-classification", model=f"{local_fold}/openai/clip-vit-large-patch14"),
|
||||
# "device": device
|
||||
# },
|
||||
"google/owlvit-base-patch32": {
|
||||
"model": pipeline(task="zero-shot-object-detection", model=f"{local_fold}/google/owlvit-base-patch32"),
|
||||
"device": device
|
||||
},
|
||||
# "microsoft/DialoGPT-medium": {
|
||||
# "model": pipeline(task="conversational", model=f"{local_fold}/microsoft/DialoGPT-medium"),
|
||||
# "device": device
|
||||
# },
|
||||
# "bert-base-uncased": {
|
||||
# "model": pipeline(task="fill-mask", model=f"{local_fold}/bert-base-uncased"),
|
||||
# "device": device
|
||||
# },
|
||||
# "deepset/roberta-base-squad2": {
|
||||
# "model": pipeline(task = "question-answering", model=f"{local_fold}/deepset/roberta-base-squad2"),
|
||||
# "device": device
|
||||
# },
|
||||
# "facebook/bart-large-cnn": {
|
||||
# "model": pipeline(task="summarization", model=f"{local_fold}/facebook/bart-large-cnn"),
|
||||
# "device": device
|
||||
# },
|
||||
# "google/tapas-base-finetuned-wtq": {
|
||||
# "model": pipeline(task="table-question-answering", model=f"{local_fold}/google/tapas-base-finetuned-wtq"),
|
||||
# "device": device
|
||||
# },
|
||||
# "distilbert-base-uncased-finetuned-sst-2-english": {
|
||||
# "model": pipeline(task="text-classification", model=f"{local_fold}/distilbert-base-uncased-finetuned-sst-2-english"),
|
||||
# "device": device
|
||||
# },
|
||||
# "gpt2": {
|
||||
# "model": pipeline(task="text-generation", model="gpt2"),
|
||||
# "device": device
|
||||
# },
|
||||
# "mrm8488/t5-base-finetuned-question-generation-ap": {
|
||||
# "model": pipeline(task="text2text-generation", model=f"{local_fold}/mrm8488/t5-base-finetuned-question-generation-ap"),
|
||||
# "device": device
|
||||
# },
|
||||
# "Jean-Baptiste/camembert-ner": {
|
||||
# "model": pipeline(task="token-classification", model=f"{local_fold}/Jean-Baptiste/camembert-ner", aggregation_strategy="simple"),
|
||||
# "device": device
|
||||
# },
|
||||
# "t5-base": {
|
||||
# "model": pipeline(task="translation", model=f"{local_fold}/t5-base"),
|
||||
# "device": device
|
||||
# },
|
||||
"impira/layoutlm-document-qa": {
|
||||
"model": pipeline(task="document-question-answering", model=f"{local_fold}/impira/layoutlm-document-qa"),
|
||||
"device": device
|
||||
},
|
||||
"ydshieh/vit-gpt2-coco-en": {
|
||||
"model": pipeline(task="image-to-text", model=f"{local_fold}/ydshieh/vit-gpt2-coco-en"),
|
||||
"device": device
|
||||
},
|
||||
"dandelin/vilt-b32-finetuned-vqa": {
|
||||
"model": pipeline(task="visual-question-answering", model=f"{local_fold}/dandelin/vilt-b32-finetuned-vqa"),
|
||||
"device": device
|
||||
}
|
||||
}
|
||||
|
||||
if local_deployment in ["full", "standard", "minimal"]:
|
||||
controlnet = ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
|
||||
controlnetpipe = StableDiffusionControlNetPipeline.from_pretrained(
|
||||
f"{local_fold}/runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
|
||||
)
|
||||
|
||||
def mlsd_control_network():
|
||||
model = MobileV2_MLSD_Large()
|
||||
model.load_state_dict(torch.load(f"{local_fold}/lllyasviel/ControlNet/annotator/ckpts/mlsd_large_512_fp32.pth"), strict=True)
|
||||
return MLSDdetector(model)
|
||||
|
||||
|
||||
hed_network = Network(f"{local_fold}/lllyasviel/ControlNet/annotator/ckpts/network-bsds500.pth")
|
||||
|
||||
controlnet_sd_pipes = {
|
||||
"openpose-control": {
|
||||
"model": OpenposeDetector(Body(f"{local_fold}/lllyasviel/ControlNet/annotator/ckpts/body_pose_model.pth"))
|
||||
},
|
||||
"mlsd-control": {
|
||||
"model": mlsd_control_network()
|
||||
},
|
||||
"hed-control": {
|
||||
"model": HEDdetector(hed_network)
|
||||
},
|
||||
"scribble-control": {
|
||||
"model": HEDdetector(hed_network)
|
||||
},
|
||||
"midas-control": {
|
||||
"model": MidasDetector(model_path=f"{local_fold}/lllyasviel/ControlNet/annotator/ckpts/dpt_hybrid-midas-501f0c75.pt")
|
||||
},
|
||||
"canny-control": {
|
||||
"model": CannyDetector()
|
||||
},
|
||||
"lllyasviel/sd-controlnet-canny":{
|
||||
"control": controlnet,
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-depth":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-depth", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-hed":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-hed", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-mlsd":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-mlsd", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-openpose":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-scribble":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-scribble", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
},
|
||||
"lllyasviel/sd-controlnet-seg":{
|
||||
"control": ControlNetModel.from_pretrained(f"{local_fold}/lllyasviel/sd-controlnet-seg", torch_dtype=torch.float16),
|
||||
"model": controlnetpipe,
|
||||
"device": device
|
||||
}
|
||||
}
|
||||
pipes = {**standard_pipes, **other_pipes, **controlnet_sd_pipes}
|
||||
return pipes
|
||||
|
||||
pipes = load_pipes(local_deployment)
|
||||
|
||||
end = time.time()
|
||||
during = end - start
|
||||
|
||||
print(f"[ ready ] {during}s")
|
||||
|
||||
@app.route('/running', methods=['GET'])
|
||||
def running():
|
||||
return jsonify({"running": True})
|
||||
|
||||
@app.route('/status/<path:model_id>', methods=['GET'])
|
||||
def status(model_id):
|
||||
disabled_models = ["microsoft/trocr-base-printed", "microsoft/trocr-base-handwritten"]
|
||||
if model_id in pipes.keys() and model_id not in disabled_models:
|
||||
print(f"[ check {model_id} ] success")
|
||||
return jsonify({"loaded": True})
|
||||
else:
|
||||
print(f"[ check {model_id} ] failed")
|
||||
return jsonify({"loaded": False})
|
||||
|
||||
@app.route('/models/<path:model_id>', methods=['POST'])
|
||||
def models(model_id):
|
||||
while "using" in pipes[model_id] and pipes[model_id]["using"]:
|
||||
print(f"[ inference {model_id} ] waiting")
|
||||
time.sleep(0.1)
|
||||
pipes[model_id]["using"] = True
|
||||
print(f"[ inference {model_id} ] start")
|
||||
|
||||
start = time.time()
|
||||
|
||||
pipe = pipes[model_id]["model"]
|
||||
|
||||
if "device" in pipes[model_id]:
|
||||
try:
|
||||
pipe.to(pipes[model_id]["device"])
|
||||
except:
|
||||
pipe.device = torch.device(pipes[model_id]["device"])
|
||||
pipe.model.to(pipes[model_id]["device"])
|
||||
|
||||
result = None
|
||||
try:
|
||||
# text to video
|
||||
if model_id == "damo-vilab/text-to-video-ms-1.7b":
|
||||
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
|
||||
# pipe.enable_model_cpu_offload()
|
||||
prompt = request.get_json()["text"]
|
||||
video_frames = pipe(prompt, num_inference_steps=50, num_frames=40).frames
|
||||
video_path = export_to_video(video_frames)
|
||||
file_name = str(uuid.uuid4())[:4]
|
||||
os.system(f"LD_LIBRARY_PATH=/usr/local/lib /usr/local/bin/ffmpeg -i {video_path} -vcodec libx264 public/videos/{file_name}.mp4")
|
||||
result = {"path": f"/videos/{file_name}.mp4"}
|
||||
|
||||
# controlnet
|
||||
if model_id.startswith("lllyasviel/sd-controlnet-"):
|
||||
pipe.controlnet.to('cpu')
|
||||
pipe.controlnet = pipes[model_id]["control"].to(pipes[model_id]["device"])
|
||||
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
|
||||
control_image = load_image(request.get_json()["img_url"])
|
||||
# generator = torch.manual_seed(66)
|
||||
out_image: Image = pipe(request.get_json()["text"], num_inference_steps=20, image=control_image).images[0]
|
||||
file_name = str(uuid.uuid4())[:4]
|
||||
out_image.save(f"public/images/{file_name}.png")
|
||||
result = {"path": f"/images/{file_name}.png"}
|
||||
|
||||
if model_id.endswith("-control"):
|
||||
image = load_image(request.get_json()["img_url"])
|
||||
if "scribble" in model_id:
|
||||
control = pipe(image, scribble = True)
|
||||
elif "canny" in model_id:
|
||||
control = pipe(image, low_threshold=100, high_threshold=200)
|
||||
else:
|
||||
control = pipe(image)
|
||||
file_name = str(uuid.uuid4())[:4]
|
||||
control.save(f"public/images/{file_name}.png")
|
||||
result = {"path": f"/images/{file_name}.png"}
|
||||
|
||||
# image to image
|
||||
if model_id == "lambdalabs/sd-image-variations-diffusers":
|
||||
im = load_image(request.get_json()["img_url"])
|
||||
file_name = str(uuid.uuid4())[:4]
|
||||
with open(f"public/images/{file_name}.png", "wb") as f:
|
||||
f.write(request.data)
|
||||
tform = transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Resize(
|
||||
(224, 224),
|
||||
interpolation=transforms.InterpolationMode.BICUBIC,
|
||||
antialias=False,
|
||||
),
|
||||
transforms.Normalize(
|
||||
[0.48145466, 0.4578275, 0.40821073],
|
||||
[0.26862954, 0.26130258, 0.27577711]),
|
||||
])
|
||||
inp = tform(im).to(pipes[model_id]["device"]).unsqueeze(0)
|
||||
out = pipe(inp, guidance_scale=3)
|
||||
out["images"][0].save(f"public/images/{file_name}.jpg")
|
||||
result = {"path": f"/images/{file_name}.jpg"}
|
||||
|
||||
# image to text
|
||||
if model_id == "Salesforce/blip-image-captioning-large":
|
||||
raw_image = load_image(request.get_json()["img_url"]).convert('RGB')
|
||||
text = request.get_json()["text"]
|
||||
inputs = pipes[model_id]["processor"](raw_image, return_tensors="pt").to(pipes[model_id]["device"])
|
||||
out = pipe.generate(**inputs)
|
||||
caption = pipes[model_id]["processor"].decode(out[0], skip_special_tokens=True)
|
||||
result = {"generated text": caption}
|
||||
if model_id == "ydshieh/vit-gpt2-coco-en":
|
||||
img_url = request.get_json()["img_url"]
|
||||
generated_text = pipe(img_url)[0]['generated_text']
|
||||
result = {"generated text": generated_text}
|
||||
if model_id == "nlpconnect/vit-gpt2-image-captioning":
|
||||
image = load_image(request.get_json()["img_url"]).convert("RGB")
|
||||
pixel_values = pipes[model_id]["feature_extractor"](images=image, return_tensors="pt").pixel_values
|
||||
pixel_values = pixel_values.to(pipes[model_id]["device"])
|
||||
generated_ids = pipe.generate(pixel_values, **{"max_length": 200, "num_beams": 1})
|
||||
generated_text = pipes[model_id]["tokenizer"].batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
result = {"generated text": generated_text}
|
||||
# image to text: OCR
|
||||
if model_id == "microsoft/trocr-base-printed" or model_id == "microsoft/trocr-base-handwritten":
|
||||
image = load_image(request.get_json()["img_url"]).convert("RGB")
|
||||
pixel_values = pipes[model_id]["processor"](image, return_tensors="pt").pixel_values
|
||||
pixel_values = pixel_values.to(pipes[model_id]["device"])
|
||||
generated_ids = pipe.generate(pixel_values)
|
||||
generated_text = pipes[model_id]["processor"].batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
result = {"generated text": generated_text}
|
||||
|
||||
# text to image
|
||||
if model_id == "runwayml/stable-diffusion-v1-5":
|
||||
file_name = str(uuid.uuid4())[:4]
|
||||
text = request.get_json()["text"]
|
||||
out = pipe(prompt=text)
|
||||
out["images"][0].save(f"public/images/{file_name}.jpg")
|
||||
result = {"path": f"/images/{file_name}.jpg"}
|
||||
|
||||
# object detection
|
||||
if model_id == "google/owlvit-base-patch32" or model_id == "facebook/detr-resnet-101":
|
||||
img_url = request.get_json()["img_url"]
|
||||
open_types = ["cat", "couch", "person", "car", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird"]
|
||||
result = pipe(img_url, candidate_labels=open_types)
|
||||
|
||||
# VQA
|
||||
if model_id == "dandelin/vilt-b32-finetuned-vqa":
|
||||
question = request.get_json()["text"]
|
||||
img_url = request.get_json()["img_url"]
|
||||
result = pipe(question=question, image=img_url)
|
||||
|
||||
#DQA
|
||||
if model_id == "impira/layoutlm-document-qa":
|
||||
question = request.get_json()["text"]
|
||||
img_url = request.get_json()["img_url"]
|
||||
result = pipe(img_url, question)
|
||||
|
||||
# depth-estimation
|
||||
if model_id == "Intel/dpt-large":
|
||||
output = pipe(request.get_json()["img_url"])
|
||||
image = output['depth']
|
||||
name = str(uuid.uuid4())[:4]
|
||||
image.save(f"public/images/{name}.jpg")
|
||||
result = {"path": f"/images/{name}.jpg"}
|
||||
|
||||
if model_id == "Intel/dpt-hybrid-midas" and model_id == "Intel/dpt-large":
|
||||
image = load_image(request.get_json()["img_url"])
|
||||
inputs = pipes[model_id]["feature_extractor"](images=image, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
outputs = pipe(**inputs)
|
||||
predicted_depth = outputs.predicted_depth
|
||||
prediction = torch.nn.functional.interpolate(
|
||||
predicted_depth.unsqueeze(1),
|
||||
size=image.size[::-1],
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
)
|
||||
output = prediction.squeeze().cpu().numpy()
|
||||
formatted = (output * 255 / np.max(output)).astype("uint8")
|
||||
image = Image.fromarray(formatted)
|
||||
name = str(uuid.uuid4())[:4]
|
||||
image.save(f"public/images/{name}.jpg")
|
||||
result = {"path": f"/images/{name}.jpg"}
|
||||
|
||||
# TTS
|
||||
if model_id == "espnet/kan-bayashi_ljspeech_vits":
|
||||
text = request.get_json()["text"]
|
||||
wav = pipe(text)["wav"]
|
||||
name = str(uuid.uuid4())[:4]
|
||||
sf.write(f"public/audios/{name}.wav", wav.cpu().numpy(), pipe.fs, "PCM_16")
|
||||
result = {"path": f"/audios/{name}.wav"}
|
||||
|
||||
if model_id == "microsoft/speecht5_tts":
|
||||
text = request.get_json()["text"]
|
||||
inputs = pipes[model_id]["processor"](text=text, return_tensors="pt")
|
||||
embeddings_dataset = pipes[model_id]["embeddings_dataset"]
|
||||
speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0).to(pipes[model_id]["device"])
|
||||
pipes[model_id]["vocoder"].to(pipes[model_id]["device"])
|
||||
speech = pipe.generate_speech(inputs["input_ids"].to(pipes[model_id]["device"]), speaker_embeddings, vocoder=pipes[model_id]["vocoder"])
|
||||
name = str(uuid.uuid4())[:4]
|
||||
sf.write(f"public/audios/{name}.wav", speech.cpu().numpy(), samplerate=16000)
|
||||
result = {"path": f"/audios/{name}.wav"}
|
||||
|
||||
# ASR
|
||||
if model_id == "openai/whisper-base" or model_id == "microsoft/speecht5_asr":
|
||||
audio_url = request.get_json()["audio_url"]
|
||||
result = { "text": pipe(audio_url)["text"]}
|
||||
|
||||
# audio to audio
|
||||
if model_id == "JorisCos/DCCRNet_Libri1Mix_enhsingle_16k":
|
||||
audio_url = request.get_json()["audio_url"]
|
||||
wav, sr = torchaudio.load(audio_url)
|
||||
with torch.no_grad():
|
||||
result_wav = pipe(wav.to(pipes[model_id]["device"]))
|
||||
name = str(uuid.uuid4())[:4]
|
||||
sf.write(f"public/audios/{name}.wav", result_wav.cpu().squeeze().numpy(), sr)
|
||||
result = {"path": f"/audios/{name}.wav"}
|
||||
|
||||
if model_id == "microsoft/speecht5_vc":
|
||||
audio_url = request.get_json()["audio_url"]
|
||||
wav, sr = torchaudio.load(audio_url)
|
||||
inputs = pipes[model_id]["processor"](audio=wav, sampling_rate=sr, return_tensors="pt")
|
||||
embeddings_dataset = pipes[model_id]["embeddings_dataset"]
|
||||
speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
|
||||
pipes[model_id]["vocoder"].to(pipes[model_id]["device"])
|
||||
speech = pipe.generate_speech(inputs["input_ids"].to(pipes[model_id]["device"]), speaker_embeddings, vocoder=pipes[model_id]["vocoder"])
|
||||
name = str(uuid.uuid4())[:4]
|
||||
sf.write(f"public/audios/{name}.wav", speech.cpu().numpy(), samplerate=16000)
|
||||
result = {"path": f"/audios/{name}.wav"}
|
||||
|
||||
# segmentation
|
||||
if model_id == "facebook/detr-resnet-50-panoptic":
|
||||
result = []
|
||||
segments = pipe(request.get_json()["img_url"])
|
||||
image = load_image(request.get_json()["img_url"])
|
||||
|
||||
colors = []
|
||||
for i in range(len(segments)):
|
||||
colors.append((random.randint(100, 255), random.randint(100, 255), random.randint(100, 255), 50))
|
||||
|
||||
for segment in segments:
|
||||
mask = segment["mask"]
|
||||
mask = mask.convert('L')
|
||||
layer = Image.new('RGBA', mask.size, colors[i])
|
||||
image.paste(layer, (0, 0), mask)
|
||||
name = str(uuid.uuid4())[:4]
|
||||
image.save(f"public/images/{name}.jpg")
|
||||
result = {"path": f"/images/{name}.jpg"}
|
||||
|
||||
if model_id == "facebook/maskformer-swin-base-coco" or model_id == "facebook/maskformer-swin-large-ade":
|
||||
image = load_image(request.get_json()["img_url"])
|
||||
inputs = pipes[model_id]["feature_extractor"](images=image, return_tensors="pt").to(pipes[model_id]["device"])
|
||||
outputs = pipe(**inputs)
|
||||
result = pipes[model_id]["feature_extractor"].post_process_panoptic_segmentation(outputs, target_sizes=[image.size[::-1]])[0]
|
||||
predicted_panoptic_map = result["segmentation"].cpu().numpy()
|
||||
predicted_panoptic_map = Image.fromarray(predicted_panoptic_map.astype(np.uint8))
|
||||
name = str(uuid.uuid4())[:4]
|
||||
predicted_panoptic_map.save(f"public/images/{name}.jpg")
|
||||
result = {"path": f"/images/{name}.jpg"}
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
traceback.print_exc()
|
||||
result = {"error": {"message": "Error when running the model inference."}}
|
||||
|
||||
if "device" in pipes[model_id]:
|
||||
try:
|
||||
pipe.to("cpu")
|
||||
torch.cuda.empty_cache()
|
||||
except:
|
||||
pipe.device = torch.device("cpu")
|
||||
pipe.model.to("cpu")
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
pipes[model_id]["using"] = False
|
||||
|
||||
if result is None:
|
||||
result = {"error": {"message": "model not found"}}
|
||||
|
||||
end = time.time()
|
||||
during = end - start
|
||||
print(f"[ complete {model_id} ] {during}s")
|
||||
print(f"[ result {model_id} ] {result}")
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# temp folders
|
||||
if not os.path.exists("public/audios"):
|
||||
os.makedirs("public/audios")
|
||||
if not os.path.exists("public/images"):
|
||||
os.makedirs("public/images")
|
||||
if not os.path.exists("public/videos"):
|
||||
os.makedirs("public/videos")
|
||||
|
||||
waitress.serve(app, host="0.0.0.0", port=port)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 65 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
@@ -0,0 +1,18 @@
|
||||
git+https://github.com/huggingface/diffusers.git@8c530fc2f6a76a2aefb6b285dce6df1675092ac6#egg=diffusers
|
||||
git+https://github.com/huggingface/transformers@c612628045822f909020f7eb6784c79700813eda#egg=transformers
|
||||
git+https://github.com/patrickvonplaten/controlnet_aux@78efc716868a7f5669c288233d65b471f542ce40#egg=controlnet_aux
|
||||
tiktoken==0.3.3
|
||||
pydub==0.25.1
|
||||
espnet==202301
|
||||
espnet_model_zoo==0.1.7
|
||||
flask==2.2.3
|
||||
flask_cors==3.0.10
|
||||
waitress==2.1.2
|
||||
datasets==2.11.0
|
||||
asteroid==0.6.0
|
||||
speechbrain==0.5.14
|
||||
timm==0.6.13
|
||||
typeguard==2.13.3
|
||||
accelerate==0.18.0
|
||||
pytesseract==0.3.10
|
||||
gradio==3.24.1
|
||||
@@ -0,0 +1,131 @@
|
||||
import uuid
|
||||
import gradio as gr
|
||||
import re
|
||||
from diffusers.utils import load_image
|
||||
import requests
|
||||
from awesome_chat import chat_huggingface
|
||||
|
||||
all_messages = []
|
||||
OPENAI_KEY = ""
|
||||
|
||||
def add_message(content, role):
|
||||
message = {"role":role, "content":content}
|
||||
all_messages.append(message)
|
||||
|
||||
def extract_medias(message):
|
||||
image_pattern = re.compile(r"(http(s?):|\/)?([\.\/_\w:-])*?\.(jpg|jpeg|tiff|gif|png)")
|
||||
image_urls = []
|
||||
for match in image_pattern.finditer(message):
|
||||
if match.group(0) not in image_urls:
|
||||
image_urls.append(match.group(0))
|
||||
|
||||
audio_pattern = re.compile(r"(http(s?):|\/)?([\.\/_\w:-])*?\.(flac|wav)")
|
||||
audio_urls = []
|
||||
for match in audio_pattern.finditer(message):
|
||||
if match.group(0) not in audio_urls:
|
||||
audio_urls.append(match.group(0))
|
||||
|
||||
video_pattern = re.compile(r"(http(s?):|\/)?([\.\/_\w:-])*?\.(mp4)")
|
||||
video_urls = []
|
||||
for match in video_pattern.finditer(message):
|
||||
if match.group(0) not in video_urls:
|
||||
video_urls.append(match.group(0))
|
||||
|
||||
return image_urls, audio_urls, video_urls
|
||||
|
||||
def set_openai_key(openai_key):
|
||||
global OPENAI_KEY
|
||||
OPENAI_KEY = openai_key
|
||||
return OPENAI_KEY
|
||||
|
||||
def add_text(messages, message):
|
||||
if len(OPENAI_KEY) == 0 or not OPENAI_KEY.startswith("sk-"):
|
||||
return messages, "Please set your OpenAI API key first."
|
||||
add_message(message, "user")
|
||||
messages = messages + [(message, None)]
|
||||
image_urls, audio_urls, video_urls = extract_medias(message)
|
||||
|
||||
for image_url in image_urls:
|
||||
if not image_url.startswith("http"):
|
||||
image_url = "public/" + image_url
|
||||
image = load_image(image_url)
|
||||
name = f"public/images/{str(uuid.uuid4())[:4]}.jpg"
|
||||
image.save(name)
|
||||
messages = messages + [((f"{name}",), None)]
|
||||
for audio_url in audio_urls:
|
||||
if not audio_url.startswith("http"):
|
||||
audio_url = "public/" + audio_url
|
||||
ext = audio_url.split(".")[-1]
|
||||
name = f"public/audios/{str(uuid.uuid4()[:4])}.{ext}"
|
||||
response = requests.get(audio_url)
|
||||
with open(name, "wb") as f:
|
||||
f.write(response.content)
|
||||
messages = messages + [((f"{name}",), None)]
|
||||
for video_url in video_urls:
|
||||
if not video_url.startswith("http"):
|
||||
video_url = "public/" + video_url
|
||||
ext = video_url.split(".")[-1]
|
||||
name = f"public/audios/{str(uuid.uuid4()[:4])}.{ext}"
|
||||
response = requests.get(video_url)
|
||||
with open(name, "wb") as f:
|
||||
f.write(response.content)
|
||||
messages = messages + [((f"{name}",), None)]
|
||||
return messages, ""
|
||||
|
||||
def bot(messages):
|
||||
if len(OPENAI_KEY) == 0 or not OPENAI_KEY.startswith("sk-"):
|
||||
return messages
|
||||
message = chat_huggingface(all_messages, OPENAI_KEY, "openai", "https://api.openai.com/v1/completions")["message"]
|
||||
image_urls, audio_urls, video_urls = extract_medias(message)
|
||||
add_message(message, "assistant")
|
||||
messages[-1][1] = message
|
||||
for image_url in image_urls:
|
||||
if not image_url.startswith("http"):
|
||||
image_url = image_url.replace("public/", "")
|
||||
messages = messages + [((None, (f"public/{image_url}",)))]
|
||||
for audio_url in audio_urls:
|
||||
if not audio_url.startswith("http"):
|
||||
audio_url = audio_url.replace("public/", "")
|
||||
messages = messages + [((None, (f"public/{audio_url}",)))]
|
||||
for video_url in video_urls:
|
||||
if not video_url.startswith("http"):
|
||||
video_url = video_url.replace("public/", "")
|
||||
messages = messages + [((None, (f"public/{video_url}",)))]
|
||||
return messages
|
||||
|
||||
with gr.Blocks() as demo:
|
||||
gr.Markdown("<h2><center>HuggingGPT (Dev)</center></h2>")
|
||||
with gr.Row():
|
||||
openai_api_key = gr.Textbox(
|
||||
show_label=False,
|
||||
placeholder="Set your OpenAI API key here and press Enter",
|
||||
lines=1,
|
||||
type="password",
|
||||
)
|
||||
|
||||
chatbot = gr.Chatbot([], elem_id="chatbot").style(height=500)
|
||||
|
||||
with gr.Row():
|
||||
txt = gr.Textbox(
|
||||
show_label=False,
|
||||
placeholder="Enter text and press enter. The url of the multimedia resource must contain the extension name.",
|
||||
).style(container=False)
|
||||
|
||||
txt.submit(add_text, [chatbot, txt], [chatbot, txt]).then(
|
||||
bot, chatbot, chatbot
|
||||
)
|
||||
openai_api_key.submit(set_openai_key, [openai_api_key], [openai_api_key])
|
||||
|
||||
gr.Examples(
|
||||
examples=["Given a collection of image A: /examples/a.jpg, B: /examples/b.jpg, C: /examples/c.jpg, please tell me how many zebras in these picture?",
|
||||
"Please generate a canny image based on /examples/f.jpg",
|
||||
"show me a joke and an image of cat",
|
||||
"what is in the /examples/a.jpg",
|
||||
"generate a video and audio about a dog is running on the grass",
|
||||
"based on the /examples/a.jpg, please generate a video and audio",
|
||||
"based on pose of /examples/d.jpg and content of /examples/e.jpg, please show me a new image",
|
||||
],
|
||||
inputs=txt
|
||||
)
|
||||
|
||||
demo.launch()
|
||||
@@ -0,0 +1,3 @@
|
||||
registry=https://registry.npmmirror.com
|
||||
electron_mirror=https://npmmirror.com/mirrors/electron/
|
||||
chromedriver_cdnurl=https://npmmirror.com/mirrors/chromedriver
|
||||
@@ -0,0 +1,18 @@
|
||||
appId: com.chatgpt.app
|
||||
productName: ChatGPT
|
||||
directories:
|
||||
output: ../electron-dist
|
||||
mac:
|
||||
category: public.app-category.productivity
|
||||
target:
|
||||
- target: dmg
|
||||
arch:
|
||||
- arm64
|
||||
- x64
|
||||
win:
|
||||
target:
|
||||
- target: nsis
|
||||
arch:
|
||||
- x64
|
||||
- ia32
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
// Modules to control application life and create native browser window
|
||||
const {app, BrowserWindow} = require('electron')
|
||||
const path = require('path')
|
||||
|
||||
function createWindow () {
|
||||
// Create the browser window.
|
||||
const mainWindow = new BrowserWindow({
|
||||
width: 800,
|
||||
height: 600,
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, 'preload.js')
|
||||
}
|
||||
})
|
||||
|
||||
// and load the index.html of the app.
|
||||
mainWindow.loadFile('index.html')
|
||||
|
||||
// Open the DevTools.
|
||||
// mainWindow.webContents.openDevTools()
|
||||
}
|
||||
|
||||
// This method will be called when Electron has finished
|
||||
// initialization and is ready to create browser windows.
|
||||
// Some APIs can only be used after this event occurs.
|
||||
app.whenReady().then(() => {
|
||||
createWindow()
|
||||
|
||||
app.on('activate', function () {
|
||||
// On macOS it's common to re-create a window in the app when the
|
||||
// dock icon is clicked and there are no other windows open.
|
||||
if (BrowserWindow.getAllWindows().length === 0) createWindow()
|
||||
})
|
||||
})
|
||||
|
||||
// Quit when all windows are closed, except on macOS. There, it's common
|
||||
// for applications and their menu bar to stay active until the user quits
|
||||
// explicitly with Cmd + Q.
|
||||
app.on('window-all-closed', function () {
|
||||
if (process.platform !== 'darwin') app.quit()
|
||||
})
|
||||
|
||||
// In this file you can include the rest of your app's specific main process
|
||||
// code. You can also put them in separate files and require them here.
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "chatgpt",
|
||||
"version": "1.0.0",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
"dev": "electron .",
|
||||
"build": "electron-builder"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^23.1.0",
|
||||
"electron-builder": "^23.6.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* The preload script runs before. It has access to web APIs
|
||||
* as well as Electron's renderer process modules and some
|
||||
* polyfilled Node.js functions.
|
||||
*
|
||||
* https://www.electronjs.org/docs/latest/tutorial/sandbox
|
||||
*/
|
||||
window.addEventListener('DOMContentLoaded', () => {
|
||||
const replaceText = (selector, text) => {
|
||||
const element = document.getElementById(selector)
|
||||
if (element) element.innerText = text
|
||||
}
|
||||
|
||||
for (const type of ['chrome', 'node', 'electron']) {
|
||||
replaceText(`${type}-version`, process.versions[type])
|
||||
}
|
||||
})
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
@@ -0,0 +1,13 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" href="/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>HuggingGPT</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "vue3-ts-vite-router-tailwindcss",
|
||||
"version": "0.0.0",
|
||||
"main": "index.html",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "run-p type-check build-only",
|
||||
"preview": "vite preview --port 4173",
|
||||
"build-only": "vite build",
|
||||
"type-check": "vue-tsc --noEmit",
|
||||
"e:dev": "yarn build && cp -r electron/. dist && cd dist && yarn && yarn dev",
|
||||
"e:build": "yarn build && cp -r electron/. dist && cd dist && yarn && yarn build"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.3.4",
|
||||
"vue": "^3.2.38",
|
||||
"vue-router": "^4.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^16.11.56",
|
||||
"@vitejs/plugin-vue": "^3.0.3",
|
||||
"@vue/tsconfig": "^0.1.3",
|
||||
"autoprefixer": "^10.4.12",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"postcss": "^8.4.18",
|
||||
"tailwindcss": "^3.2.1",
|
||||
"typescript": "~4.7.4",
|
||||
"vite": "^3.2.5",
|
||||
"vue-tsc": "^0.40.7"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
module.exports = {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg t="1680067212539" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3600" width="200" height="200"><path d="M682.666667 384V298.666667h-170.666667v234.666666c-17.92-13.226667-39.68-21.333333-64-21.333333A106.666667 106.666667 0 0 0 341.333333 618.666667a106.666667 106.666667 0 0 0 106.666667 106.666666 106.666667 106.666667 0 0 0 106.666667-106.666666V384h128m-170.666667-298.666667a426.666667 426.666667 0 0 1 426.666667 426.666667 426.666667 426.666667 0 0 1-426.666667 426.666667A426.666667 426.666667 0 0 1 85.333333 512 426.666667 426.666667 0 0 1 512 85.333333z" fill="#EF5350" p-id="3601"></path></svg>
|
||||
|
After Width: | Height: | Size: 658 B |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
@@ -0,0 +1 @@
|
||||
<svg t="1680064882350" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5287" width="200" height="200"><path d="M512 653.784194c-52.199845 0-94.525873 42.319874-94.525873 94.51818 0 52.212152 42.326028 94.525873 94.525873 94.525873 52.198306 0 94.51818-42.31372 94.51818-94.525873C606.51818 696.104068 564.198306 653.784194 512 653.784194L512 653.784194zM976.734002 929.474143 808.219119 929.474143c130.533458-92.787416 215.780897-245.131578 215.780897-417.474143 0-282.763774-229.236241-512.000015-512.000015-512.000015S-1.5e-05 229.236226-1.5e-05 512 229.236226 1024.000015 512 1024.000015l0.035385 0 464.698618 0c26.106076 0 47.266013-21.159937 47.266013-47.266013C1024.000015 950.63408 1002.840078 929.474143 976.734002 929.474143L976.734002 929.474143zM512.035385 929.474143l-0.035385 0-0.018461 0C281.423763 929.467989 94.524319 742.56393 94.524319 512s186.911752-417.475681 417.475681-417.475681S929.474143 281.43607 929.474143 512C929.474143 742.545468 742.588545 929.451066 512.035385 929.474143L512.035385 929.474143zM275.689934 417.474127c-52.198306 0-94.51818 42.326028-94.51818 94.525873 0 52.198306 42.31372 94.51818 94.51818 94.51818 52.212152 0 94.525873-42.319874 94.525873-94.51818C370.215806 459.800155 327.889778 417.474127 275.689934 417.474127L275.689934 417.474127zM551.379883 519.878438c0-21.749166-17.626101-39.387575-39.379883-39.387575-21.749166 0-39.387575 17.638409-39.387575 39.387575 0 21.743012 17.638409 39.381421 39.387575 39.381421C533.749166 559.259859 551.379883 541.62145 551.379883 519.878438L551.379883 519.878438zM740.43009 417.474127c-52.198306 0-94.524334 42.326028-94.524334 94.525873 0 52.198306 42.31372 94.51818 94.524334 94.51818 52.205999 0 94.519719-42.319874 94.519719-94.51818C834.949809 459.800155 792.629934 417.474127 740.43009 417.474127L740.43009 417.474127zM512 181.171753c-52.199845 0-94.525873 42.319874-94.525873 94.51818 0 52.212152 42.326028 94.525873 94.525873 94.525873 52.198306 0 94.51818-42.31372 94.51818-94.525873C606.51818 223.491627 564.198306 181.171753 512 181.171753L512 181.171753zM512 181.171753" fill="#d81e06" p-id="5288"></path></svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Suspense>
|
||||
<RouterView />
|
||||
</Suspense>
|
||||
</template>
|
||||
|
||||
<style></style>
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { CleanChatMessage } from "@/types";
|
||||
import axios, { AxiosError } from "axios";
|
||||
import { CHAT_GPT_URL, CHAT_GPT_LLM } from "@/config";
|
||||
|
||||
axios.defaults.headers.post["Content-Type"] = "application/json";
|
||||
|
||||
export async function chatgpt(messageList: CleanChatMessage[], apiKey: string) {
|
||||
var endpoint = `${CHAT_GPT_URL}/v1/chat/completions`
|
||||
|
||||
try {
|
||||
const completion = await axios({
|
||||
url: endpoint,
|
||||
method: "post",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
data: {
|
||||
model: CHAT_GPT_LLM,
|
||||
messages: messageList
|
||||
},
|
||||
timeout: 60000, // 180 seconds
|
||||
});
|
||||
return {
|
||||
status: "success",
|
||||
data: completion.data.choices[0].message.content,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
status: "error",
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { CleanChatMessage } from "@/types";
|
||||
import axios, { AxiosError } from "axios";
|
||||
import { HUGGINGGPT_BASE_URL } from "@/config";
|
||||
|
||||
const model = "gpt-3.5-turbo";
|
||||
|
||||
axios.defaults.headers.post["Content-Type"] = "application/json";
|
||||
|
||||
export async function hugginggpt(messageList: CleanChatMessage[]) {
|
||||
var endpoint = `${HUGGINGGPT_BASE_URL}/hugginggpt`
|
||||
try {
|
||||
const response = await axios({
|
||||
url: endpoint,
|
||||
method: "post",
|
||||
data: {
|
||||
model,
|
||||
messages: messageList.slice(1)
|
||||
},
|
||||
timeout: 180000, // 180 seconds
|
||||
});
|
||||
return {
|
||||
status: "success",
|
||||
data: response.data.message,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
status: "error",
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg t="1678698788188" class="icon" viewBox="0 0 1215 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="9770" width="200" height="200"><path d="M577.044084 1023.99936a43.31985 43.31985 0 0 1-43.319851-43.31985V43.511814a43.31985 43.31985 0 0 1 86.639701 0v937.167696a43.31985 43.31985 0 0 1-43.31985 43.31985z m-266.829803-213.399854a43.31985 43.31985 0 0 1-43.383838-43.255863V231.060532a43.31985 43.31985 0 0 1 86.639701 0v536.219124a43.31985 43.31985 0 0 1-43.319851 43.31985z m548.312849 0a43.31985 43.31985 0 0 1-43.319851-43.255863V231.060532a43.31985 43.31985 0 0 1 86.639701 0v536.219124a43.31985 43.31985 0 0 1-43.31985 43.31985zM43.32049 643.526936A43.31985 43.31985 0 0 1 0.00064 600.207086V454.69846a43.31985 43.31985 0 0 1 86.639701 0v145.508626a43.31985 43.31985 0 0 1-43.319851 43.31985z m1073.718007 0a43.31985 43.31985 0 0 1-43.31985-43.31985V454.69846a43.31985 43.31985 0 0 1 86.575713 0v145.508626a43.31985 43.31985 0 0 1-43.255863 43.31985z" fill="#d81e06" p-id="9771"></path></svg>
|
||||
|
After Width: | Height: | Size: 1014 B |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 5.9 KiB |
@@ -0,0 +1,3 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M18.375 12.739l-7.693 7.693a4.5 4.5 0 01-6.364-6.364l10.94-10.94A3 3 0 1119.5 7.372L8.552 18.32m.009-.01l-.01.01m5.699-9.941l-7.81 7.81a1.5 1.5 0 002.112 2.13" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 358 B |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
@@ -0,0 +1,73 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg
|
||||
width="95"
|
||||
height="88"
|
||||
viewBox="0 0 95 88"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path
|
||||
d="M47.2119 76.5C66.4037 76.5 81.9619 60.9419 81.9619 41.75C81.9619 22.5581 66.4037 7 47.2119 7C28.02 7 12.4619 22.5581 12.4619 41.75C12.4619 60.9419 28.02 76.5 47.2119 76.5Z"
|
||||
fill="#FFD21E"
|
||||
/>
|
||||
<path
|
||||
d="M81.9619 41.75C81.9619 22.5581 66.4037 7 47.2119 7C28.02 7 12.4619 22.5581 12.4619 41.75C12.4619 60.9419 28.02 76.5 47.2119 76.5C66.4037 76.5 81.9619 60.9419 81.9619 41.75ZM8.46185 41.75C8.46185 20.349 25.8108 3 47.2119 3C68.6129 3 85.9619 20.349 85.9619 41.75C85.9619 63.151 68.6129 80.5 47.2119 80.5C25.8108 80.5 8.46185 63.151 8.46185 41.75Z"
|
||||
fill="#FF9D0B"
|
||||
/>
|
||||
<path
|
||||
d="M58.5024 32.2915C59.7768 32.7415 60.2839 35.3615 61.5713 34.6769C64.0095 33.3805 64.9351 30.353 63.6387 27.9148C62.3423 25.4767 59.3148 24.5511 56.8766 25.8475C54.4384 27.1439 53.5128 30.1714 54.8092 32.6096C55.4211 33.7604 57.3632 31.8892 58.5024 32.2915Z"
|
||||
fill="#3A3B45"
|
||||
/>
|
||||
<path
|
||||
d="M34.9454 32.2915C33.671 32.7415 33.164 35.3615 31.8766 34.6769C29.4384 33.3805 28.5128 30.353 29.8092 27.9148C31.1056 25.4767 34.1331 24.5511 36.5713 25.8475C39.0095 27.1439 39.9351 30.1714 38.6387 32.6096C38.0268 33.7604 36.0846 31.8892 34.9454 32.2915Z"
|
||||
fill="#3A3B45"
|
||||
/>
|
||||
<path
|
||||
d="M46.9619 56.289C56.7903 56.289 59.9619 47.5261 59.9619 43.0262C59.9619 40.6875 58.3898 41.4236 55.8718 42.6702C53.5449 43.8222 50.4102 45.4101 46.9619 45.4101C39.7822 45.4101 33.9619 38.5263 33.9619 43.0262C33.9619 47.5261 37.1334 56.289 46.9619 56.289Z"
|
||||
fill="#3A3B45"
|
||||
/>
|
||||
<mask
|
||||
id="mask0"
|
||||
mask-type="alpha"
|
||||
maskUnits="userSpaceOnUse"
|
||||
x="33"
|
||||
y="41"
|
||||
width="27"
|
||||
height="16"
|
||||
>
|
||||
<path
|
||||
d="M46.9619 56.289C56.7903 56.289 59.9619 47.5261 59.9619 43.0262C59.9619 40.6875 58.3898 41.4236 55.8718 42.6702C53.5449 43.8222 50.4102 45.4101 46.9619 45.4101C39.7822 45.4101 33.9619 38.5263 33.9619 43.0262C33.9619 47.5261 37.1334 56.289 46.9619 56.289Z"
|
||||
fill="white"
|
||||
/>
|
||||
</mask>
|
||||
<g mask="url(#mask0)">
|
||||
<path
|
||||
d="M47.2119 66.5C52.0018 66.5 55.8848 62.617 55.8848 57.8271C55.8848 54.0962 53.5291 50.9156 50.224 49.6915C50.1023 49.6464 49.9794 49.604 49.8553 49.5643C49.0219 49.2979 48.1337 52.1623 47.2119 52.1623C46.3506 52.1623 45.5186 49.2797 44.7332 49.5135C41.151 50.5799 38.5389 53.8984 38.5389 57.8271C38.5389 62.617 42.4219 66.5 47.2119 66.5Z"
|
||||
fill="#F94040"
|
||||
/>
|
||||
</g>
|
||||
<path
|
||||
d="M70.7119 37C72.5068 37 73.9619 35.5449 73.9619 33.75C73.9619 31.9551 72.5068 30.5 70.7119 30.5C68.9169 30.5 67.4619 31.9551 67.4619 33.75C67.4619 35.5449 68.9169 37 70.7119 37Z"
|
||||
fill="#FF9D0B"
|
||||
/>
|
||||
<path
|
||||
d="M24.2119 37C26.0068 37 27.4619 35.5449 27.4619 33.75C27.4619 31.9551 26.0068 30.5 24.2119 30.5C22.4169 30.5 20.9619 31.9551 20.9619 33.75C20.9619 35.5449 22.4169 37 24.2119 37Z"
|
||||
fill="#FF9D0B"
|
||||
/>
|
||||
<path
|
||||
d="M17.5238 48C15.9048 48 14.4578 48.665 13.4488 49.871C12.8248 50.618 12.1728 51.822 12.1198 53.625C11.4408 53.43 10.7878 53.321 10.1778 53.321C8.6278 53.321 7.2278 53.915 6.2378 54.994C4.9658 56.379 4.4008 58.081 4.6468 59.784C4.7638 60.595 5.0348 61.322 5.4398 61.995C4.5858 62.686 3.9568 63.648 3.6528 64.805C3.4148 65.712 3.1708 67.601 4.4448 69.547C4.3638 69.674 4.2878 69.806 4.2168 69.941C3.4508 71.395 3.4018 73.038 4.0778 74.568C5.1028 76.887 7.6498 78.714 12.5958 80.675C15.6728 81.895 18.4878 82.675 18.5128 82.682C22.5808 83.737 26.2598 84.273 29.4448 84.273C35.2988 84.273 39.4898 82.48 41.9018 78.944C45.7838 73.25 45.2288 68.042 40.2058 63.022C37.4258 60.244 35.5778 56.148 35.1928 55.249C34.4168 52.587 32.3648 49.628 28.9538 49.628H28.9528C28.6658 49.628 28.3758 49.651 28.0898 49.696C26.5958 49.931 25.2898 50.791 24.3568 52.085C23.3498 50.833 22.3718 49.837 21.4868 49.275C20.1528 48.429 18.8198 48 17.5238 48ZM17.5238 52C18.0338 52 18.6568 52.217 19.3438 52.653C21.4768 54.006 25.5928 61.081 27.0998 63.833C27.6048 64.755 28.4678 65.145 29.2448 65.145C30.7868 65.145 31.9908 63.612 29.3858 61.664C25.4688 58.733 26.8428 53.942 28.7128 53.647C28.7948 53.634 28.8758 53.628 28.9538 53.628C30.6538 53.628 31.4038 56.558 31.4038 56.558C31.4038 56.558 33.6018 62.078 37.3778 65.851C41.1538 69.625 41.3488 72.654 38.5968 76.69C36.7198 79.442 33.1268 80.273 29.4448 80.273C25.6258 80.273 21.7108 79.379 19.5168 78.81C19.4088 78.782 6.0658 75.013 7.7558 71.805C8.0398 71.266 8.5078 71.05 9.0968 71.05C11.4768 71.05 15.8058 74.592 17.6668 74.592C18.0828 74.592 18.3758 74.415 18.4958 73.983C19.2888 71.138 6.4388 69.942 7.5218 65.821C7.7128 65.092 8.2308 64.796 8.9588 64.797C12.1038 64.797 19.1598 70.328 20.6388 70.328C20.7518 70.328 20.8328 70.295 20.8768 70.225C21.6178 69.029 21.2118 68.194 15.9888 65.033C10.7658 61.871 7.0998 59.969 9.1848 57.699C9.4248 57.437 9.7648 57.321 10.1778 57.321C13.3488 57.322 20.8408 64.14 20.8408 64.14C20.8408 64.14 22.8628 66.243 24.0858 66.243C24.3668 66.243 24.6058 66.132 24.7678 65.858C25.6348 64.396 16.7148 57.636 16.2118 54.847C15.8708 52.957 16.4508 52 17.5238 52Z"
|
||||
fill="#FF9D0B"
|
||||
/>
|
||||
<path
|
||||
d="M38.5967 76.6898C41.3487 72.6538 41.1537 69.6248 37.3777 65.8508C33.6017 62.0778 31.4037 56.5578 31.4037 56.5578C31.4037 56.5578 30.5827 53.3518 28.7127 53.6468C26.8427 53.9418 25.4697 58.7328 29.3867 61.6638C33.3037 64.5938 28.6067 66.5848 27.0997 63.8328C25.5927 61.0808 21.4777 54.0058 19.3437 52.6528C17.2107 51.2998 15.7087 52.0578 16.2117 54.8468C16.7147 57.6358 25.6357 64.3958 24.7677 65.8588C23.8997 67.3208 20.8407 64.1398 20.8407 64.1398C20.8407 64.1398 11.2687 55.4288 9.18465 57.6988C7.10065 59.9688 10.7657 61.8708 15.9887 65.0328C21.2127 68.1938 21.6177 69.0288 20.8767 70.2248C20.1347 71.4208 8.60465 61.6998 7.52165 65.8208C6.43965 69.9418 19.2887 71.1378 18.4957 73.9828C17.7027 76.8288 9.44465 68.5978 7.75565 71.8048C6.06565 75.0128 19.4087 78.7818 19.5167 78.8098C23.8267 79.9278 34.7727 82.2968 38.5967 76.6898Z"
|
||||
fill="#FFD21E"
|
||||
/>
|
||||
<path
|
||||
d="M77.3999 48C79.0189 48 80.4659 48.665 81.4749 49.871C82.0989 50.618 82.7509 51.822 82.8039 53.625C83.4829 53.43 84.1359 53.321 84.7459 53.321C86.2959 53.321 87.6959 53.915 88.6859 54.994C89.9579 56.379 90.5229 58.081 90.2769 59.784C90.1599 60.595 89.8889 61.322 89.4839 61.995C90.3379 62.686 90.9669 63.648 91.2709 64.805C91.5089 65.712 91.7529 67.601 90.4789 69.547C90.5599 69.674 90.6359 69.806 90.7069 69.941C91.4729 71.395 91.5219 73.038 90.8459 74.568C89.8209 76.887 87.2739 78.714 82.3279 80.675C79.2509 81.895 76.4359 82.675 76.4109 82.682C72.3429 83.737 68.6639 84.273 65.4789 84.273C59.6249 84.273 55.4339 82.48 53.0219 78.944C49.1399 73.25 49.6949 68.042 54.7179 63.022C57.4979 60.244 59.3459 56.148 59.7309 55.249C60.5069 52.587 62.5589 49.628 65.9699 49.628H65.9709C66.2579 49.628 66.5479 49.651 66.8339 49.696C68.3279 49.931 69.6339 50.791 70.5669 52.085C71.5739 50.833 72.5519 49.837 73.4369 49.275C74.7709 48.429 76.1039 48 77.3999 48ZM77.3999 52C76.8899 52 76.2669 52.217 75.5799 52.653C73.4469 54.006 69.3309 61.081 67.8239 63.833C67.3189 64.755 66.4559 65.145 65.6789 65.145C64.1369 65.145 62.9329 63.612 65.5379 61.664C69.4549 58.733 68.0809 53.942 66.2109 53.647C66.1289 53.634 66.0479 53.628 65.9699 53.628C64.2699 53.628 63.5199 56.558 63.5199 56.558C63.5199 56.558 61.3219 62.078 57.5459 65.851C53.7699 69.625 53.5749 72.654 56.3269 76.69C58.2039 79.442 61.7969 80.273 65.4789 80.273C69.2979 80.273 73.2129 79.379 75.4069 78.81C75.5149 78.782 88.8579 75.013 87.1679 71.805C86.8839 71.266 86.4159 71.05 85.8269 71.05C83.4469 71.05 79.1179 74.592 77.2569 74.592C76.8409 74.592 76.5479 74.415 76.4279 73.983C75.6349 71.138 88.4849 69.942 87.4019 65.821C87.2109 65.092 86.6929 64.796 85.9649 64.797C82.8199 64.797 75.7639 70.328 74.2849 70.328C74.1719 70.328 74.0909 70.295 74.0469 70.225C73.3059 69.029 73.7119 68.194 78.9349 65.033C84.1579 61.871 87.8239 59.969 85.7389 57.699C85.4989 57.437 85.1589 57.321 84.7459 57.321C81.5749 57.322 74.0829 64.14 74.0829 64.14C74.0829 64.14 72.0609 66.243 70.8379 66.243C70.5569 66.243 70.3179 66.132 70.1559 65.858C69.2889 64.396 78.2089 57.636 78.7119 54.847C79.0529 52.957 78.4729 52 77.3999 52Z"
|
||||
fill="#FF9D0B"
|
||||
/>
|
||||
<path
|
||||
d="M56.3271 76.6898C53.5751 72.6538 53.7701 69.6248 57.5461 65.8508C61.3221 62.0778 63.5201 56.5578 63.5201 56.5578C63.5201 56.5578 64.3411 53.3518 66.2111 53.6468C68.0811 53.9418 69.4541 58.7328 65.5371 61.6638C61.6201 64.5938 66.3171 66.5848 67.8241 63.8328C69.3311 61.0808 73.4461 54.0058 75.5801 52.6528C77.7131 51.2998 79.2151 52.0578 78.7121 54.8468C78.2091 57.6358 69.2881 64.3958 70.1561 65.8588C71.0241 67.3208 74.0831 64.1398 74.0831 64.1398C74.0831 64.1398 83.6551 55.4288 85.7391 57.6988C87.8231 59.9688 84.1581 61.8708 78.9351 65.0328C73.7111 68.1938 73.3061 69.0288 74.0471 70.2248C74.7891 71.4208 86.3191 61.6998 87.4021 65.8208C88.4841 69.9418 75.6351 71.1378 76.4281 73.9828C77.2211 76.8288 85.4791 68.5978 87.1681 71.8048C88.8581 75.0128 75.5151 78.7818 75.4071 78.8098C71.0971 79.9278 60.1511 82.2968 56.3271 76.6898Z"
|
||||
fill="#FFD21E"
|
||||
/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 8.7 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 10 KiB |
@@ -0,0 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M10.343 3.94c.09-.542.56-.94 1.11-.94h1.093c.55 0 1.02.398 1.11.94l.149.894c.07.424.384.764.78.93.398.164.855.142 1.205-.108l.737-.527a1.125 1.125 0 011.45.12l.773.774c.39.389.44 1.002.12 1.45l-.527.737c-.25.35-.272.806-.107 1.204.165.397.505.71.93.78l.893.15c.543.09.94.56.94 1.109v1.094c0 .55-.397 1.02-.94 1.11l-.893.149c-.425.07-.765.383-.93.78-.165.398-.143.854.107 1.204l.527.738c.32.447.269 1.06-.12 1.45l-.774.773a1.125 1.125 0 01-1.449.12l-.738-.527c-.35-.25-.806-.272-1.203-.107-.397.165-.71.505-.781.929l-.149.894c-.09.542-.56.94-1.11.94h-1.094c-.55 0-1.019-.398-1.11-.94l-.148-.894c-.071-.424-.384-.764-.781-.93-.398-.164-.854-.142-1.204.108l-.738.527c-.447.32-1.06.269-1.45-.12l-.773-.774a1.125 1.125 0 01-.12-1.45l.527-.737c.25-.35.273-.806.108-1.204-.165-.397-.505-.71-.93-.78l-.894-.15c-.542-.09-.94-.56-.94-1.109v-1.094c0-.55.398-1.02.94-1.11l.894-.149c.424-.07.765-.383.93-.78.165-.398.143-.854-.107-1.204l-.527-.738a1.125 1.125 0 01.12-1.45l.773-.773a1.125 1.125 0 011.45-.12l.737.527c.35.25.807.272 1.204.107.397-.165.71-.505.78-.929l.15-.894z" />
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1 @@
|
||||
<svg t="1681790041175" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6311" width="200" height="200"><path d="M100.571429 438.857143V365.641143h652.288L538.477714 143.36 588.617143 91.428571 923.428571 438.857143zM100.571429 566.857143h822.857142v73.216H270.811429l214.345142 222.244571L435.382857 914.285714z" fill="#181818" p-id="6312"></path></svg>
|
||||
|
After Width: | Height: | Size: 396 B |
@@ -0,0 +1,12 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
@layer components {
|
||||
.btn {
|
||||
@apply px-4 py-2 text-sm font-medium tracking-wide text-white capitalize transition-colors duration-300 transform bg-blue-700 rounded-md hover:bg-blue-600 focus:outline-none focus:bg-blue-600 whitespace-nowrap disabled:bg-blue-300;
|
||||
}
|
||||
.input {
|
||||
@apply px-4 py-2 text-gray-700 bg-white border rounded-md mr-2 sm:mr-4 focus:border-blue-400 focus:outline-none focus:ring focus:ring-blue-300 focus:ring-opacity-40 flex-grow;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
<template>
|
||||
<div class="com__box">
|
||||
<div class="loading">
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div></div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.loading,
|
||||
.loading > div {
|
||||
position: relative;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.loading {
|
||||
display: block;
|
||||
font-size: 0;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.loading.la-dark {
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.loading > div {
|
||||
display: inline-block;
|
||||
float: none;
|
||||
background-color: currentColor;
|
||||
border: 0 solid currentColor;
|
||||
}
|
||||
|
||||
.loading {
|
||||
width: 17px;
|
||||
height: 17px;
|
||||
}
|
||||
|
||||
.loading > div {
|
||||
width: 3px;
|
||||
height: 3px;
|
||||
margin: 1px;
|
||||
border-radius: 100%;
|
||||
animation-name: ball-grid-beat;
|
||||
animation-iteration-count: infinite;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(1) {
|
||||
animation-duration: 0.65s;
|
||||
animation-delay: 0.03s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(2) {
|
||||
animation-duration: 1.02s;
|
||||
animation-delay: 0.09s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(3) {
|
||||
animation-duration: 1.06s;
|
||||
animation-delay: -0.69s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(4) {
|
||||
animation-duration: 1.5s;
|
||||
animation-delay: -0.41s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(5) {
|
||||
animation-duration: 1.6s;
|
||||
animation-delay: 0.04s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(6) {
|
||||
animation-duration: 0.84s;
|
||||
animation-delay: 0.07s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(7) {
|
||||
animation-duration: 0.68s;
|
||||
animation-delay: -0.66s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(8) {
|
||||
animation-duration: 0.93s;
|
||||
animation-delay: -0.76s;
|
||||
}
|
||||
|
||||
.loading > div:nth-child(9) {
|
||||
animation-duration: 1.24s;
|
||||
animation-delay: -0.76s;
|
||||
}
|
||||
|
||||
@keyframes ball-grid-beat {
|
||||
0% {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
50% {
|
||||
opacity: 0.35;
|
||||
}
|
||||
|
||||
100% {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,10 @@
|
||||
const HUGGINGGPT_BASE_URL = "http://localhost:8004"
|
||||
|
||||
// use ChatGPT: double click on the setting icon
|
||||
const CHAT_GPT_URL = "https://api.openai.com"
|
||||
const CHAT_GPT_LLM = "gpt-3.5-turbo" // gpt-3.5-turbo, gpt-4
|
||||
// Dev: local endpoint
|
||||
// const CHAT_GPT_URL = "http://localhost:8006"
|
||||
|
||||
|
||||
export {HUGGINGGPT_BASE_URL, CHAT_GPT_URL, CHAT_GPT_LLM}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createApp } from "vue";
|
||||
import App from "./App.vue";
|
||||
import router from "./router";
|
||||
import "./assets/tailwind.css";
|
||||
|
||||
const app = createApp(App);
|
||||
|
||||
app.use(router).mount("#app");
|
||||
@@ -0,0 +1,98 @@
|
||||
import type {Collection, ChatMessage } from "@/types";
|
||||
|
||||
const ChatGPTTerminalMessage:ChatMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Hi there! I am OpenAI ChatGPT, an AI assistant for you. How can I help you? ",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "I want you to act as a linux terminal. I will type commands and you will reply with what the terminal should show. I want you to only reply with the terminal output inside one unique code block, and nothing else. do not write explanations. do not type commands unless I instruct you to do so. When I need to tell you something in English, I will do so by putting text inside curly brackets {like this}.",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Yes, I will do it for you. Please type the command and I will reply with the terminal output.",
|
||||
type: "text",
|
||||
first: true
|
||||
}
|
||||
]
|
||||
|
||||
const ChatGPTPolishMessage:ChatMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Hi there! I am OpenAI ChatGPT, an AI assistant for you. How can I help you? ",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "You are a well-trained AI writing assistant with expertise in writing academic papers for computer conferences. By giving you a draft paragraph, I hope you can help me polish my writing with your knowledge. The language should be concise and consistent with the style of an academic paper.",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "No problem, I will think carefully and polish the paper for you.",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
]
|
||||
|
||||
const ChatGPTTranslationMessage:ChatMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Hi there! I am OpenAI ChatGPT, an AI assistant for you. How can I help you? ",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "I want you to act as an English translator, spelling corrector and improver. I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. Keep the meaning same, but make them more literary. I want you to only reply the correction, the improvements and nothing else, do not write explanations.",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Sure, I will act as an English translator and improver.",
|
||||
type: "text",
|
||||
first: true
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
const defaultChatGPTMessage:ChatMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Hi there! I am OpenAI ChatGPT, an AI assistant for you. How can I help you? ",
|
||||
type: "text",
|
||||
first: true
|
||||
}
|
||||
]
|
||||
|
||||
const defaultHuggingGPTMessage:ChatMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Hi there, I am HuggingGPT empowered by Huggingface family! Yes, I can provide thousands of models for dozens of tasks. For more fun and creativity, I have invited Diffusers family to join our team. Feel free to experience it!",
|
||||
type: "text",
|
||||
first: true
|
||||
}
|
||||
]
|
||||
|
||||
const promptCollection: Collection = {
|
||||
"chatgpt": {
|
||||
"terminal": ChatGPTTerminalMessage,
|
||||
"polish": ChatGPTPolishMessage,
|
||||
"translation": ChatGPTTranslationMessage,
|
||||
"default": defaultChatGPTMessage,
|
||||
},
|
||||
"hugginggpt": {
|
||||
"default": defaultHuggingGPTMessage
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export default promptCollection
|
||||
@@ -0,0 +1,14 @@
|
||||
import { createRouter, createWebHashHistory } from "vue-router";
|
||||
|
||||
const router = createRouter({
|
||||
history: createWebHashHistory(import.meta.env.BASE_URL),
|
||||
routes: [
|
||||
{
|
||||
path: "/",
|
||||
name: "home",
|
||||
component: () => import("@/views/home.vue"),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,20 @@
|
||||
export interface ChatMessage {
|
||||
role: "user" | "assistant" | "system";
|
||||
type: "text" | "image" | "audio" | "video" | "code";
|
||||
first: boolean;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface CleanChatMessage {
|
||||
role: "user" | "assistant" | "system";
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface Collection {
|
||||
chatgpt: {
|
||||
[key: string]: ChatMessage[];
|
||||
};
|
||||
hugginggpt: {
|
||||
[key: string]: ChatMessage[];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
<script setup lang="ts">
|
||||
import type { ChatMessage, CleanChatMessage } from "@/types";
|
||||
import { ref, watch, nextTick, onMounted, computed } from "vue";
|
||||
import { RouterLink } from "vue-router";
|
||||
import { hugginggpt } from "@/api/hugginggpt";
|
||||
import { chatgpt } from "@/api/chatgpt";
|
||||
import Loading from "@/components/Loading.vue";
|
||||
import promptCollection from "@/prompt";
|
||||
import { HUGGINGGPT_BASE_URL } from "@/config";
|
||||
|
||||
let isChatgpt = ref(false);
|
||||
let isTalking = ref(false);
|
||||
let isConfig = ref<boolean>(true);
|
||||
let title = ref<string>();
|
||||
let mode = ref<string>("default");
|
||||
|
||||
title.value = isChatgpt.value? "ChatGPT": "HuggingGPT";
|
||||
|
||||
isConfig.value = (isChatgpt.value)? true : false
|
||||
|
||||
const chatListDom = ref<HTMLDivElement>();
|
||||
// const pdf = ref<HTMLDivElement>();
|
||||
let messageContent = ref("");
|
||||
|
||||
const roleAliasChatHuggingGPT = { user: "Human", assistant: "HuggingGPT", system: "System" };
|
||||
const roleAliasChatGPT = { user: "Human", assistant: "ChatGPT", system: "System" };
|
||||
const roleAlias = ref(isChatgpt.value? roleAliasChatGPT: roleAliasChatHuggingGPT);
|
||||
const messageList = ref<ChatMessage[]>(isChatgpt.value? promptCollection["chatgpt"][mode.value]: promptCollection["hugginggpt"][mode.value]);
|
||||
|
||||
onMounted(() => {
|
||||
const apiKey = loadConfig();
|
||||
if (apiKey) {
|
||||
// switchConfigStatus(); //close
|
||||
isConfig.value = false
|
||||
}
|
||||
});
|
||||
|
||||
async function sendChatMessage() {
|
||||
isTalking.value = true;
|
||||
const input = messageContent.value
|
||||
messageList.value.push(
|
||||
{ role: "user", content: input, type: "text", first: true},
|
||||
)
|
||||
|
||||
clearMessageContent();
|
||||
var clean_messages: CleanChatMessage[] = []
|
||||
for (let message of messageList.value) {
|
||||
if (message.first && message.role != "system") {
|
||||
clean_messages.push({role: message.role, content: message.content})
|
||||
}
|
||||
}
|
||||
messageList.value.push(
|
||||
{ role: "assistant", content: "", type: "text", first: true},
|
||||
)
|
||||
if (isChatgpt.value) {
|
||||
var { status, data, message } = await chatgpt(clean_messages, loadConfig());
|
||||
} else {
|
||||
var { status, data, message } = await hugginggpt(clean_messages);
|
||||
}
|
||||
|
||||
messageList.value.pop()
|
||||
if (status === "success" ) {
|
||||
if (data) {
|
||||
messageList.value.push(
|
||||
{ role: "assistant", content: data, type: "text", first: true }
|
||||
);
|
||||
} else {
|
||||
messageList.value.push(
|
||||
{ role: "assistant", content: "empty content", type: "text", first: true }
|
||||
);
|
||||
}
|
||||
} else {
|
||||
messageList.value.push(
|
||||
{ role: "system", content: message, type: "text", first: true }
|
||||
);
|
||||
}
|
||||
isTalking.value = false;
|
||||
}
|
||||
|
||||
|
||||
const messageListMM = computed(() => {
|
||||
var messageListMM: ChatMessage[] = []
|
||||
for (var i = 0; i < messageList.value.length; i++) {
|
||||
var message = messageList.value[i]
|
||||
if (message.type != "text") {
|
||||
messageListMM.push(message)
|
||||
continue
|
||||
}
|
||||
var content = message.content
|
||||
var role = message.role
|
||||
|
||||
var image_urls = content.match(/(http(s?):|\/)([/|.|\S||\w|:|-])*?\.(?:jpg|jpeg|tiff|gif|png)/g)
|
||||
var image_reg = new RegExp(/(http(s?):|\/)([/|.|\S|\w|:|-])*?\.(?:jpg|jpeg|tiff|gif|png)/g)
|
||||
|
||||
var orig_content = content
|
||||
var seq_added_accum = 0
|
||||
if (image_urls){
|
||||
for (var j = 0; j < image_urls.length; j++) {
|
||||
// @ts-ignore
|
||||
var start = image_reg.exec(orig_content).index
|
||||
var end = start + image_urls[j].length
|
||||
start += seq_added_accum
|
||||
end += seq_added_accum
|
||||
const replace_str = `<span class="inline-flex items-baseline">
|
||||
<a class="inline-flex text-sky-800 font-bold items-baseline" target="_blank" href="${image_urls[j].startsWith("http")?image_urls[j]:HUGGINGGPT_BASE_URL+image_urls[j]}">
|
||||
<img src="${image_urls[j].startsWith("http")?image_urls[j]:HUGGINGGPT_BASE_URL+image_urls[j]}" alt="" class="inline-flex self-center w-5 h-5 rounded-full mx-1" />
|
||||
<span class="mx-1">[Image]</span>
|
||||
</a>
|
||||
</span>`
|
||||
const rep_length = replace_str.length
|
||||
seq_added_accum += (rep_length - image_urls[j].length)
|
||||
content = content.slice(0, start) + replace_str + content.slice(end)
|
||||
|
||||
if(!image_urls[j].startsWith("http")){
|
||||
image_urls[j] = HUGGINGGPT_BASE_URL + image_urls[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
orig_content = content
|
||||
var audio_urls = content.match(/(http(s?):|\/)([/|.|\w|\S|:|-])*?\.(?:flac|wav)/g)
|
||||
var audio_reg = new RegExp(/(http(s?):|\/)([/|.|\w|\S|:|-])*?\.(?:flac|wav)/g)
|
||||
|
||||
var seq_added_accum = 0
|
||||
if (audio_urls){
|
||||
for (var j = 0; j < audio_urls.length; j++) {
|
||||
// @ts-ignore
|
||||
var start = audio_reg.exec(orig_content).index
|
||||
var end = start + audio_urls[j].length
|
||||
start += seq_added_accum
|
||||
end += seq_added_accum
|
||||
const replace_str = `<span class="inline-flex items-baseline">
|
||||
<a class="text-sky-800 inline-flex font-bold items-baseline" target="_blank" href="${audio_urls[j].startsWith("http")?audio_urls[j]:HUGGINGGPT_BASE_URL+audio_urls[j]}">
|
||||
<img class="inline-flex self-center w-5 h-5 rounded-full mx-1" src="/audio.svg"/>
|
||||
<span class="mx-1">[Audio]</span>
|
||||
</a>
|
||||
</span>`
|
||||
const rep_length = replace_str.length
|
||||
seq_added_accum += (rep_length - audio_urls[j].length)
|
||||
content = content.slice(0, start) + replace_str + content.slice(end)
|
||||
|
||||
if(!audio_urls[j].startsWith("http")){
|
||||
audio_urls[j] = HUGGINGGPT_BASE_URL + audio_urls[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
orig_content = content
|
||||
var video_urls = content.match(/(http(s?):|\/)([/|.|\w|\s|:|-])*?\.(?:mp4)/g)
|
||||
var video_reg = new RegExp(/(http(s?):|\/)([/|.|\w|\s|:|-])*?\.(?:mp4)/g)
|
||||
|
||||
var seq_added_accum = 0
|
||||
if (video_urls){
|
||||
for (var j = 0; j < video_urls.length; j++) {
|
||||
// @ts-ignore
|
||||
var start = video_reg.exec(orig_content).index
|
||||
var end = start + video_urls[j].length
|
||||
start += seq_added_accum
|
||||
end += seq_added_accum
|
||||
const replace_str = `<span class="inline-flex items-baseline">
|
||||
<a class="text-sky-800 inline-flex font-bold items-baseline" target="_blank" href="${video_urls[j].startsWith("http")?video_urls[j]:HUGGINGGPT_BASE_URL+video_urls[j]}">
|
||||
<img class="inline-flex self-center w-5 h-5 rounded-full mx-1" src="/video.svg"/>
|
||||
<span class="mx-1">[video]</span>
|
||||
</a>
|
||||
</span>`
|
||||
const rep_length = replace_str.length
|
||||
seq_added_accum += (rep_length - video_urls[j].length)
|
||||
content = content.slice(0, start) + replace_str + content.slice(end)
|
||||
|
||||
if(!video_urls[j].startsWith("http")){
|
||||
video_urls[j] = HUGGINGGPT_BASE_URL + video_urls[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
message = {role: role, content: content, type: "text", first: true}
|
||||
messageListMM.push(message)
|
||||
// de-depulicate
|
||||
// @ts-ignore
|
||||
image_urls = [...new Set(image_urls)]
|
||||
// @ts-ignore
|
||||
audio_urls = [...new Set(audio_urls)]
|
||||
// @ts-ignore
|
||||
video_urls = [...new Set(video_urls)]
|
||||
if (image_urls) {
|
||||
|
||||
for (var j = 0; j < image_urls.length; j++) {
|
||||
messageListMM.push({role: role, content: image_urls[j], type: "image", first: false})
|
||||
}
|
||||
}
|
||||
if (audio_urls) {
|
||||
for (var j = 0; j < audio_urls.length; j++) {
|
||||
messageListMM.push({role: role, content: audio_urls[j], type: "audio", first: false})
|
||||
}
|
||||
}
|
||||
if (video_urls) {
|
||||
for (var j = 0; j < video_urls.length; j++) {
|
||||
messageListMM.push({role: role, content: video_urls[j], type: "video", first: false})
|
||||
}
|
||||
}
|
||||
// if (code_blocks){
|
||||
// for (var j = 0; j < code_blocks.length; j++) {
|
||||
// messageListMM.push({role: role, content: code_blocks[j], type: "code", first: false})
|
||||
// }
|
||||
// }
|
||||
}
|
||||
// nextTick(()=>scrollToBottom())
|
||||
return messageListMM
|
||||
})
|
||||
|
||||
const sendOrSave = () => {
|
||||
if (!messageContent.value.length) return;
|
||||
if (isConfig.value) {
|
||||
if (saveConfig(messageContent.value.trim())) {
|
||||
switchConfigStatus();
|
||||
}
|
||||
clearMessageContent();
|
||||
} else {
|
||||
sendChatMessage();
|
||||
}
|
||||
};
|
||||
|
||||
const clickConfig = () => {
|
||||
if (!isConfig.value) {
|
||||
messageContent.value = loadConfig();
|
||||
} else {
|
||||
clearMessageContent();
|
||||
}
|
||||
switchConfigStatus();
|
||||
};
|
||||
|
||||
|
||||
const switchChatGPT = () => {
|
||||
isChatgpt.value = !isChatgpt.value;
|
||||
if (isChatgpt.value) {
|
||||
title.value = "ChatGPT"
|
||||
roleAlias.value = roleAliasChatGPT
|
||||
const apiKey = loadConfig();
|
||||
if (!apiKey) {
|
||||
isConfig.value = true;
|
||||
}
|
||||
} else {
|
||||
isConfig.value = false;
|
||||
title.value = "HuggingGPT"
|
||||
roleAlias.value = roleAliasChatHuggingGPT
|
||||
}
|
||||
};
|
||||
|
||||
function saveConfig(apiKey: string) {
|
||||
if (apiKey.slice(0, 3) !== "sk-" || apiKey.length !== 51) {
|
||||
alert("Illegal API Key");
|
||||
return false;
|
||||
}
|
||||
localStorage.setItem("apiKey", apiKey);
|
||||
return true;
|
||||
}
|
||||
|
||||
function loadConfig() {
|
||||
return localStorage.getItem("apiKey") ?? "";
|
||||
}
|
||||
|
||||
function scrollToBottom() {
|
||||
if (!chatListDom.value) return;
|
||||
// scrollTo(0, chatListDom.value.scrollHeight);
|
||||
chatListDom.value.scrollIntoView(false);
|
||||
}
|
||||
|
||||
function switchConfigStatus() {
|
||||
isConfig.value = !isConfig.value;
|
||||
}
|
||||
|
||||
function clearMessageContent() {
|
||||
messageContent.value = "";
|
||||
}
|
||||
|
||||
// const generateScreenshot = async ()=>{
|
||||
// const canvas = await html2canvas(pdf.value)
|
||||
// let a = new jsPDF("p", "mm", "a4")
|
||||
// //
|
||||
// a.addImage(canvas.toDataURL("image/png"), "PNG", 0, 0, 211, 298);
|
||||
// a.save("screenshot.pdf")
|
||||
// }
|
||||
|
||||
watch(mode, ()=> {
|
||||
if (isChatgpt.value) {
|
||||
messageList.value = promptCollection["chatgpt"][mode.value]
|
||||
} else {
|
||||
messageList.value = promptCollection["hugginggpt"][mode.value]
|
||||
}
|
||||
})
|
||||
|
||||
watch(isChatgpt, () => {
|
||||
if (isChatgpt.value) {
|
||||
mode.value = "default"
|
||||
messageList.value = promptCollection["chatgpt"]["default"]
|
||||
} else {
|
||||
mode.value = "default"
|
||||
messageList.value = promptCollection["hugginggpt"]["default"]
|
||||
}
|
||||
});
|
||||
|
||||
// messageList -> messageListMM
|
||||
watch(messageListMM, () => nextTick(() => {
|
||||
nextTick(()=>scrollToBottom())
|
||||
}));
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex flex-row justify-center verflow-auto">
|
||||
<!-- <button @click="generateScreenshot">Generate Screenshot</button> -->
|
||||
<div class="flex flex-col h-screen max-w-lg border-x-2 border-slate-200">
|
||||
|
||||
|
||||
<div class="flex flex-col h-20">
|
||||
<div class="flex flex-nowrap fixed max-w-lg w-full items-center justify-between top-0 px-6 py-6 bg-gray-100 z-50 h-20">
|
||||
<div class="font-bold w-1/4">
|
||||
<!-- <img src="@/assets/chatgpt.svg" class="w-7 mr-1 inline"/>
|
||||
x
|
||||
<img src="@/assets/huggingface.svg" class="w-8 ml-1 inline"/> -->
|
||||
<img src="@/assets/logo.svg" class="w-24 ml-1 inline"/>
|
||||
</div>
|
||||
|
||||
<div class="text-2xl font-bold w-1/2 flex justify-center">
|
||||
<RouterLink to="/">{{title}}</RouterLink>
|
||||
</div>
|
||||
|
||||
<div class="text-sm cursor-pointer w-1/4 flex flex-row justify-end" @click="!isChatgpt || clickConfig()" @dblclick="switchChatGPT()">
|
||||
<img src="@/assets/setting.svg" class="w-7 block" title="click to switch to configuration OpenAI key or double click to switch HuggingGPT and ChatGPT"/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex-1 overflow-auto" ref="pdf">
|
||||
<div class="m-5" ref="chatListDom">
|
||||
<div class="relative border-2 rounded-xl p-3" :class="{'bg-violet-50':item.role=='user', 'bg-blue-50':item.role=='assistant', 'bg-yellow-50':item.role=='system', 'mt-4': item.first, 'mt-1': !item.first }" v-for="item of messageListMM" >
|
||||
<svg xmlns="http://www.w3.org/2000/svg" v-if="!item.first" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 absolute -top-4 left-4 stroke-slate-400">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M18.375 12.739l-7.693 7.693a4.5 4.5 0 01-6.364-6.364l10.94-10.94A3 3 0 1119.5 7.372L8.552 18.32m.009-.01l-.01.01m5.699-9.941l-7.81 7.81a1.5 1.5 0 002.112 2.13" />
|
||||
</svg>
|
||||
<div v-if="item.first" class="font-bold text-sm mb-3 inline">{{roleAlias[item.role]}} :</div>
|
||||
<span
|
||||
class="text-sm text-slate-600 whitespace-pre-wrap leading-relaxed"
|
||||
v-if="item.content && item.type === 'text'"
|
||||
><div class="break-words" v-html="item.content" ></div>
|
||||
</span>
|
||||
<img
|
||||
class="text-sm text-slate-600 whitespace-pre-wrap leading-relaxed"
|
||||
v-else-if="item.content && item.type === 'image'" :src="item.content"
|
||||
/>
|
||||
<audio controls class="w-full text-blue-100" v-else-if="item.content && item.type === 'audio'" :src="item.content">
|
||||
</audio>
|
||||
|
||||
<video class="w-full" v-else-if="item.content && item.type === 'video'" controls>
|
||||
<source :src="item.content" type="video/mp4">
|
||||
</video>
|
||||
|
||||
<pre class="" v-else-if="item.content && item.type === 'code'">
|
||||
<code>
|
||||
{{item.content}}
|
||||
</code>
|
||||
</pre>
|
||||
|
||||
<Loading class="mt-2" v-else />
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="sticky bottom-0 w-full p-3 bg-gray-100">
|
||||
<div class="-mt-2 m-1 text-sm text-gray-500" v-if="isConfig">
|
||||
Please input OpenAI key:
|
||||
</div>
|
||||
<div class="flex">
|
||||
<textarea
|
||||
rows="2"
|
||||
style="resize:none"
|
||||
class="input"
|
||||
type="text"
|
||||
:placeholder="isConfig ? 'sk-xxxxxxxxxx' : 'Input your message'"
|
||||
v-model="messageContent"
|
||||
@keydown.enter.prevent="isTalking || sendOrSave()"
|
||||
>
|
||||
</textarea>
|
||||
<!-- <input
|
||||
class="input"
|
||||
type="text"
|
||||
:placeholder="isConfig ? 'sk-xxxxxxxxxx' : 'Input your message'"
|
||||
v-model="messageContent"
|
||||
@keydown.enter="isTalking || sendOrSave()"
|
||||
/> -->
|
||||
<div class="flex flex-col justify-center">
|
||||
<select v-model="mode" class="text-sm input w-20 m-1 h-7 p-1">
|
||||
<option :selected="m=='default'" v-for="m in Object.keys(promptCollection[isChatgpt?'chatgpt':'hugginggpt'])">{{m}}</option>
|
||||
</select>
|
||||
|
||||
<button
|
||||
class="btn bg-green-700 hover:bg-green-800 disabled:bg-green-400 focus:bg-green-800 text-sm w-20 m-1 h-7 p-1"
|
||||
:disabled="!messageList[messageList.length - 1].content"
|
||||
@click="sendOrSave()"
|
||||
>
|
||||
{{ isConfig ? "Save" : "Submit" }}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
pre {
|
||||
font-family: -apple-system, "Noto Sans", "Helvetica Neue", Helvetica,
|
||||
"Nimbus Sans L", Arial, "Liberation Sans", "PingFang SC", "Hiragino Sans GB",
|
||||
"Noto Sans CJK SC", "Source Han Sans SC", "Source Han Sans CN",
|
||||
"Microsoft YaHei", "Wenquanyi Micro Hei", "WenQuanYi Zen Hei", "ST Heiti",
|
||||
SimHei, "WenQuanYi Zen Hei Sharp", sans-serif;
|
||||
}
|
||||
audio {
|
||||
width: 100%;
|
||||
background-color: #fff;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 0.25rem;
|
||||
padding: 0.25rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar {
|
||||
/*隐藏滚轮*/
|
||||
display: none;
|
||||
}
|
||||
|
||||
</style>
|
||||
@@ -0,0 +1,8 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
module.exports = {
|
||||
content: ["./index.html", "./src/**/*.{vue,js,ts,jsx,tsx}"],
|
||||
theme: {
|
||||
extend: {},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "@vue/tsconfig/tsconfig.node.json",
|
||||
"include": ["vite.config.*", "vitest.config.*", "cypress.config.*"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"extends": "@vue/tsconfig/tsconfig.web.json",
|
||||
"include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
|
||||
"references": [
|
||||
{
|
||||
"path": "./tsconfig.config.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { fileURLToPath, URL } from "node:url";
|
||||
import path from "path";
|
||||
import { defineConfig } from "vite";
|
||||
import vue from "@vitejs/plugin-vue";
|
||||
|
||||
// https://vitejs.dev/config/
|
||||
export default defineConfig({
|
||||
server: {
|
||||
host: "0.0.0.0",
|
||||
port: 9999,
|
||||
open: true,
|
||||
cors: true,
|
||||
},
|
||||
plugins: [vue()],
|
||||
base: "./",
|
||||
resolve: {
|
||||
alias: {
|
||||
"@": path.resolve(__dirname, "./src"),
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,360 @@
|
||||
<p align="center">
|
||||
<img src="./assets/logo2.png" width="20%"> <br>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
<h1>TaskBench</h1>
|
||||
<div align="center">
|
||||
<a href="https://opensource.org/licenses/Apache-2.0">
|
||||
<img alt="License: Apache 2.0" src="https://img.shields.io/badge/License-Apache%202.0-4E94CE.svg">
|
||||
</a>
|
||||
<a href="https://arxiv.org/abs/2311.18760">
|
||||
<img alt="License: Apache 2.0" src="https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg">
|
||||
</a>
|
||||
</div>
|
||||
<h3>Benchmarking Large Language Models for Task Automation<h3>
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
<img width="100%" alt="image" src="./assets/eval.png">
|
||||
</p>
|
||||
|
||||
|
||||
## What's New
|
||||
|
||||
+ [2023.11.30] We release TaskBench for evaluating the task automation capability of LLMs.
|
||||
+ The code and datasets are available at [TaskBench](#).
|
||||
+ The paper is available at [TaskBench: Benchmarking Large Language Models for Task Automation](https://arxiv.org/abs/2311.18760).
|
||||
|
||||
## Table of Contents
|
||||
|
||||
+ [Introduction](#introduction)
|
||||
+ [Dataset](#dataset)
|
||||
+ [Evaluation with TaskBench](#evaluation-with-taskbench)
|
||||
+ [Dataset Construction with Back-Instruct](#dataset-construction-with-back-instruct)
|
||||
+ [Leaderboard](#leaderboard)
|
||||
+ [Citation](#citation)
|
||||
|
||||
## Introduction
|
||||
|
||||
TaskBench is a benchmark for evaluating large language models (LLMs) on task automation. Task automation can be formulated into three critical stages: task decomposition, tool invocation, and parameter prediction. This complexity makes data collection and evaluation more challenging compared to common NLP tasks. To address this challenge, we propose a comprehensive evaluation framework and a high-quality dataset for task automation. We also provide a leaderboard of 17 LLMs on TaskBench, including GPT-4, Claude-2, and other open-source LLMs.
|
||||
|
||||
|
||||
### Dataset
|
||||
|
||||
To generate high-quality evaluation datasets, we introduce the concept of Tool Graph to represent the decomposed tasks in user intent, and adopt a Back-Instruct method to simulate user instruction and annotations. The data collection process consists of three stages:
|
||||
|
||||
+ **Tool Graph Construction:** we first build a tool library and use the tool library to construct the tool graph. The nodes in the tool graph represent the tools, and the edges represent the dependencies between the tools, including the resource dependency and temporal dependency.
|
||||
+ **Graph Sampling:** we sample the tool graph to generate the tool graph for each sample. The sampled tool graph is used to generate the tool invocation graph and the instruction. According to the topology of the sampled tool graph, we sample the tool graph in three ways: node, chain and DAGs, which represent different structures of task decomposition for task automation.
|
||||
+ **Back-Instruct:** we first use the sampled tool graph to generate the task steps and the instructions. Then, we use the instruction to generate the tool invocation parameters to complete the tool invocation graph.
|
||||
|
||||
<p align="center">
|
||||
<img width="100%" alt="image" src="./assets/backinstruct.png">
|
||||
</p>
|
||||
|
||||
To improve the quality of the dataset, we use LLM-based and rule-based critics to verify the dataset. The former aims to use LLM to check the alignments between the generated data and the sampled tool graph. While the latter uses straightforward rules to determine the alignment between the tool graphs in created data and the sampled tool graphs. Here, we use the nodes and edges of the sampled graph to determine the consistency. Details statistics of the processing are shown in [the table](#LLM-based-and-Rule-based-Critics).
|
||||
|
||||
After LLM-based and rule-based critics, we further verify the dataset with human annotators, including checking the syntax of the instructions, the correctness of the tool invocation graph, and the correctness of the tool invocation parameters. The final dataset contains 28,271 samples in three domains: HuggingFace Tools, Multimedia Tools, and Daily Life APIs. Details statistics of the human verification are shown in [the table](#Human-Verification).
|
||||
|
||||
#### Introduction
|
||||
|
||||
The TaskBench dataset contains datasets in three areas: HuggingFace Tools, Multimedia Tools, and Dailylife APIs. Each dataset directory includes three files:
|
||||
|
||||
+ `data.json`: the dataset file, which contains the samples in the dataset.
|
||||
+ `graph_desc.json`: the tool graph description file, which contains the tool graph of the dataset.
|
||||
+ `user_requests.json`: contains the user requests of the dataset.
|
||||
+ `tool_desc.json`: the tool description file, which contains the tool descriptions of the dataset.
|
||||
|
||||
```
|
||||
├─data_dailylifeapis
|
||||
│ data.json
|
||||
│ graph_desc.json
|
||||
│ user_requests.json
|
||||
│ tool_desc.json
|
||||
│
|
||||
├─data_huggingface
|
||||
│ data.json
|
||||
│ graph_desc.json
|
||||
│ user_requests.json
|
||||
│ tool_desc.json
|
||||
│
|
||||
└─data_multimedia
|
||||
data.json
|
||||
graph_desc.json
|
||||
user_requests.json
|
||||
tool_desc.json
|
||||
```
|
||||
|
||||
#### Processing Statistics
|
||||
|
||||
We provide the statistics of the dataset processing in the following tables:
|
||||
|
||||
+ **Overview**: we provide the number of samples in each dataset, the number of samples checked by critics, and the number of samples verified by humans. Grouped by the tool invocation graph structure, e.g. node, chain, and DAGs, we also provide the number of samples in each group.
|
||||
+ **LLM-based and Rule-based Critics**: we provide the number of samples checked by LLM-based critics, rule-based critics and both critics.
|
||||
+ **Human Verification**: Human verification is built on the samples checked by critics, which includes three parts: syntax checking, instruction checking, and tool invocation graph checking. We provide the number of samples in each part, and along with the number of samples that are discarded or fixed.
|
||||
|
||||
| Dataset | #Samples | #Samples Checked by Critics (%) | #Samples Verified by Humans (%) | Node | Chain | DAG |
|
||||
| :-----: | :------: | :----------------: | :--------------: | :------: | :------: | :------: |
|
||||
| Hugging Face Models | 12,217 | 8,457 (69.22%) | 7,546 (61.76%) | 3,067 | 3,642 | 837 |
|
||||
| Multimedia Tools | 8,904 | 6,281 (70.54%) | 5,584 (62.71%) | 2,037 | 2,982 | 565 |
|
||||
| Dailylife APIs | 7,150 | 5,432 (75.97%) | 4,320 (60.42%) | 1,258 | 2,787 | 275 |
|
||||
|
||||
<div id="LLM-based-and-Rule-based-Critics">
|
||||
|
||||
| Dataset | #Samples | #Checked by LLM-based Critics (%) | #Checked by Rule-based Critics (%) | #Checked by Both Critics (%) |
|
||||
| :-----: | :------: | :-----------------------------: | :------------------------------: | :-------------------------: |
|
||||
| Hugging Face Models | 12,217 | 9,042 (74.01%) | 10,289 (84.22%) | 8,457 (69.22%) |
|
||||
| Multimedia Tools | 8,904 | 6,959 (78.16%) | 7,363 (82.69%) | 6,281 (70.54%) |
|
||||
| Dailylife APIs | 7,150 | 5,694 (79.63%) | 6,271 (87.70%) | 5,432 (75.97%) |
|
||||
|
||||
<div id="Human-Verification">
|
||||
|
||||
| Dataset | #Samples Checked by Critics | #Correct Samples (%) | #Discarded (%) | #Fixed for Syntax (%) | #Fixed for Instructions (%) | #Fixed for Tool Invocation Graph (%) |
|
||||
| :-----: | :-------------------------: | :-------------------: | :-------------------: | :---------------------------: | :-----------------------------------: | :------------: |
|
||||
| Hugging Face Models | 8,457 | 6,974 (82.46%) | 911 (10.77%) | 27 (0.32%) | 328 (3.87%) | 843 (9.96%) |
|
||||
| Multimedia Tools | 6,281 | 5,262 (83.77%) | 697 (11.09%) | 11 (0.17%) | 107 (1.70%) | 526 (9.96%) |
|
||||
| Dailylife APIs | 5,432 | 4,307 (79.29%) | 714 (13.14%) | 6 (0.11%) | 92 (1.68%) | 332 (6.11%) |
|
||||
|
||||
## Evaluation with TaskBench
|
||||
|
||||
On top of the TaskBench dataset, we provide a comprehensive evaluation framework for task automation. The evaluation framework consists of three stages: task decomposition, tool invocation, and parameter prediction. We provide the evaluation metrics for each stage:
|
||||
|
||||
+ **Task Decomposition**: Since task steps are diverse text distributions, we use the Rouge-1 (R1), Rouge-2 (R2), and Bertscore F1 (BsF) metrics to evaluate the task decomposition results.
|
||||
+ **Tool Invocation**: We report the F1 of node prediction (n-F1) and edge prediction (e-F1) in the tool invocation graph to evaluate the tool invocation results. Edge prediction reflects the correctness of the dependencies between tools, while node prediction reflects the correctness of the tool prediction.
|
||||
+ **Parameter Prediction**: For tool parameters prediction, we report the parameter type (or name) F1 (t-F1) and parameter value F1 (v-F1).
|
||||
|
||||
To evaluate the task automation performance of LLMs on TaskBench we provide the evaluation code and data, please follow the instructions below:
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
conda create -n taskbench python=3.8
|
||||
conda activate taskbench
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Additionally, if you wish to evaluate open-source large language models, you will also need to deploy the LLMs locally using an **OpenAI-compatible API**. We recommend using the `fastchat` tool to deploy the service to the `localhost:4000` endpoint.
|
||||
|
||||
```bash
|
||||
pip install fastchat
|
||||
pip install vllm
|
||||
pip install "fastapi[all]"
|
||||
|
||||
python3 -m fastchat.serve.controller
|
||||
python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-7b-v1.3
|
||||
python3 -m fastchat.serve.openai_api_server --host localhost --port 4000
|
||||
```
|
||||
|
||||
### Inference
|
||||
|
||||
For convenience, it is recommended to deploy all LLMs to the same endpoint, such as `localhost:4000`. To generate the prediction file on TaskBench, specify the name of the LLM using the following command:
|
||||
|
||||
```bash
|
||||
export YOUR_API_KEY=API_KEY
|
||||
python inference.py \
|
||||
--llm gpt-4 \
|
||||
--data_dir data_multimedia \
|
||||
--temperature 0.2 \
|
||||
--top_p 0.1 \
|
||||
--api_addr localhost \
|
||||
--api_port 4000 \
|
||||
--api_key $YOUR_API_KEY \
|
||||
--multiworker 5 \
|
||||
--use_demos 0 \
|
||||
--reformat true \
|
||||
--reformat_by self \
|
||||
--log_first_detail true \
|
||||
--use_demos 2 \
|
||||
--dependency_type resource \
|
||||
--tag true
|
||||
```
|
||||
|
||||
### Evaluation
|
||||
|
||||
With the predictions in place, you can now evaluate the LLMs. The predictions file is saved by default in the dataset's folder under the name `predictions`. Execute the following command to calculate the evaluation metrics (saved in the `metrics` folder):
|
||||
|
||||
```bash
|
||||
python evaluate.py \
|
||||
--data_dir data_multimedia \
|
||||
--prediction_dir $prediction_dir \
|
||||
--llm gpt-4 \
|
||||
--splits all \
|
||||
--n_tools all \
|
||||
--mode add \
|
||||
--dependency_type resource \
|
||||
-m all
|
||||
```
|
||||
|
||||
## Dataset Construction with Back-Instruct
|
||||
|
||||
We have provided the dataset for three domains: Hugging Face Tools (`data_huggingface`), Multimedia Tools (`data_multimedia`), and Daily Life APIs (`data_dailylifeapis`). If you want to generate your own dataset, please follow the instructions below:
|
||||
|
||||
### Construct Your Own Tool Graph
|
||||
|
||||
First, you need to build your own tool library. The tool library is a JSON file that contains the description of the tools and tool parameters. Two formats of the tool are supported:
|
||||
|
||||
```json
|
||||
// Tool with type-specific parameters
|
||||
{
|
||||
"id": "Image-to-Image",
|
||||
"desc": "Image-to-image is the task of transforming a source image to match the characteristics of a target image or a target image domain. Any image manipulation and enhancement is possible with image to image models.",
|
||||
"input-type": [
|
||||
"image"
|
||||
],
|
||||
"output-type": [
|
||||
"image"
|
||||
]
|
||||
}
|
||||
// API with request parameters
|
||||
{
|
||||
"id": "send_sms",
|
||||
"desc": "Send an sms to a specific phone number",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "phone_number",
|
||||
"type": "string",
|
||||
"desc": "The phone number to send the sms to"
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"type": "string",
|
||||
"desc": "The content of the sms"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Then based on the tool library, you can use the script `generate_graph.py` to generate the tool graph. Now we support two type of tool graph: resource dependency graph and temporal dependency graph. For type-specific parameters, we use the resource dependency graph. For API with request parameters, we use the temporal dependency graph. You can specify the tool graph type by the parameter `--dependency_type`. In the future, we will support more types of tool graphs.
|
||||
|
||||
```bash
|
||||
python generate_graph.py \
|
||||
--tool_desc tool_desc.json \
|
||||
--dependency_type resource \
|
||||
--data_dir data_multimedia
|
||||
```
|
||||
|
||||
> Note: The auto-generated tool graph may not be perfect. You can manually modify the tool graph to make it more reasonable. You can check the tool graph through the visualization tool `visualize_graph.py`. We recommend that you manually create the tool graph thoroughly, which will help you to generate a high-quality dataset.
|
||||
|
||||
### Generate the Dataset
|
||||
|
||||
After generating the tool graph, you can use the script `data_engine.py` to generate the dataset. You need to specify the tool graph description file to `--graph_desc` and the tool description file to `--tool_desc`.
|
||||
|
||||
```bash
|
||||
# specify the graph and tool description file
|
||||
python data_engine.py \
|
||||
--graph_desc data_multimedia/graph_desc.json \
|
||||
--tool_desc data_multimedia/tool_desc.json \
|
||||
--llm gpt-4 \
|
||||
--temperature 1.0 \
|
||||
--top_p 1.0 \
|
||||
--dependency_type resource \
|
||||
--save_figure false \
|
||||
--api_addr localhost \
|
||||
--api_port 4000 \
|
||||
--api_key $YOUR_API_KEY \
|
||||
--check true \
|
||||
--use_async true \
|
||||
--multiworker 5
|
||||
|
||||
# specify the data_dir to resume data generation
|
||||
python data_engine.py \
|
||||
--data_dir result_20240317170826_gpt-4-32k_t1_0_p1_0_check \
|
||||
--llm gpt-4-32k \
|
||||
--temperature 1.0 \
|
||||
--top_p 1.0 \
|
||||
--dependency_type temporal \
|
||||
--save_figure false \
|
||||
--api_addr localhost \
|
||||
--api_port 4000 \
|
||||
--api_key $YOUR_API_KEY \
|
||||
--check true \
|
||||
--use_async true \
|
||||
--multiworker 5
|
||||
|
||||
python format_data.py \
|
||||
--data_dir data_multimedia \
|
||||
--dependency_type resource
|
||||
```
|
||||
|
||||
## Leaderboard
|
||||
|
||||
Based on the evaluation framework and the TaskBench dataset, we provide a leaderboard of task automation performance of 17 LLMs. We provide the evaluation results of each LLM in the following tables:
|
||||
|
||||
### Multimedia Tools Domain
|
||||
|
||||
| LLM | R1 | R2 | BsF | n-F1 | e-F1 | t-F1 | v-F1 |
|
||||
|----------------------|-------|-------|------|------|------|------|------|
|
||||
| gpt-4 | 60.84 | 40.08 | 91.19 | 90.90 | 69.27 | 87.06 | 72.31 |
|
||||
| claude-2 | 48.85 | 23.59 | 89.22 | 80.94 | 53.01 | 71.63 | 51.58 |
|
||||
| gpt-3.5-turbo | 49.66 | 28.51 | 89.54 | 72.83 | 44.02 | 65.91 | 40.80 |
|
||||
| text-davinci-003 | 49.23 | 27.97 | 89.21 | 73.97 | 45.81 | 68.48 | 40.70 |
|
||||
| codellama-13b | 44.46 | 23.30 | 88.66 | 62.78 | 24.61 | 48.19 | 29.13 |
|
||||
| codellama-7b | 43.76 | 22.93 | 88.81 | 53.29 | 14.76 | 38.04 | 24.45 |
|
||||
| vicuna-13b-v1.5 | 44.75 | 23.75 | 88.94 | 60.61 | 14.78 | 41.62 | 23.62 |
|
||||
| nous-hermes-13b | 35.73 | 16.11 | 87.53 | 58.97 | 8.90 | 43.60 | 21.69 |
|
||||
| wizardlm-13b | 35.87 | 17.55 | 87.29 | 51.24 | 4.82 | 39.10 | 18.74 |
|
||||
| vicuna-7b-v1.5 | 39.46 | 19.83 | 88.53 | 46.06 | 4.26 | 29.72 | 13.74 |
|
||||
| longchat-7b-v1.5 | 37.85 | 18.14 | 87.64 | 43.08 | 3.95 | 27.89 | 13.41 |
|
||||
| baichuan-13b-chat | 20.41 | 3.77 | 83.31 | 42.51 | 5.19 | 28.04 | 11.77 |
|
||||
| llama-2-13b-chat | 26.16 | 7.88 | 84.82 | 43.87 | 1.63 | 29.99 | 11.32 |
|
||||
| internlm-chat-7b | 16.64 | 3.56 | 82.91 | 23.60 | 1.14 | 13.75 | 6.09 |
|
||||
| llama-2-7b-chat | 34.51 | 15.91 | 87.56 | 26.47 | 0.91 | 18.27 | 5.84 |
|
||||
| mpt-7b-chat | 30.94 | 11.90 | 86.08 | 8.68 | 0.18 | 3.19 | 1.02 |
|
||||
| vicuna-33b-v1.3 | 31.27 | 13.37 | 86.17 | 6.40 | 0.01 | 2.47 | 1.09 |
|
||||
|
||||
### HuggingFace Tools Domain
|
||||
|
||||
| LLM | R1 | R2 | BsF | n-F1 | e-F1 | t-F1 | v-F1 |
|
||||
|----------------------|-------|-------|------|------|------|------|------|
|
||||
| gpt-4 | 52.42 | 30.38 | 90.12 | 81.54 | 54.70 | 77.31 | 60.86 |
|
||||
| claude-2 | 44.21 | 21.12 | 88.71 | 79.00 | 43.51 | 63.00 | 43.08 |
|
||||
| text-davinci-003 | 36.68 | 17.61 | 87.03 | 59.38 | 29.37 | 52.53 | 36.04 |
|
||||
| gpt-3.5-turbo | 42.99 | 21.58 | 88.47 | 69.49 | 33.36 | 55.88 | 36.32 |
|
||||
| codellama-13b | 38.75 | 18.37 | 88.32 | 53.16 | 14.64 | 32.06 | 18.87 |
|
||||
| nous-hermes-13b | 37.36 | 16.91 | 88.18 | 53.62 | 8.29 | 37.51 | 17.66 |
|
||||
| wizardlm-13b | 34.47 | 15.38 | 87.38 | 54.40 | 2.05 | 38.76 | 15.35 |
|
||||
| llama-2-13b-chat | 39.37 | 18.64 | 88.67 | 48.47 | 7.30 | 31.61 | 15.38 |
|
||||
| longchat-7b-v1.5 | 27.09 | 8.97 | 85.50 | 48.18 | 0.56 | 33.57 | 13.94 |
|
||||
| baichuan-13b-chat | 19.93 | 5.97 | 83.85 | 53.85 | 7.65 | 33.17 | 13.53 |
|
||||
| vicuna-13b-v1.5 | 37.12 | 17.03 | 87.90 | 50.82 | 7.28 | 28.34 | 11.85 |
|
||||
| vicuna-7b-v1.5 | 27.17 | 10.02 | 85.61 | 42.87 | 2.76 | 24.65 | 10.81 |
|
||||
| vicuna-33b-v1.3 | 33.52 | 14.75 | 86.73 | 43.40 | 4.82 | 22.71 | 10.07 |
|
||||
| codellama-7b | 38.97 | 18.62 | 88.46 | 37.59 | 5.35 | 22.50 | 9.20 |
|
||||
| internlm-chat-7b | 20.53 | 7.16 | 83.74 | 24.39 | 0.83 | 15.41 | 6.64 |
|
||||
| llama-2-7b-chat | 24.12 | 8.68 | 85.43 | 27.30 | 0.74 | 13.05 | 2.79 |
|
||||
| mpt-7b-chat | 33.21 | 12.73 | 87.23 | 20.86 | 0.12 | 9.61 | 1.83 |
|
||||
|
||||
### Daily Life APIs Domain
|
||||
|
||||
| LLM | R1 | R2 | BsF | n-F1 | e-F1 | t-F1 | v-F1 |
|
||||
|----------------------|-------|-------|------|------|------|------|------|
|
||||
| gpt-4 | 85.07 | 72.36 | 96.91 | 96.91 | 80.53 | 97.02 | 71.14 |
|
||||
| claude-2 | 82.26 | 69.88 | 96.64 | 93.52 | 75.31 | 92.71 | 64.72 |
|
||||
| codellama-13b | 89.86 | 83.27 | 97.90 | 87.73 | 63.16 | 84.26 | 62.38 |
|
||||
| gpt-3.5-turbo | 58.53 | 39.90 | 91.29 | 85.37 | 60.67 | 81.97 | 55.66 |
|
||||
| text-davinci-003 | 68.27 | 50.30 | 93.59 | 80.42 | 54.90 | 78.37 | 53.40 |
|
||||
| nous-hermes-13b | 78.49 | 68.04 | 95.61 | 73.45 | 3.50 | 64.47 | 47.22 |
|
||||
| vicuna-13b-v1.5 | 81.76 | 71.76 | 96.31 | 75.67 | 12.48 | 64.27 | 47.31 |
|
||||
| wizardlm-13b | 82.02 | 72.43 | 96.36 | 69.34 | 14.18 | 55.00 | 40.53 |
|
||||
| codellama-7b | 56.98 | 38.83 | 91.31 | 59.33 | 27.23 | 52.99 | 34.81 |
|
||||
| vicuna-33b-v1.3 | 54.96 | 39.71 | 91.40 | 52.49 | 16.37 | 39.95 | 29.64 |
|
||||
| vicuna-7b-v1.5 | 40.26 | 21.19 | 87.27 | 52.73 | 14.23 | 36.30 | 24.67 |
|
||||
| baichuan-13b-chat | 49.43 | 27.25 | 88.32 | 52.55 | 10.61 | 37.48 | 23.77 |
|
||||
| llama-2-13b-chat | 45.39 | 22.42 | 87.74 | 55.77 | 17.02 | 35.11 | 22.94 |
|
||||
| longchat-7b-v1.5 | 29.05 | 14.84 | 83.90 | 47.26 | 14.44 | 25.73 | 18.18 |
|
||||
| internlm-chat-7b | 42.94 | 21.02 | 86.14 | 29.14 | 6.63 | 19.21 | 13.48 |
|
||||
| llama-2-7b-chat | 37.06 | 16.49 | 86.31 | 30.17 | 4.27 | 14.94 | 9.34 |
|
||||
| mpt-7b-chat | 44.54 | 20.98 | 87.17 | 15.95 | 1.69 | 5.34 | 3.45 |
|
||||
|
||||
More details can be found in our paper: [TaskBench: Benchmarking Large Language Models for Task Automation](https://arxiv.org/abs/2311.18760).
|
||||
|
||||
## Citation
|
||||
|
||||
If you find this work useful in your method, you can cite the paper as below:
|
||||
|
||||
@article{shen2023taskbench,
|
||||
title = {TaskBench: Benchmarking Large Language Models for Task Automation},
|
||||
author = {Shen, Yongliang and Song, Kaitao and Tan, Xu and Zhang, Wenqi and Ren, Kan and Yuan, Siyu and Lu, Weiming and Li, Dongsheng and Zhuang, Yueting},
|
||||
journal = {arXiv preprint arXiv:2311.18760},
|
||||
year = {2023}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 404 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 496 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 992 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 692 KiB |
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -e
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
|
||||
data_dir=$1
|
||||
prediction_dir=$2
|
||||
|
||||
if [[ $data_dir == *"dailylifeapis"* ]]; then
|
||||
dependency_type="temporal"
|
||||
else
|
||||
dependency_type="resource"
|
||||
fi
|
||||
|
||||
for file in $data_dir/$prediction_dir/*.json
|
||||
do
|
||||
llm=$(basename $file .json)
|
||||
# replace prediction_dir's "predictions" with "metrics"
|
||||
metrics=$(echo $prediction_dir | sed 's/predictions/metrics/g')
|
||||
if [ -f $data_dir/$metrics/${llm}_splits_all_tools_all_metric_all.json ] && [ -s $data_dir/$metrics/${llm}_splits_all_tools_all_metric_all.json ];
|
||||
then
|
||||
continue
|
||||
fi
|
||||
echo $llm
|
||||
python evaluate.py --data_dir $data_dir --prediction_dir $prediction_dir --llm $llm --splits all --n_tools all --mode add --dependency_type $dependency_type -m all
|
||||
done
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user